From f80229b530c6325d3514547912c90cbbceca2b60 Mon Sep 17 00:00:00 2001 From: Arne Gevaert Date: Mon, 30 Oct 2017 16:03:40 +0100 Subject: [PATCH 01/18] deleted datex spec example --- datex_spec.xml | 78 -------------------------------------------------- 1 file changed, 78 deletions(-) delete mode 100644 datex_spec.xml diff --git a/datex_spec.xml b/datex_spec.xml deleted file mode 100644 index 059659a..0000000 --- a/datex_spec.xml +++ /dev/null @@ -1,78 +0,0 @@ - - - - - - - - Parkings per stad - - - **ISO TIME** - - - - **STAD** - - - - - - P07 Sint-Michiels - - - - - Sint-Michiels - - - 450 - - - - 1.0 - 2.0 - - - - - - - Sint-Michielsplein 8 9000 Gent - - - Tel.: 09 266 29 20 - - - - active - - - - 00:00:00.000+02:00 - 23:59:00.000+02:00 - - - - - - - ... - - - - - - - **ISO TIMESTAMP** - - 164 - - spacesAvailable - open - - ... - - - - \ No newline at end of file From dffc57461ffeceaf794bef63b787fd3a17012912 Mon Sep 17 00:00:00 2001 From: Arne Gevaert Date: Mon, 30 Oct 2017 16:56:19 +0100 Subject: [PATCH 02/18] added settings class --- src/Settings.php | 78 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 78 insertions(+) create mode 100644 src/Settings.php diff --git a/src/Settings.php b/src/Settings.php new file mode 100644 index 0000000..3f15e09 --- /dev/null +++ b/src/Settings.php @@ -0,0 +1,78 @@ +dotenv = new Dotenv(__DIR__ . '/../'); + $this->dotenv->load(); + + $this->required(); + $this->parseDatasets(); + + $this->data_dir = $_ENV["DATA_DIR"]; + $this->resource_dir = $_ENV["RESOURCE_DIR"]; + $this->default_gather_interval = $_ENV["DEFAULT_GATHER_INTERVAL"]; + $this->range_gates_config = $_ENV["RANGE_GATES_CONFIG"]; + } + + function getDatasets() { + return $this->datasets; + } + + function getDatasetsGather() { + return $this->datasets_gather; + } + + function getDefaultGatherInterval() { + return $this->default_gather_interval; + } + + function getRangeGatesConfig() { + return $this->range_gates_config; + } + + private function required() { + $this->dotenv->required('DATA_DIR'); + $this->dotenv->required('RESOURCE_DIR'); + $this->dotenv->required('DATASETS'); + $this->dotenv->required('DATASETS_GATHER'); + $this->dotenv->required('DEFAULT_GATHER_INTERVAL'); + } + + private function parseDatasets() { + $datasets = explode(',', $_ENV['DATASETS']); + $datasets_gather = explode(',', $_ENV['DATASETS_GATHER']); + + foreach($datasets as $dataset) { + array_push($this->datasets, $this->parseDataset($dataset)); + } + + foreach($datasets_gather as $dataset) { + array_push($this->datasets_gather, $this->parseDataset($dataset)); + } + } + + private function parseDataset($dataset) { + try { + $this->dotenv->required($dataset . "_PATH"); + $class = $_ENV[$dataset . "_PATH"]; + return new $class; + } catch (\Exception $e) { + error_log("Invalid .env configuration: dataset " . $dataset . " has no corresponding class path." + . " Please add the variable " . $dataset . "_PATH."); + } + return null; + } +} \ No newline at end of file From 4178cd8e62daf1264420f35a5d92072070fa267e Mon Sep 17 00:00:00 2001 From: Arne Gevaert Date: Thu, 2 Nov 2017 18:49:50 +0100 Subject: [PATCH 03/18] more .env delegation --- public/index.php | 20 ++++++++++++-------- src/Router.php | 6 ++---- src/Settings.php | 9 +++++++++ 3 files changed, 23 insertions(+), 12 deletions(-) diff --git a/public/index.php b/public/index.php index 5fa8de3..89a42ed 100644 --- a/public/index.php +++ b/public/index.php @@ -14,8 +14,15 @@ $dotenv = new Dotenv(__DIR__ . '/../'); $dotenv->load(); -$processors = get_datasets_from_names(explode(',', $_ENV['DATASETS']), $dotenv); -$processors_gather = get_datasets_from_names(explode(',', $_ENV["DATASETS_GATHER"]), $dotenv); +$settings = new Settings(); +$processors = $settings->getDatasets(); +$processors_gather = $settings->getDatasetsGather(); +$out_dirname = $settings->getOutDir(); +$res_dirname = $settings->getResourcesDir(); +$second_interval = $settings->getDefaultGatherInterval(); + +//$processors = get_datasets_from_names(explode(',', $_ENV['DATASETS']), $dotenv); +//$processors_gather = get_datasets_from_names(explode(',', $_ENV["DATASETS_GATHER"]), $dotenv); $nameToGP = []; foreach($processors as $gp) { @@ -24,14 +31,11 @@ $nameToGP[$name_lower] = $gp; } -$out_dirname = __DIR__ . "/../out"; -$res_dirname = __DIR__ . "/../resources"; -$second_interval = 60*60*3; // TODO store this in .env! - -$router = new Router($_SERVER['HTTP_HOST'], $out_dirname, $res_dirname, $second_interval, $nameToGP, $processors_gather); +$router = new Router($out_dirname, $res_dirname, $second_interval, $nameToGP, $processors_gather); $router->init(); $router->run(); +/* function get_datasets_from_names($names, $dotenv) { $result = array(); foreach($names as $dataset) { @@ -45,4 +49,4 @@ function get_datasets_from_names($names, $dotenv) { } } return $result; -} \ No newline at end of file +}*/ \ No newline at end of file diff --git a/src/Router.php b/src/Router.php index 1b45358..2b65e1b 100644 --- a/src/Router.php +++ b/src/Router.php @@ -9,18 +9,16 @@ class Router { private $router; - private $http_host; private $out_dirname; private $res_dirname; private $second_interval; private $nameToGP; private $processors_gather; - public function __construct($http_host, $out_dirname, $res_dirname, $second_interval, $nameToGP, $processors_gather) { + public function __construct($out_dirname, $res_dirname, $second_interval, $nameToGP, $processors_gather) { $this->router = new Bramus\Router\Router(); $this->router->set404(function() {echo "Page not found.";}); - $this->http_host = $http_host; $this->out_dirname = $out_dirname; $this->res_dirname = $res_dirname; $this->second_interval = $second_interval; @@ -38,7 +36,7 @@ public function init() { $processors_gather = $this->processors_gather; $found = false; - $dataset_name = explode('.', $this->http_host)[0]; + $dataset_name = explode('.', $_SERVER['HTTP_HOST'])[0]; $dataset = null; $fs = null; $calc = null; foreach($this->nameToGP as $name => $gp) { if ($name === $dataset_name) { diff --git a/src/Settings.php b/src/Settings.php index 3f15e09..3da83a5 100644 --- a/src/Settings.php +++ b/src/Settings.php @@ -3,6 +3,7 @@ namespace oSoc\Smartflanders; use Dotenv\Dotenv; +use oSoc\Smartflanders\Helpers\IGraphProcessor; class Settings { @@ -27,6 +28,14 @@ function __construct() { $this->range_gates_config = $_ENV["RANGE_GATES_CONFIG"]; } + function getOutDir() { + return $this->data_dir; + } + + function getResourcesDir() { + return $this->resource_dir; + } + function getDatasets() { return $this->datasets; } From 1492a6682198ba45f1c27de5eb1179f23da8c624 Mon Sep 17 00:00:00 2001 From: Arne Gevaert Date: Thu, 2 Nov 2017 19:09:35 +0100 Subject: [PATCH 04/18] relative paths fixed --- public/index.php | 1 - src/Filesystem/FileSystemProcessor.php | 6 +++--- src/Settings.php | 8 ++++---- 3 files changed, 7 insertions(+), 8 deletions(-) diff --git a/public/index.php b/public/index.php index 89a42ed..a55bce2 100644 --- a/public/index.php +++ b/public/index.php @@ -30,7 +30,6 @@ $name_lower = strtolower($name); $nameToGP[$name_lower] = $gp; } - $router = new Router($out_dirname, $res_dirname, $second_interval, $nameToGP, $processors_gather); $router->init(); $router->run(); diff --git a/src/Filesystem/FileSystemProcessor.php b/src/Filesystem/FileSystemProcessor.php index 88b3a22..a33a806 100644 --- a/src/Filesystem/FileSystemProcessor.php +++ b/src/Filesystem/FileSystemProcessor.php @@ -28,11 +28,11 @@ public function __construct($out_dirname, $res_dirname, $second_interval, IGraph $this->res_dirname = $res_dirname; $this->second_interval = $second_interval; date_default_timezone_set("Europe/Brussels"); - $out_adapter = new Local($out_dirname . "/" . $graph_processor->getName()); + $out_adapter = new Local($this->out_dirname . "/" . $graph_processor->getName()); $this->out_fs = new Filesystem($out_adapter); - $res_adapter = new Local($res_dirname); + $res_adapter = new Local($this->res_dirname); $this->res_fs = new Filesystem($res_adapter); - $stat_adapter = new Local($out_dirname . "/" . $graph_processor->getName() . "/statistical"); + $stat_adapter = new Local($this->out_dirname . "/" . $graph_processor->getName() . "/statistical"); $this->stat_fs = new Filesystem($stat_adapter); $this->graph_processor = $graph_processor; $this->static_data_filename = $graph_processor->getName() . "_static_data.turtle"; diff --git a/src/Settings.php b/src/Settings.php index 3da83a5..9b8cd28 100644 --- a/src/Settings.php +++ b/src/Settings.php @@ -12,8 +12,8 @@ class Settings private $default_gather_interval; private $range_gates_config; private $dotenv; - private $datasets; - private $datasets_gather; + private $datasets = array(); + private $datasets_gather = array(); function __construct() { $this->dotenv = new Dotenv(__DIR__ . '/../'); @@ -22,8 +22,8 @@ function __construct() { $this->required(); $this->parseDatasets(); - $this->data_dir = $_ENV["DATA_DIR"]; - $this->resource_dir = $_ENV["RESOURCE_DIR"]; + $this->data_dir = __DIR__ . '/../' . $_ENV["DATA_DIR"]; + $this->resource_dir = __DIR__ . '/../' . $_ENV["RESOURCE_DIR"]; $this->default_gather_interval = $_ENV["DEFAULT_GATHER_INTERVAL"]; $this->range_gates_config = $_ENV["RANGE_GATES_CONFIG"]; } From c5d92712810c86d3c831ed5c4afcb8093eafaef1 Mon Sep 17 00:00:00 2001 From: Arne Gevaert Date: Thu, 2 Nov 2017 22:12:34 +0100 Subject: [PATCH 05/18] removed unnecessary code --- public/index.php | 25 +------------------------ 1 file changed, 1 insertion(+), 24 deletions(-) diff --git a/public/index.php b/public/index.php index a55bce2..f894df5 100644 --- a/public/index.php +++ b/public/index.php @@ -9,10 +9,6 @@ require __DIR__ . '/../vendor/autoload.php'; use oSoc\Smartflanders\Datasets; -use Dotenv\Dotenv; - -$dotenv = new Dotenv(__DIR__ . '/../'); -$dotenv->load(); $settings = new Settings(); $processors = $settings->getDatasets(); @@ -21,9 +17,6 @@ $res_dirname = $settings->getResourcesDir(); $second_interval = $settings->getDefaultGatherInterval(); -//$processors = get_datasets_from_names(explode(',', $_ENV['DATASETS']), $dotenv); -//$processors_gather = get_datasets_from_names(explode(',', $_ENV["DATASETS_GATHER"]), $dotenv); - $nameToGP = []; foreach($processors as $gp) { $name = $gp->getName(); @@ -32,20 +25,4 @@ } $router = new Router($out_dirname, $res_dirname, $second_interval, $nameToGP, $processors_gather); $router->init(); -$router->run(); - -/* -function get_datasets_from_names($names, $dotenv) { - $result = array(); - foreach($names as $dataset) { - try { - $dotenv->required($dataset . "_PATH"); - $class = $_ENV[$dataset . "_PATH"]; - array_push($result, new $class); - } catch (\Exception $e) { - error_log("Invalid .env configuration: dataset " . $dataset . " has no corresponding class path." - . " Please add the variable " . $dataset . "_PATH."); - } - } - return $result; -}*/ \ No newline at end of file +$router->run(); \ No newline at end of file From da7fbf538b1615200b4765c62bc6dc32eba8e876 Mon Sep 17 00:00:00 2001 From: Arne Gevaert Date: Fri, 3 Nov 2017 11:08:04 +0100 Subject: [PATCH 06/18] code cleanup using settings --- public/index.php | 13 +---- src/Filesystem/FileSystemProcessor.php | 15 +++--- src/Filesystem/FileWriter.php | 5 +- src/RangeGate/RangeGate.php | 4 -- src/RangeGate/RangeGateIntervalCalculator.php | 18 ------- src/Router.php | 34 +++++-------- src/View.php | 7 +-- src/index-singular.php | 51 ------------------- 8 files changed, 30 insertions(+), 117 deletions(-) delete mode 100644 src/index-singular.php diff --git a/public/index.php b/public/index.php index f894df5..199560e 100644 --- a/public/index.php +++ b/public/index.php @@ -11,18 +11,7 @@ use oSoc\Smartflanders\Datasets; $settings = new Settings(); -$processors = $settings->getDatasets(); -$processors_gather = $settings->getDatasetsGather(); -$out_dirname = $settings->getOutDir(); -$res_dirname = $settings->getResourcesDir(); -$second_interval = $settings->getDefaultGatherInterval(); -$nameToGP = []; -foreach($processors as $gp) { - $name = $gp->getName(); - $name_lower = strtolower($name); - $nameToGP[$name_lower] = $gp; -} -$router = new Router($out_dirname, $res_dirname, $second_interval, $nameToGP, $processors_gather); +$router = new Router($settings); $router->init(); $router->run(); \ No newline at end of file diff --git a/src/Filesystem/FileSystemProcessor.php b/src/Filesystem/FileSystemProcessor.php index a33a806..7a2e72a 100644 --- a/src/Filesystem/FileSystemProcessor.php +++ b/src/Filesystem/FileSystemProcessor.php @@ -7,6 +7,7 @@ use \League\Flysystem\Adapter\Local; use \League\Flysystem\Filesystem; use oSoc\Smartflanders\Helpers\IGraphProcessor; +use oSoc\Smartflanders\Settings; use pietercolpaert\hardf\TriGWriter; @@ -20,13 +21,15 @@ protected $static_data_filename; protected $out_dirname; protected $res_dirname; + protected $settings; const REFRESH_STATIC = false; - public function __construct($out_dirname, $res_dirname, $second_interval, IGraphProcessor $graph_processor) + public function __construct(Settings $settings, IGraphProcessor $graph_processor) { - $this->out_dirname = $out_dirname; - $this->res_dirname = $res_dirname; - $this->second_interval = $second_interval; + $this->settings = $settings; + $this->out_dirname = $settings->getOutDir(); + $this->res_dirname = $settings->getResourcesDir(); + $this->second_interval = $settings->getDefaultGatherInterval(); date_default_timezone_set("Europe/Brussels"); $out_adapter = new Local($this->out_dirname . "/" . $graph_processor->getName()); $this->out_fs = new Filesystem($out_adapter); @@ -122,10 +125,10 @@ public function hasFile($filename) { } public function getFileReader() { - return new FileReader($this->out_dirname, $this->res_dirname, $this->second_interval, $this->graph_processor); + return new FileReader($this->settings, $this->graph_processor); } public function getFileWriter() { - return new FileWriter($this->out_dirname, $this->res_dirname, $this->second_interval, $this->graph_processor); + return new FileWriter($this->settings, $this->graph_processor); } } diff --git a/src/Filesystem/FileWriter.php b/src/Filesystem/FileWriter.php index 216fbb8..2d32602 100644 --- a/src/Filesystem/FileWriter.php +++ b/src/Filesystem/FileWriter.php @@ -2,6 +2,7 @@ namespace oSoc\Smartflanders\Filesystem; +use oSoc\Smartflanders\Settings; use pietercolpaert\hardf\TriGWriter; use pietercolpaert\hardf\TriGParser; use pietercolpaert\hardf\Util; @@ -11,9 +12,9 @@ class FileWriter extends FileSystemProcessor { private $oldest_timestamp_filename; - public function __construct($out_dirname, $res_dirname, $second_interval, Helpers\IGraphProcessor $graph_processor) + public function __construct(Settings $settings, Helpers\IGraphProcessor $graph_processor) { - parent::__construct($out_dirname, $res_dirname, $second_interval, $graph_processor); + parent::__construct($settings, $graph_processor); $this->oldest_timestamp_filename = $this->graph_processor->getName() . "_oldest_timestamp"; } diff --git a/src/RangeGate/RangeGate.php b/src/RangeGate/RangeGate.php index 65f1d80..1aa55af 100644 --- a/src/RangeGate/RangeGate.php +++ b/src/RangeGate/RangeGate.php @@ -2,7 +2,6 @@ namespace oSoc\Smartflanders\RangeGate; -use Dotenv\Dotenv; use oSoc\Smartflanders\Filesystem\FileSystemProcessor; use oSoc\Smartflanders\Helpers\IGraphProcessor; use oSoc\Smartflanders\Helpers\TripleHelper; @@ -24,9 +23,6 @@ public function __construct($gatename, IGraphProcessor $dataset, FileSystemProce $this->dataset = $dataset; $this->fs = $fs; - $dotenv = new Dotenv(__DIR__ . '/../../'); - $dotenv->load(); - $this->intervalCalculator = new RangeGateIntervalCalculator($_ENV['RANGE_GATES_CONFIG'], $fs->getOldestTimestamp()); $this->baseUrl = $this->dataset->getBaseUrl() . '/rangegate/'; if ($this->gatename !== self::$ROOT_GATE) { diff --git a/src/RangeGate/RangeGateIntervalCalculator.php b/src/RangeGate/RangeGateIntervalCalculator.php index f6d5af7..20a2f8d 100644 --- a/src/RangeGate/RangeGateIntervalCalculator.php +++ b/src/RangeGate/RangeGateIntervalCalculator.php @@ -15,10 +15,6 @@ class RangeGateIntervalCalculator ); public function __construct($configString, $oldest_timestamp) { - // Round the oldest timestamp up to 1 day - /*$day = 60*60*24; - $rest = $oldest_timestamp % $day; - $this->oldest = $oldest_timestamp + $day - $rest - 2*60*60;*/ $this->oldest = \DateTime::createFromFormat('U', $oldest_timestamp); // Parse the configuration string @@ -47,13 +43,6 @@ public function isLegal($intervalString) { return false; } - // Determine if start is valid (must be a multiple of diff greater than oldest_timestamp) - //$start_relative = $interval[0] - $this->oldest; - /*$start_relative = $interval[0]->diff($this->oldest)->days; - var_dump($start_relative % $diff); - if ($start_relative % $diff !== 0) { - return false; - }*/ return true; } @@ -68,13 +57,6 @@ public function getRootSubRangeGates() { return $this->calculateSubRangeGates(-1, $this->oldest); } - /*private function dayDiff($interval) { - $from = new \DateTime(date('c', $interval[0])); - $to = new \DateTime(date('c', $interval[1])); - $diff = $from->diff($to); - return $diff->days; - }*/ - private function calculateSubRangeGates($level_index, \DateTime $lower_bound) { $result = array(); if ($level_index < count($this->levels)-1) { diff --git a/src/Router.php b/src/Router.php index 2b65e1b..4d71918 100644 --- a/src/Router.php +++ b/src/Router.php @@ -3,37 +3,29 @@ namespace oSoc\Smartflanders; use Bramus; -use Dotenv\Dotenv; use oSoc\Smartflanders\RangeGate; class Router { private $router; - private $out_dirname; - private $res_dirname; - private $second_interval; - private $nameToGP; - private $processors_gather; + private $settings; + private $nameToGP = array(); - public function __construct($out_dirname, $res_dirname, $second_interval, $nameToGP, $processors_gather) { + public function __construct(Settings $settings) { $this->router = new Bramus\Router\Router(); $this->router->set404(function() {echo "Page not found.";}); - $this->out_dirname = $out_dirname; - $this->res_dirname = $res_dirname; - $this->second_interval = $second_interval; - $this->nameToGP = $nameToGP; - $this->processors_gather = $processors_gather; + foreach($settings->getDatasets() as $gp) { + $name = $gp->getName(); + $name_lower = strtolower($name); + $this->nameToGP[$name_lower] = $gp; + } - $dotenv = new Dotenv(__DIR__ . '/../'); - $dotenv->load(); + $this->settings = $settings; } public function init() { - $out_dirname = $this->out_dirname; - $res_dirname = $this->res_dirname; - $second_interval = $this->second_interval; - $processors_gather = $this->processors_gather; + $settings = $this->settings; $found = false; $dataset_name = explode('.', $_SERVER['HTTP_HOST'])[0]; @@ -42,14 +34,14 @@ public function init() { if ($name === $dataset_name) { $found = true; $dataset = $this->nameToGP[$dataset_name]; - $fs = new Filesystem\FileSystemProcessor($out_dirname, $res_dirname ,$second_interval, $dataset); + $fs = new Filesystem\FileSystemProcessor($this->settings, $dataset); } } $this->router->get('/parking', - function() use ($found, $dataset, $out_dirname, $res_dirname, $second_interval, $processors_gather) { + function() use ($settings, $found, $dataset) { if ($found) { - View::view($dataset, $out_dirname, $res_dirname, $second_interval, $processors_gather); + View::view($settings, $dataset); } else { http_response_code(404); die("Route not found: " . $dataset); diff --git a/src/View.php b/src/View.php index fbf9a14..32d2fe6 100644 --- a/src/View.php +++ b/src/View.php @@ -55,7 +55,8 @@ public static function viewRangeGate($rangegate) { } } - public static function view($graph_processor, $out_dirname, $res_dirname, $second_interval, $processors_gather) { + public static function view(Settings $settings, Helpers\IGraphProcessor $graph_processor) { + $processors_gather = $settings->getDatasetsGather(); if (in_array($graph_processor, $processors_gather)) { // This data is being gathered here, get the file // If no preferred content type is specified, prefer turtle @@ -65,7 +66,7 @@ public static function view($graph_processor, $out_dirname, $res_dirname, $secon $filename = null; - $fs = new Filesystem\FileSystemProcessor($out_dirname, $res_dirname ,$second_interval, $graph_processor); + $fs = new Filesystem\FileSystemProcessor($settings, $graph_processor); if (!isset($_GET['page']) && !isset($_GET['time'])) { $timestamp = $fs->getLastPage(); @@ -96,7 +97,7 @@ public static function view($graph_processor, $out_dirname, $res_dirname, $secon header('Location: ' . $graph_processor->getBaseUrl() . '?page=' . $filename); } else { // This is sloppy coding - $fileReader = new Filesystem\FileReader($out_dirname, $res_dirname ,$second_interval, $graph_processor); + $fileReader = new Filesystem\FileReader($settings, $graph_processor); $graphs = $fileReader->getFullyDressedGraphsFromFile($filename); $historic = true; if ((string)$filename === $fs->getLastPage()) { diff --git a/src/index-singular.php b/src/index-singular.php deleted file mode 100644 index 37d05ad..0000000 --- a/src/index-singular.php +++ /dev/null @@ -1,51 +0,0 @@ -getLastPage(); -} - -else if (isset($_GET['page'])) { - // If page name is provided, it must be exact - $filename = $_GET['page']; - if (!$fs->hasFile($filename)) { - http_response_code(404); - die("Page not found"); - } -} - -else if (isset($_GET['time'])) { - // If timestamp is provided, find latest file before timestamp - $filename = $fs->getClosestPage(strtotime($_GET['time'])); - if (!$filename) { - http_response_code(404); - die("Time not found"); - } -} - -if (!isset($_GET['page'])) { - header("Access-Control-Allow-Origin: *"); - header('Location: ' . $graph_processor->getBaseUrl() . '?page=' . $filename); -} else { - // This is sloppy coding - $fileReader = new Filesystem\FileReader($out_dirname, $res_dirname ,$second_interval, $graph_processor); - $graphs = $fileReader->getFullyDressedGraphsFromFile($filename); - $historic = true; - if ($filename === $fs->getLastPage()) { - $historic = false; - } - View::view($_SERVER['HTTP_ACCEPT'], $graphs, $historic, $graph_processor->getBaseUrl(), $graph_processor->getRealTimeMaxAge()); -} \ No newline at end of file From 1647002c3e9614adda4e66d0706dbda97c2a87f6 Mon Sep 17 00:00:00 2001 From: Arne Gevaert Date: Fri, 3 Nov 2017 11:17:12 +0100 Subject: [PATCH 07/18] more cleanup in filesystem --- src/Filesystem/FileReader.php | 4 ++-- src/Filesystem/FileSystemProcessor.php | 27 ++++++++------------------ 2 files changed, 10 insertions(+), 21 deletions(-) diff --git a/src/Filesystem/FileReader.php b/src/Filesystem/FileReader.php index bd81a42..f7db6e4 100644 --- a/src/Filesystem/FileReader.php +++ b/src/Filesystem/FileReader.php @@ -183,7 +183,7 @@ private function getNextFileFromTimestamp($timestamp) { if ($this->out_fs->has($next_ts)) { return date("Y-m-d\TH:i:s", $next_ts); } else { - return $this->getNextFileFromTimestamp($next_ts + $this->second_interval); + return $this->getNextFileFromTimestamp($next_ts + $this->settings->getDefaultGatherInterval()); } } return false; @@ -193,7 +193,7 @@ private function getNextFileFromTimestamp($timestamp) { private function getPreviousFileFromTimestamp($timestamp) { $prev_ts = $this->getPreviousTimestampFromTimestamp($timestamp); if ($prev_ts) { - $prev_ts -= $this->second_interval; + $prev_ts -= $this->settings->getDefaultGatherInterval(); if ($this->out_fs->has($prev_ts)) { return date("Y-m-d\TH:i:s", $prev_ts); } else { diff --git a/src/Filesystem/FileSystemProcessor.php b/src/Filesystem/FileSystemProcessor.php index 7a2e72a..23455c9 100644 --- a/src/Filesystem/FileSystemProcessor.php +++ b/src/Filesystem/FileSystemProcessor.php @@ -15,27 +15,21 @@ protected $out_fs; protected $res_fs; protected $stat_fs; - protected $second_interval; protected $writer; protected $graph_processor; protected $static_data_filename; - protected $out_dirname; - protected $res_dirname; protected $settings; const REFRESH_STATIC = false; public function __construct(Settings $settings, IGraphProcessor $graph_processor) { $this->settings = $settings; - $this->out_dirname = $settings->getOutDir(); - $this->res_dirname = $settings->getResourcesDir(); - $this->second_interval = $settings->getDefaultGatherInterval(); date_default_timezone_set("Europe/Brussels"); - $out_adapter = new Local($this->out_dirname . "/" . $graph_processor->getName()); + $out_adapter = new Local($this->settings->getOutDir() . "/" . $graph_processor->getName()); $this->out_fs = new Filesystem($out_adapter); - $res_adapter = new Local($this->res_dirname); + $res_adapter = new Local($this->settings->getResourcesDir()); $this->res_fs = new Filesystem($res_adapter); - $stat_adapter = new Local($this->out_dirname . "/" . $graph_processor->getName() . "/statistical"); + $stat_adapter = new Local($this->settings->getOutDir() . "/" . $graph_processor->getName() . "/statistical"); $this->stat_fs = new Filesystem($stat_adapter); $this->graph_processor = $graph_processor; $this->static_data_filename = $graph_processor->getName() . "_static_data.turtle"; @@ -54,7 +48,7 @@ public function getFilesForDay(\DateTime $day) { $start = $start - ($start % 60*60*24); // Round down to day $start = $this->getPreviousTimestampFromTimestamp($start); // Get valid timestamp $end = $start + 60*60*24; - for ($i = $start; $i < $end; $i += $this->second_interval) { + for ($i = $start; $i < $end; $i += $this->settings->getDefaultGatherInterval()) { if ($this->hasFile($i)) { array_push($result, $i); } @@ -64,12 +58,7 @@ public function getFilesForDay(\DateTime $day) { public function getSecondInterval() { - return $this->second_interval; - } - - public function setSecondInterval($second_interval) - { - $this->second_interval = $second_interval; + return $this->settings->getDefaultGatherInterval(); } // Get the last written page (closest to now) @@ -88,7 +77,7 @@ public function getOldestTimestamp() { // Round a timestamp to its respective file timestamp protected function roundTimestamp($timestamp) { - $timestamp -= $timestamp % $this->second_interval; + $timestamp -= $timestamp % $this->settings->getDefaultGatherInterval(); return $timestamp; } @@ -101,7 +90,7 @@ public function getPreviousTimestampFromTimestamp($timestamp) { if ($this->out_fs->has($filename)) { return $timestamp; } - $timestamp -= $this->second_interval; + $timestamp -= $this->settings->getDefaultGatherInterval(); } } return false; @@ -111,7 +100,7 @@ public function getNextTimestampForTimestamp($timestamp) { $timestamp = $this->roundTimestamp($timestamp); $now = time(); while($timestamp < $now) { - $timestamp += $this->second_interval; + $timestamp += $this->settings->getDefaultGatherInterval(); $filename = $this->roundTimestamp($timestamp); if ($this->out_fs->has($filename)) { return $timestamp; From a21eabebaa98839580701e3ae8f1b1a7c313f495 Mon Sep 17 00:00:00 2001 From: Arne Gevaert Date: Fri, 3 Nov 2017 11:41:47 +0100 Subject: [PATCH 08/18] cleanup in datasets --- public/index.php | 2 +- src/Datasets/GentParking/GhentToRDF.php | 9 +++--- src/Datasets/Ixor/IxorLeuven.php | 10 ------ src/Datasets/Ixor/IxorMechelen.php | 10 ------ src/Datasets/Ixor/IxorSintNiklaas.php | 10 ------ src/Datasets/Ixor/IxorToRDF.php | 8 +++-- src/Datasets/Netherlands/NetherlandsToRDF.php | 31 ++++++++++--------- src/Datasets/ParkoKortrijk/ParkoToRDF.php | 13 ++++---- src/Helpers/IGraphProcessor.php | 1 + src/Settings.php | 19 ++++++++++-- 10 files changed, 52 insertions(+), 61 deletions(-) diff --git a/public/index.php b/public/index.php index 199560e..410d95b 100644 --- a/public/index.php +++ b/public/index.php @@ -10,7 +10,7 @@ use oSoc\Smartflanders\Datasets; -$settings = new Settings(); +$settings = Settings::getInstance(); $router = new Router($settings); $router->init(); diff --git a/src/Datasets/GentParking/GhentToRDF.php b/src/Datasets/GentParking/GhentToRDF.php index a459766..86fba89 100644 --- a/src/Datasets/GentParking/GhentToRDF.php +++ b/src/Datasets/GentParking/GhentToRDF.php @@ -7,7 +7,6 @@ namespace oSoc\Smartflanders\Datasets\GentParking; use oSoc\Smartflanders\Helpers; -use Dotenv; Class GhentToRDF implements Helpers\IGraphProcessor { @@ -19,14 +18,12 @@ private static $parkingURIs; private static $sameAs; - public function __construct() + public function __construct($publish) { - $dotenv = new Dotenv\Dotenv(__DIR__ . '/../../../'); - $dotenv->load(); $this->urls = [ self::STAT => "http://opendataportaalmobiliteitsbedrijf.stad.gent/datex2/v2/parkings/", self::DYN => "http://opendataportaalmobiliteitsbedrijf.stad.gent/datex2/v2/parkingsstatus", - self::BASE => $_ENV['GHENT_PUBLISH'] + self::BASE => $publish ]; } @@ -148,4 +145,6 @@ private static function preProcessing() return $graph; } + + public function setFetchUrl($url) {} } diff --git a/src/Datasets/Ixor/IxorLeuven.php b/src/Datasets/Ixor/IxorLeuven.php index 462381b..11c7af9 100644 --- a/src/Datasets/Ixor/IxorLeuven.php +++ b/src/Datasets/Ixor/IxorLeuven.php @@ -1,19 +1,9 @@ load(); - $fetch = $_ENV["IXOR_LEUVEN_FETCH"]; - $publish = $_ENV["IXOR_LEUVEN_PUBLISH"]; - parent::__construct($fetch, $publish); - } - public function getName() { return "Leuven"; diff --git a/src/Datasets/Ixor/IxorMechelen.php b/src/Datasets/Ixor/IxorMechelen.php index 0efda5b..6f0e4b5 100644 --- a/src/Datasets/Ixor/IxorMechelen.php +++ b/src/Datasets/Ixor/IxorMechelen.php @@ -1,19 +1,9 @@ load(); - $fetch = $_ENV["IXOR_MECHELEN_FETCH"]; - $publish = $_ENV["IXOR_MECHELEN_PUBLISH"]; - parent::__construct($fetch, $publish); - } - public function getName() { return "Mechelen"; diff --git a/src/Datasets/Ixor/IxorSintNiklaas.php b/src/Datasets/Ixor/IxorSintNiklaas.php index e9e876b..7b2b9d8 100644 --- a/src/Datasets/Ixor/IxorSintNiklaas.php +++ b/src/Datasets/Ixor/IxorSintNiklaas.php @@ -1,19 +1,9 @@ load(); - $fetch = $_ENV["IXOR_SINT-NIKLAAS_FETCH"]; - $publish = $_ENV["IXOR_SINT-NIKLAAS_PUBLISH"]; - parent::__construct($fetch, $publish); - } - public function getName() { return "Sint-Niklaas"; diff --git a/src/Datasets/Ixor/IxorToRDF.php b/src/Datasets/Ixor/IxorToRDF.php index a1ecadf..96d133e 100644 --- a/src/Datasets/Ixor/IxorToRDF.php +++ b/src/Datasets/Ixor/IxorToRDF.php @@ -9,9 +9,8 @@ abstract class IxorToRDF implements Helpers\IGraphProcessor private $fetch_url, $publish_url; private $authHeader; - public function __construct($fetch_url, $publish_url) + public function __construct($publish_url) { - $this->fetch_url = $fetch_url; $this->publish_url = $publish_url; $dotenv = new Dotenv\Dotenv(__DIR__ . '/../../../'); $dotenv->load(); @@ -73,4 +72,9 @@ public function mustQuery() { return true; } + + public function setFetchUrl($url) + { + $this->fetch_url = $url; + } } diff --git a/src/Datasets/Netherlands/NetherlandsToRDF.php b/src/Datasets/Netherlands/NetherlandsToRDF.php index 680b241..e0ee378 100644 --- a/src/Datasets/Netherlands/NetherlandsToRDF.php +++ b/src/Datasets/Netherlands/NetherlandsToRDF.php @@ -2,24 +2,18 @@ namespace oSoc\Smartflanders\Datasets\Netherlands; use oSoc\Smartflanders\Helpers; -use Dotenv; use \League\Flysystem\Adapter\Local; use \League\Flysystem\Filesystem; +use oSoc\Smartflanders\Settings; class NetherlandsToRDF implements Helpers\IGraphProcessor { - private $publish_url, $fetch_url, $res_fs; + private $publish_url, $fetch_url; - public function __construct() + public function __construct($publish) { - $dotenv = new Dotenv\Dotenv(__DIR__ . '/../../../'); - $dotenv->load(); - $this->publish_url = $_ENV['NEDERLAND_PUBLISH']; - $this->fetch_url = $_ENV['NEDERLAND_FETCH']; - // TODO this is a bad dependency. Resources directory should be added to .env. - $res_adapter = new Local(__DIR__ . '/../../../resources'); - $this->res_fs = new Filesystem($res_adapter); + $this->publish_url = $publish; } public function getDynamicGraph() @@ -97,20 +91,24 @@ public function getRealTimeMaxAge() public function mustQuery() { + $settings = Settings::getInstance(); + $res_adapter = new Local($settings->getResourcesDir()); + $res_fs = new Filesystem($res_adapter); + $filename = 'Nederland_last_measurement'; $now = time(); - if ($this->res_fs->has($filename)) { - $string_value = $this->res_fs->read($filename); + if ($res_fs->has($filename)) { + $string_value = $res_fs->read($filename); $int_value = intval($string_value); if ($now - $int_value > 30*60) { // Only query once every 30 minutes - $this->res_fs->put($filename, $now); + $res_fs->put($filename, $now); return true; } else { return false; } } - $this->res_fs->write($filename, $now); + $res_fs->write($filename, $now); return true; } @@ -127,4 +125,9 @@ private function getAccessibleParkings() { } return $accessible_parkings; } + + public function setFetchUrl($url) + { + $this->fetch_url = $url; + } } \ No newline at end of file diff --git a/src/Datasets/ParkoKortrijk/ParkoToRDF.php b/src/Datasets/ParkoKortrijk/ParkoToRDF.php index 2a29141..d8b61fd 100644 --- a/src/Datasets/ParkoKortrijk/ParkoToRDF.php +++ b/src/Datasets/ParkoKortrijk/ParkoToRDF.php @@ -3,18 +3,14 @@ namespace oSoc\Smartflanders\Datasets\ParkoKortrijk; use oSoc\Smartflanders\Helpers; -use Dotenv; class ParkoToRDF implements Helpers\IGraphProcessor { private $publish_url, $fetch_url; - public function __construct() + public function __construct($publish) { - $dotenv = new Dotenv\Dotenv(__DIR__ . '/../../../'); - $dotenv->load(); - $this->publish_url = $_ENV['PARKO_KORTRIJK_PUBLISH']; - $this->fetch_url = $_ENV['PARKO_KORTRIJK_FETCH']; + $this->publish_url = $publish; } public function getDynamicGraph() @@ -84,4 +80,9 @@ public function mustQuery() { return true; } + + public function setFetchUrl($url) + { + $this->fetch_url = $url; + } } diff --git a/src/Helpers/IGraphProcessor.php b/src/Helpers/IGraphProcessor.php index 2df494b..3bca84d 100644 --- a/src/Helpers/IGraphProcessor.php +++ b/src/Helpers/IGraphProcessor.php @@ -9,4 +9,5 @@ public function getName(); public function getBaseUrl(); public function getRealTimeMaxAge(); public function mustQuery(); + public function setFetchUrl($url); } diff --git a/src/Settings.php b/src/Settings.php index 9b8cd28..cf6f337 100644 --- a/src/Settings.php +++ b/src/Settings.php @@ -3,10 +3,10 @@ namespace oSoc\Smartflanders; use Dotenv\Dotenv; -use oSoc\Smartflanders\Helpers\IGraphProcessor; class Settings { + private static $instance = null; private $data_dir; private $resource_dir; private $default_gather_interval; @@ -15,7 +15,7 @@ class Settings private $datasets = array(); private $datasets_gather = array(); - function __construct() { + private function __construct() { $this->dotenv = new Dotenv(__DIR__ . '/../'); $this->dotenv->load(); @@ -76,12 +76,25 @@ private function parseDatasets() { private function parseDataset($dataset) { try { $this->dotenv->required($dataset . "_PATH"); + $this->dotenv->required($dataset . "_PUBLISH"); $class = $_ENV[$dataset . "_PATH"]; - return new $class; + $publish = $_ENV[$dataset . "_PUBLISH"]; + $processor = new $class($publish); + if (isset($_ENV[$dataset . "_FETCH"])) { + $processor->setFetchUrl($_ENV[$dataset . "_FETCH"]); + } + return $processor; } catch (\Exception $e) { error_log("Invalid .env configuration: dataset " . $dataset . " has no corresponding class path." . " Please add the variable " . $dataset . "_PATH."); } return null; } + + public static function getInstance() { + if (self::$instance === null) { + self::$instance = new Settings(); + } + return self::$instance; + } } \ No newline at end of file From e4630ec73369120403cff5b0770a6a3334f3d10f Mon Sep 17 00:00:00 2001 From: Arne Gevaert Date: Fri, 3 Nov 2017 12:10:33 +0100 Subject: [PATCH 09/18] fixed cron file --- cron.php | 27 ++++------------------- test/TimeStampCalculatorTest.php | 38 -------------------------------- 2 files changed, 4 insertions(+), 61 deletions(-) delete mode 100644 test/TimeStampCalculatorTest.php diff --git a/cron.php b/cron.php index b809ff2..0c01b84 100644 --- a/cron.php +++ b/cron.php @@ -22,36 +22,17 @@ } else if ($argv[1] === "debug") { acquire_data(); } -/** - * This function simply periodically saves the entire turtle file with the current ISO timestamp as filename - * + triples for timestamp and filename of previous file - */ + function acquire_data() { - $dotenv = new Dotenv(__DIR__); - $dotenv->load(); - $datasets = explode(',', $_ENV["DATASETS_GATHER"]); - $processors = array(); - foreach($datasets as $dataset) { - try { - $dotenv->required($dataset . "_PATH"); - $class = $_ENV[$dataset . "_PATH"]; - array_push($processors, new $class); - } catch (Exception $e) { - error_log("Invalid .env configuration: dataset " . $dataset . " was has no corresponding class path." - . " Please add the variable " . $dataset . "_PATH."); - } - } - foreach ($processors as $processor) { + $settings = \oSoc\Smartflanders\Settings::getInstance(); + foreach ($settings->getDatasetsGather() as $processor) { if ($processor->mustQuery()) { - $interval = 60*60*3; // 3 hour interval results in files of a few 100 KB - // TODO add out and resources directories to .env. Right now Dutch dataset class has dependency on this. - $fs = new Filesystem\FileWriter(__DIR__ . "/out", __DIR__ . "/resources", $interval, $processor); + $fs = new Filesystem\FileWriter(\oSoc\Smartflanders\Settings::getInstance(), $processor); echo "Fetching graph for " . $processor->getName() . "\n"; $graph = $processor->getDynamicGraph(); $now = time(); echo "Writing data for " . $processor->getName() . "\n"; $fs->writeToFile($now, $graph); - // Temporarily disabling range gates, semantically incorrect echo "Updating statistical summary for " . $processor->getName() . "\n"; $fs->updateStatisticalSummary($now, $graph); } diff --git a/test/TimeStampCalculatorTest.php b/test/TimeStampCalculatorTest.php deleted file mode 100644 index 7edc043..0000000 --- a/test/TimeStampCalculatorTest.php +++ /dev/null @@ -1,38 +0,0 @@ -assertEquals($calculators[0],$calculators[1]); - } - - /** - * @dataProvider prov - */ - public function testGetPrevious($calculators) { - $this->assertEquals($calculators[0],$calculators[2]); - } - - /** - * @dataProvider prov - */ - public function testGetFor($calculators) { - $this->assertEquals($calculators[0],$calculators[3]); - } - - public function prov() { - $res = array(); - for ($i = 5; $i <= 20; $i+=5) { - array_push($res, [new TimeStampCalculator($i*60)]); - } - return $res; - // for each array the methods will be called with the contents of this array as arguments - } -} \ No newline at end of file From 87d55e5d99e92fd582b42e06889d1e6d6539b6a1 Mon Sep 17 00:00:00 2001 From: Arne Gevaert Date: Fri, 3 Nov 2017 14:07:40 +0100 Subject: [PATCH 10/18] skeleton --- .gitignore | 2 ++ benchmark/config.json | 30 ++++++++++++++++++++++++++++++ benchmark/deploy_benchmark.js | 22 ++++++++++++++++++++++ benchmark/package-lock.json | 11 +++++++++++ benchmark/package.json | 6 ++++++ 5 files changed, 71 insertions(+) create mode 100644 benchmark/config.json create mode 100644 benchmark/deploy_benchmark.js create mode 100644 benchmark/package-lock.json create mode 100644 benchmark/package.json diff --git a/.gitignore b/.gitignore index 3169366..ac1d80f 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,5 @@ +benchmark/node_modules/* + # Hide .env from repo .env diff --git a/benchmark/config.json b/benchmark/config.json new file mode 100644 index 0000000..a7f736d --- /dev/null +++ b/benchmark/config.json @@ -0,0 +1,30 @@ +{ + "deploy:base_url": "example.com", + + "cities:number": 1, + + "parkings:min_num_per_city": 1, + "parkings:max_num_per_city": 1, + "parkings:min_spaces": 100, + "parkings:max_spaces": 100, + + "time:start": "2017-09-25T12:00:00", + "time:end": "2017-09-26T12:00:00", + "time:time_per_file": 86400, + "time:interval": 30, + + "events:min_events": 1, + "events:max_events": 3, + "events:min_duration": 1, + "events:max_duration": 8, + "events:min_occupation": 0.2, + "events:max_occupation": 0.8, + + "events:min_lambda": 0.5, + "events:max_lambda": 2, + "events:lambda_var": 0.07, + + "events:min_sigma": 0.05, + "events:max_sigma": 1, + "events:sigma_var": 0.04 +} \ No newline at end of file diff --git a/benchmark/deploy_benchmark.js b/benchmark/deploy_benchmark.js new file mode 100644 index 0000000..e236a58 --- /dev/null +++ b/benchmark/deploy_benchmark.js @@ -0,0 +1,22 @@ +const fs = require('fs'); + +if (process.argv.length !== 3) { + console.error("Usage: node deploy_benchmark.js CONFIG"); + process.exit(1); +} + +const config = JSON.parse(fs.readFileSync(process.argv[2])); +console.log(config); + +// Generate .env file + // Publish URLs + +// Create out/ and resources/ if necessary + +// Generate lpd config + +// Generate data + +// Generate resources: oldest timestamps, static data + +// Deploy php -S \ No newline at end of file diff --git a/benchmark/package-lock.json b/benchmark/package-lock.json new file mode 100644 index 0000000..01312aa --- /dev/null +++ b/benchmark/package-lock.json @@ -0,0 +1,11 @@ +{ + "requires": true, + "lockfileVersion": 1, + "dependencies": { + "fs": { + "version": "0.0.1-security", + "resolved": "https://registry.npmjs.org/fs/-/fs-0.0.1-security.tgz", + "integrity": "sha1-invTcYa23d84E/I4WLV+yq9eQdQ=" + } + } +} diff --git a/benchmark/package.json b/benchmark/package.json new file mode 100644 index 0000000..cddc065 --- /dev/null +++ b/benchmark/package.json @@ -0,0 +1,6 @@ +{ + "dependencies": { + "fs": "0.0.1-security" + }, + "devDependencies": {} +} From ce0aa0b23f2c94119e7f520342d3cbd494ddccd7 Mon Sep 17 00:00:00 2001 From: Arne Gevaert Date: Fri, 3 Nov 2017 14:23:12 +0100 Subject: [PATCH 11/18] created benchmark graph processor --- src/Datasets/Benchmark/Benchmark.php | 58 ++++++++++++++++++++++++++++ 1 file changed, 58 insertions(+) create mode 100644 src/Datasets/Benchmark/Benchmark.php diff --git a/src/Datasets/Benchmark/Benchmark.php b/src/Datasets/Benchmark/Benchmark.php new file mode 100644 index 0000000..1e6d952 --- /dev/null +++ b/src/Datasets/Benchmark/Benchmark.php @@ -0,0 +1,58 @@ +publish_url = $publish; + $this->name = explode('.', $publish)[0]; + } + + public function getDynamicGraph() + { + return array(); + } + + public function getStaticGraph() + { + return array(); + } + + public function getName() + { + return $this->name; + } + + public function getBaseUrl() + { + return $this->publish_url; + } + + public function getRealTimeMaxAge() + { + return 30; + } + + public function mustQuery() + { + return true; + } + + public function setFetchUrl($url) + { + return; + } +} \ No newline at end of file From e6fbbaac91644c09ef73c30afc8bc3fdfb67df71 Mon Sep 17 00:00:00 2001 From: Arne Gevaert Date: Fri, 3 Nov 2017 14:55:10 +0100 Subject: [PATCH 12/18] generating data --- .gitignore | 2 ++ benchmark/config.json | 10 +++++--- benchmark/deploy_benchmark.js | 46 ++++++++++++++++++++++++++++++++--- benchmark/lpd_config.json | 1 + benchmark/package-lock.json | 5 ++++ benchmark/package.json | 1 + 6 files changed, 57 insertions(+), 8 deletions(-) create mode 100644 benchmark/lpd_config.json diff --git a/.gitignore b/.gitignore index ac1d80f..0db60ea 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,6 @@ benchmark/node_modules/* +benchmark/out/* +benchmark/resources/* # Hide .env from repo .env diff --git a/benchmark/config.json b/benchmark/config.json index a7f736d..09c66de 100644 --- a/benchmark/config.json +++ b/benchmark/config.json @@ -1,16 +1,18 @@ { "deploy:base_url": "example.com", + "deploy:rangegate_config": "1Y,1M,1W,1D", + "deploy:lpdgen_location": "../../lpdgenerator/bin", - "cities:number": 1, + "cities:number": 5, "parkings:min_num_per_city": 1, - "parkings:max_num_per_city": 1, + "parkings:max_num_per_city": 8, "parkings:min_spaces": 100, - "parkings:max_spaces": 100, + "parkings:max_spaces": 300, "time:start": "2017-09-25T12:00:00", "time:end": "2017-09-26T12:00:00", - "time:time_per_file": 86400, + "time:time_per_file": 10800, "time:interval": 30, "events:min_events": 1, diff --git a/benchmark/deploy_benchmark.js b/benchmark/deploy_benchmark.js index e236a58..ce109ee 100644 --- a/benchmark/deploy_benchmark.js +++ b/benchmark/deploy_benchmark.js @@ -1,4 +1,6 @@ const fs = require('fs'); +const childProcess = require('child_process'); +const path = require('path'); if (process.argv.length !== 3) { console.error("Usage: node deploy_benchmark.js CONFIG"); @@ -6,16 +8,52 @@ if (process.argv.length !== 3) { } const config = JSON.parse(fs.readFileSync(process.argv[2])); -console.log(config); - -// Generate .env file - // Publish URLs // Create out/ and resources/ if necessary +if (!fs.existsSync('out')) { + fs.mkdirSync('out'); +} +if (!fs.existsSync('resources')) { + fs.mkdirSync('resources'); +} + +// Cleanup +let outdirs = fs.readdirSync('out'); +let resourcefiles = fs.readdirSync('resources'); +outdirs.forEach(dir => { + let files = fs.readdirSync(path.join('out', dir)); + files.forEach(file => { + fs.unlinkSync(path.join('out', dir, file), err => { + if (err) console.error(err); + }) + }); +}); +resourcefiles.forEach(file => { + fs.unlinkSync(path.join('resources', file), err => { + if (err) console.error(err); + }) +}); // Generate lpd config +config["file:output"] = 'out'; +config["file:output_meta_data"] = false; +config["file:extension"] = ''; +config["file:name_format"] = "UNIX"; +fs.writeFileSync('lpd_config.json', JSON.stringify(config)); // Generate data +let script = config['deploy:lpdgen_location'] + '/generator.js'; +let genProc = childProcess.fork(script, ['lpd_config.json']); +genProc.on('exit', code => console.log("Process exited with exit code", code)); + +// Generate .env file + // Publish URLs + // Datasets, Datasets_gather + // Paths: oSoc\Smartflanders\Datasets\Benchmark\Benchmark + // Rangegate config === config.rangegate_config + // Data, resource dir: benchmark/out, benchmark/resources + // Default gather interval: arbitrary + // base_publish === config.base_url // Generate resources: oldest timestamps, static data diff --git a/benchmark/lpd_config.json b/benchmark/lpd_config.json new file mode 100644 index 0000000..8e055af --- /dev/null +++ b/benchmark/lpd_config.json @@ -0,0 +1 @@ +{"deploy:base_url":"example.com","deploy:rangegate_config":"1Y,1M,1W,1D","deploy:lpdgen_location":"../../lpdgenerator/bin","cities:number":5,"parkings:min_num_per_city":1,"parkings:max_num_per_city":8,"parkings:min_spaces":100,"parkings:max_spaces":300,"time:start":"2017-09-25T12:00:00","time:end":"2017-09-26T12:00:00","time:time_per_file":10800,"time:interval":30,"events:min_events":1,"events:max_events":3,"events:min_duration":1,"events:max_duration":8,"events:min_occupation":0.2,"events:max_occupation":0.8,"events:min_lambda":0.5,"events:max_lambda":2,"events:lambda_var":0.07,"events:min_sigma":0.05,"events:max_sigma":1,"events:sigma_var":0.04,"file:output":"out","file:output_meta_data":false,"file:extension":"","file:name_format":"UNIX"} \ No newline at end of file diff --git a/benchmark/package-lock.json b/benchmark/package-lock.json index 01312aa..fdf4ff1 100644 --- a/benchmark/package-lock.json +++ b/benchmark/package-lock.json @@ -2,6 +2,11 @@ "requires": true, "lockfileVersion": 1, "dependencies": { + "child_process": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/child_process/-/child_process-1.0.2.tgz", + "integrity": "sha1-sffn/HPSXn/R1FWtyU4UODAYK1o=" + }, "fs": { "version": "0.0.1-security", "resolved": "https://registry.npmjs.org/fs/-/fs-0.0.1-security.tgz", diff --git a/benchmark/package.json b/benchmark/package.json index cddc065..2ab32b3 100644 --- a/benchmark/package.json +++ b/benchmark/package.json @@ -1,5 +1,6 @@ { "dependencies": { + "child_process": "^1.0.2", "fs": "0.0.1-security" }, "devDependencies": {} From 1408dc84b09ac7cbae6f980f805974dcd2e0770f Mon Sep 17 00:00:00 2001 From: Arne Gevaert Date: Fri, 3 Nov 2017 17:48:57 +0100 Subject: [PATCH 13/18] generating .env file --- .gitignore | 1 + benchmark/deploy_benchmark.js | 16 ++++++++++++++++ src/Settings.php | 12 +++++++++++- 3 files changed, 28 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 0db60ea..ff2037c 100644 --- a/.gitignore +++ b/.gitignore @@ -4,6 +4,7 @@ benchmark/resources/* # Hide .env from repo .env +.env.benchmark log/* sandbox.php diff --git a/benchmark/deploy_benchmark.js b/benchmark/deploy_benchmark.js index ce109ee..dee1912 100644 --- a/benchmark/deploy_benchmark.js +++ b/benchmark/deploy_benchmark.js @@ -54,6 +54,22 @@ genProc.on('exit', code => console.log("Process exited with exit code", code)); // Data, resource dir: benchmark/out, benchmark/resources // Default gather interval: arbitrary // base_publish === config.base_url +let env = ""; +let datasets = 'DATASETS="'; +outdirs.forEach(dir => { + let caps = dir.toUpperCase(); + env += caps + '_PUBLISH="http://' + dir + '.' + config["deploy:base_url"] + '"\n'; + datasets += caps + ','; + env += caps + '_PATH=oSoc\\Smartflanders\\Datasets\\Benchmark\\Benchmark' + '\n'; +}); +env += datasets + '"\n'; +env += 'RANGE_GATES_CONFIG="' + config["deploy:rangegate_config"] + '"\n'; +env += 'DATA_DIR="benchmark/out"\n'; +env += 'RESOURCE_DIR="benchmark/resources"\n'; +env += 'DEFAULT_GATHER_INTERVAL=' + config["time:time_per_file"] + '\n'; // TODO this is not right in .env! +env += 'BASE_PUBLISH="' + config["deploy:base_url"] + '"\n'; + +fs.writeFileSync('../.env.benchmark', env); // Generate resources: oldest timestamps, static data diff --git a/src/Settings.php b/src/Settings.php index cf6f337..d487997 100644 --- a/src/Settings.php +++ b/src/Settings.php @@ -3,6 +3,8 @@ namespace oSoc\Smartflanders; use Dotenv\Dotenv; +use League\Flysystem\Adapter\Local; +use League\Flysystem\Filesystem; class Settings { @@ -16,7 +18,15 @@ class Settings private $datasets_gather = array(); private function __construct() { - $this->dotenv = new Dotenv(__DIR__ . '/../'); + + $env_adapter = new Local(__DIR__ . '/../'); + $env_fs = new Filesystem($env_adapter); + if ($env_fs->has('.env.benchmark')) { + $this->dotenv = new Dotenv(__DIR__ . '/../', '.env.benchmark'); + } else { + $this->dotenv = new Dotenv(__DIR__ . '/../'); + } + $this->dotenv->load(); $this->required(); From ec3980ee953e4a02336d56585a5ca72eb6529c41 Mon Sep 17 00:00:00 2001 From: Arne Gevaert Date: Fri, 3 Nov 2017 17:53:41 +0100 Subject: [PATCH 14/18] error in .env --- benchmark/deploy_benchmark.js | 7 ------- src/Filesystem/FileReader.php | 4 ++-- src/Filesystem/FileSystemProcessor.php | 13 ++++--------- src/Settings.php | 8 ++++---- 4 files changed, 10 insertions(+), 22 deletions(-) diff --git a/benchmark/deploy_benchmark.js b/benchmark/deploy_benchmark.js index dee1912..0841f56 100644 --- a/benchmark/deploy_benchmark.js +++ b/benchmark/deploy_benchmark.js @@ -47,13 +47,6 @@ let genProc = childProcess.fork(script, ['lpd_config.json']); genProc.on('exit', code => console.log("Process exited with exit code", code)); // Generate .env file - // Publish URLs - // Datasets, Datasets_gather - // Paths: oSoc\Smartflanders\Datasets\Benchmark\Benchmark - // Rangegate config === config.rangegate_config - // Data, resource dir: benchmark/out, benchmark/resources - // Default gather interval: arbitrary - // base_publish === config.base_url let env = ""; let datasets = 'DATASETS="'; outdirs.forEach(dir => { diff --git a/src/Filesystem/FileReader.php b/src/Filesystem/FileReader.php index f7db6e4..f404d17 100644 --- a/src/Filesystem/FileReader.php +++ b/src/Filesystem/FileReader.php @@ -183,7 +183,7 @@ private function getNextFileFromTimestamp($timestamp) { if ($this->out_fs->has($next_ts)) { return date("Y-m-d\TH:i:s", $next_ts); } else { - return $this->getNextFileFromTimestamp($next_ts + $this->settings->getDefaultGatherInterval()); + return $this->getNextFileFromTimestamp($next_ts + $this->settings->getTimePerFile()); } } return false; @@ -193,7 +193,7 @@ private function getNextFileFromTimestamp($timestamp) { private function getPreviousFileFromTimestamp($timestamp) { $prev_ts = $this->getPreviousTimestampFromTimestamp($timestamp); if ($prev_ts) { - $prev_ts -= $this->settings->getDefaultGatherInterval(); + $prev_ts -= $this->settings->getTimePerFile(); if ($this->out_fs->has($prev_ts)) { return date("Y-m-d\TH:i:s", $prev_ts); } else { diff --git a/src/Filesystem/FileSystemProcessor.php b/src/Filesystem/FileSystemProcessor.php index 23455c9..ec46523 100644 --- a/src/Filesystem/FileSystemProcessor.php +++ b/src/Filesystem/FileSystemProcessor.php @@ -48,7 +48,7 @@ public function getFilesForDay(\DateTime $day) { $start = $start - ($start % 60*60*24); // Round down to day $start = $this->getPreviousTimestampFromTimestamp($start); // Get valid timestamp $end = $start + 60*60*24; - for ($i = $start; $i < $end; $i += $this->settings->getDefaultGatherInterval()) { + for ($i = $start; $i < $end; $i += $this->settings->getTimePerFile()) { if ($this->hasFile($i)) { array_push($result, $i); } @@ -56,11 +56,6 @@ public function getFilesForDay(\DateTime $day) { return $result; } - public function getSecondInterval() - { - return $this->settings->getDefaultGatherInterval(); - } - // Get the last written page (closest to now) public function getLastPage() { return $this->getPreviousTimestampFromTimestamp(time()); @@ -77,7 +72,7 @@ public function getOldestTimestamp() { // Round a timestamp to its respective file timestamp protected function roundTimestamp($timestamp) { - $timestamp -= $timestamp % $this->settings->getDefaultGatherInterval(); + $timestamp -= $timestamp % $this->settings->getTimePerFile(); return $timestamp; } @@ -90,7 +85,7 @@ public function getPreviousTimestampFromTimestamp($timestamp) { if ($this->out_fs->has($filename)) { return $timestamp; } - $timestamp -= $this->settings->getDefaultGatherInterval(); + $timestamp -= $this->settings->getTimePerFile(); } } return false; @@ -100,7 +95,7 @@ public function getNextTimestampForTimestamp($timestamp) { $timestamp = $this->roundTimestamp($timestamp); $now = time(); while($timestamp < $now) { - $timestamp += $this->settings->getDefaultGatherInterval(); + $timestamp += $this->settings->getTimePerFile(); $filename = $this->roundTimestamp($timestamp); if ($this->out_fs->has($filename)) { return $timestamp; diff --git a/src/Settings.php b/src/Settings.php index d487997..3e47e2a 100644 --- a/src/Settings.php +++ b/src/Settings.php @@ -11,7 +11,7 @@ class Settings private static $instance = null; private $data_dir; private $resource_dir; - private $default_gather_interval; + private $time_per_file; private $range_gates_config; private $dotenv; private $datasets = array(); @@ -34,7 +34,7 @@ private function __construct() { $this->data_dir = __DIR__ . '/../' . $_ENV["DATA_DIR"]; $this->resource_dir = __DIR__ . '/../' . $_ENV["RESOURCE_DIR"]; - $this->default_gather_interval = $_ENV["DEFAULT_GATHER_INTERVAL"]; + $this->time_per_file = $_ENV["TIME_PER_FILE"]; $this->range_gates_config = $_ENV["RANGE_GATES_CONFIG"]; } @@ -54,8 +54,8 @@ function getDatasetsGather() { return $this->datasets_gather; } - function getDefaultGatherInterval() { - return $this->default_gather_interval; + function getTimePerFile() { + return $this->time_per_file; } function getRangeGatesConfig() { From bf18ed5a67d45f32fd482723e984a59f3b4d8edf Mon Sep 17 00:00:00 2001 From: Arne Gevaert Date: Sat, 4 Nov 2017 09:48:41 +0100 Subject: [PATCH 15/18] tweaks --- benchmark/deploy_benchmark.js | 9 ++++++++- benchmark/package-lock.json | 5 +++++ benchmark/package.json | 3 ++- src/Router.php | 4 ++-- 4 files changed, 17 insertions(+), 4 deletions(-) diff --git a/benchmark/deploy_benchmark.js b/benchmark/deploy_benchmark.js index 0841f56..23a8c94 100644 --- a/benchmark/deploy_benchmark.js +++ b/benchmark/deploy_benchmark.js @@ -55,7 +55,8 @@ outdirs.forEach(dir => { datasets += caps + ','; env += caps + '_PATH=oSoc\\Smartflanders\\Datasets\\Benchmark\\Benchmark' + '\n'; }); -env += datasets + '"\n'; +env += datasets.slice(0,-1) + '"\n'; +env += 'DATASETS_GATHER=""'; env += 'RANGE_GATES_CONFIG="' + config["deploy:rangegate_config"] + '"\n'; env += 'DATA_DIR="benchmark/out"\n'; env += 'RESOURCE_DIR="benchmark/resources"\n'; @@ -65,5 +66,11 @@ env += 'BASE_PUBLISH="' + config["deploy:base_url"] + '"\n'; fs.writeFileSync('../.env.benchmark', env); // Generate resources: oldest timestamps, static data +outdirs.forEach(dir => { + const files = fs.readdirSync(path.join('out', dir)); + files.forEach(file => { + + }) +}); // Deploy php -S \ No newline at end of file diff --git a/benchmark/package-lock.json b/benchmark/package-lock.json index fdf4ff1..22fcb39 100644 --- a/benchmark/package-lock.json +++ b/benchmark/package-lock.json @@ -11,6 +11,11 @@ "version": "0.0.1-security", "resolved": "https://registry.npmjs.org/fs/-/fs-0.0.1-security.tgz", "integrity": "sha1-invTcYa23d84E/I4WLV+yq9eQdQ=" + }, + "n3": { + "version": "0.11.2", + "resolved": "https://registry.npmjs.org/n3/-/n3-0.11.2.tgz", + "integrity": "sha512-ICSiOmFLbZ4gI35+4H3e2vYGHDC944WZkCa1iVNRAx/mRZESEevQNFhfHaui/lhqynoZYvBVDNjM/2Tfd3TICQ==" } } } diff --git a/benchmark/package.json b/benchmark/package.json index 2ab32b3..7156faf 100644 --- a/benchmark/package.json +++ b/benchmark/package.json @@ -1,7 +1,8 @@ { "dependencies": { "child_process": "^1.0.2", - "fs": "0.0.1-security" + "fs": "0.0.1-security", + "n3": "^0.11.2" }, "devDependencies": {} } diff --git a/src/Router.php b/src/Router.php index 4d71918..783c8d5 100644 --- a/src/Router.php +++ b/src/Router.php @@ -39,12 +39,12 @@ public function init() { } $this->router->get('/parking', - function() use ($settings, $found, $dataset) { + function() use ($settings, $found, $dataset, $dataset_name) { if ($found) { View::view($settings, $dataset); } else { http_response_code(404); - die("Route not found: " . $dataset); + die("Dataset not found: " . $dataset_name); } } ); From 1ef667e0888b5f9eaba514ed81baba23c5835cb5 Mon Sep 17 00:00:00 2001 From: Arne Gevaert Date: Sat, 4 Nov 2017 10:59:21 +0100 Subject: [PATCH 16/18] splitting realtime from static data --- benchmark/deploy_benchmark.js | 95 ++++++++++++++++++++++++++--------- 1 file changed, 70 insertions(+), 25 deletions(-) diff --git a/benchmark/deploy_benchmark.js b/benchmark/deploy_benchmark.js index 23a8c94..ff4a441 100644 --- a/benchmark/deploy_benchmark.js +++ b/benchmark/deploy_benchmark.js @@ -1,6 +1,7 @@ const fs = require('fs'); const childProcess = require('child_process'); const path = require('path'); +const N3 = require('n3'); if (process.argv.length !== 3) { console.error("Usage: node deploy_benchmark.js CONFIG"); @@ -9,6 +10,13 @@ if (process.argv.length !== 3) { const config = JSON.parse(fs.readFileSync(process.argv[2])); +const buildingBlocks = { + 'ParkingSite': 'http://vocab.datex.org/terms#UrbanParkingSite', + 'pLabel': 'http://www.w3.org/2000/01/rdf-schema#label', + 'pNumberOfSpaces': 'http://vocab.datex.org/terms#parkingNumberOfSpaces', + 'pRdfType': 'http://www.w3.org/1999/02/22-rdf-syntax-ns#type' +}; + // Create out/ and resources/ if necessary if (!fs.existsSync('out')) { fs.mkdirSync('out'); @@ -44,33 +52,70 @@ fs.writeFileSync('lpd_config.json', JSON.stringify(config)); // Generate data let script = config['deploy:lpdgen_location'] + '/generator.js'; let genProc = childProcess.fork(script, ['lpd_config.json']); -genProc.on('exit', code => console.log("Process exited with exit code", code)); +genProc.on('exit', code => { + console.log("Process exited with exit code", code); -// Generate .env file -let env = ""; -let datasets = 'DATASETS="'; -outdirs.forEach(dir => { - let caps = dir.toUpperCase(); - env += caps + '_PUBLISH="http://' + dir + '.' + config["deploy:base_url"] + '"\n'; - datasets += caps + ','; - env += caps + '_PATH=oSoc\\Smartflanders\\Datasets\\Benchmark\\Benchmark' + '\n'; -}); -env += datasets.slice(0,-1) + '"\n'; -env += 'DATASETS_GATHER=""'; -env += 'RANGE_GATES_CONFIG="' + config["deploy:rangegate_config"] + '"\n'; -env += 'DATA_DIR="benchmark/out"\n'; -env += 'RESOURCE_DIR="benchmark/resources"\n'; -env += 'DEFAULT_GATHER_INTERVAL=' + config["time:time_per_file"] + '\n'; // TODO this is not right in .env! -env += 'BASE_PUBLISH="' + config["deploy:base_url"] + '"\n'; + outdirs = fs.readdirSync('out'); + resourcefiles = fs.readdirSync('resources'); -fs.writeFileSync('../.env.benchmark', env); + // Generate .env file + let env = ""; + let datasets = 'DATASETS="'; + outdirs.forEach(dir => { + let caps = dir.toUpperCase(); + env += caps + '_PUBLISH="http://' + dir + '.' + config["deploy:base_url"] + '"\n'; + datasets += caps + ','; + env += caps + '_PATH=oSoc\\Smartflanders\\Datasets\\Benchmark\\Benchmark' + '\n'; + }); + env += datasets.slice(0,-1) + '"\n'; + env += 'DATASETS_GATHER=""\n'; + env += 'RANGE_GATES_CONFIG="' + config["deploy:rangegate_config"] + '"\n'; + env += 'DATA_DIR="benchmark/out"\n'; + env += 'RESOURCE_DIR="benchmark/resources"\n'; + env += 'DEFAULT_GATHER_INTERVAL=' + config["time:time_per_file"] + '\n'; + env += 'BASE_PUBLISH="' + config["deploy:base_url"] + '"\n'; -// Generate resources: oldest timestamps, static data -outdirs.forEach(dir => { - const files = fs.readdirSync(path.join('out', dir)); - files.forEach(file => { + fs.writeFileSync('../.env.benchmark', env); - }) -}); + // Generate resources: oldest timestamps, static data + outdirs.forEach(dir => { + let hasStaticData = false; + let staticData = {triples: [], prefixes: []}; + const files = fs.readdirSync(path.join('out', dir)); + files.forEach(file => { + let contents = fs.readFileSync(path.join('out', dir, file), "utf8"); + let foundStaticData = false; + let realTimeData = {triples: [], prefixes: []}; + let triples = N3.Parser().parse(contents); + triples.forEach(t => { + if ((t.predicate === buildingBlocks.pRdfType && t.object === buildingBlocks.ParkingSite) || + t.predicate === buildingBlocks.pLabel || t.predicate === buildingBlocks.pNumberOfSpaces) { + if (!hasStaticData) { + foundStaticData = true; + staticData.triples.push(t); + } + } else { + realTimeData.triples.push(t); + } + }); + if (foundStaticData) { + hasStaticData = true; + let filename = path.join('resources', dir + '_static_data'); + let writer = N3.Writer(); + writer.addTriples(staticData.triples); + writer.addPrefixes(staticData.prefixes); + writer.end((e, r) => { + fs.writeFileSync(filename, r) + }); + } + let writer = N3.Writer(); + writer.addTriples(realTimeData.triples); + writer.addPrefixes(realTimeData.prefixes); + writer.end((e, r) => { + fs.writeFileSync(path.join('out', dir, file), r) + }); + }) + }); -// Deploy php -S \ No newline at end of file + // Deploy php -S +}); \ No newline at end of file From 68e5ce4f11caa723db2298281e84ee19e20c82fb Mon Sep 17 00:00:00 2001 From: Arne Gevaert Date: Tue, 7 Nov 2017 16:39:08 +0100 Subject: [PATCH 17/18] using npm library --- benchmark/config.json | 1 - benchmark/deploy_benchmark.js | 1 + 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/benchmark/config.json b/benchmark/config.json index 09c66de..2d92b2c 100644 --- a/benchmark/config.json +++ b/benchmark/config.json @@ -1,7 +1,6 @@ { "deploy:base_url": "example.com", "deploy:rangegate_config": "1Y,1M,1W,1D", - "deploy:lpdgen_location": "../../lpdgenerator/bin", "cities:number": 5, diff --git a/benchmark/deploy_benchmark.js b/benchmark/deploy_benchmark.js index ff4a441..544f9cd 100644 --- a/benchmark/deploy_benchmark.js +++ b/benchmark/deploy_benchmark.js @@ -2,6 +2,7 @@ const fs = require('fs'); const childProcess = require('child_process'); const path = require('path'); const N3 = require('n3'); +const lpdgen = require('lpdgenerator'); if (process.argv.length !== 3) { console.error("Usage: node deploy_benchmark.js CONFIG"); From 5ca00b9c8a7a076c16bc3fe1fbbc24fbccd88d32 Mon Sep 17 00:00:00 2001 From: Arne Gevaert Date: Tue, 7 Nov 2017 17:05:12 +0100 Subject: [PATCH 18/18] fix dotenv required --- src/Settings.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Settings.php b/src/Settings.php index 3e47e2a..c5bdda0 100644 --- a/src/Settings.php +++ b/src/Settings.php @@ -67,7 +67,7 @@ private function required() { $this->dotenv->required('RESOURCE_DIR'); $this->dotenv->required('DATASETS'); $this->dotenv->required('DATASETS_GATHER'); - $this->dotenv->required('DEFAULT_GATHER_INTERVAL'); + $this->dotenv->required('TIME_PER_FILE'); } private function parseDatasets() {