diff --git a/.gitignore b/.gitignore index 3169366..ff2037c 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,10 @@ +benchmark/node_modules/* +benchmark/out/* +benchmark/resources/* + # Hide .env from repo .env +.env.benchmark log/* sandbox.php diff --git a/benchmark/config.json b/benchmark/config.json new file mode 100644 index 0000000..2d92b2c --- /dev/null +++ b/benchmark/config.json @@ -0,0 +1,31 @@ +{ + "deploy:base_url": "example.com", + "deploy:rangegate_config": "1Y,1M,1W,1D", + + "cities:number": 5, + + "parkings:min_num_per_city": 1, + "parkings:max_num_per_city": 8, + "parkings:min_spaces": 100, + "parkings:max_spaces": 300, + + "time:start": "2017-09-25T12:00:00", + "time:end": "2017-09-26T12:00:00", + "time:time_per_file": 10800, + "time:interval": 30, + + "events:min_events": 1, + "events:max_events": 3, + "events:min_duration": 1, + "events:max_duration": 8, + "events:min_occupation": 0.2, + "events:max_occupation": 0.8, + + "events:min_lambda": 0.5, + "events:max_lambda": 2, + "events:lambda_var": 0.07, + + "events:min_sigma": 0.05, + "events:max_sigma": 1, + "events:sigma_var": 0.04 +} \ No newline at end of file diff --git a/benchmark/deploy_benchmark.js b/benchmark/deploy_benchmark.js new file mode 100644 index 0000000..544f9cd --- /dev/null +++ b/benchmark/deploy_benchmark.js @@ -0,0 +1,122 @@ +const fs = require('fs'); +const childProcess = require('child_process'); +const path = require('path'); +const N3 = require('n3'); +const lpdgen = require('lpdgenerator'); + +if (process.argv.length !== 3) { + console.error("Usage: node deploy_benchmark.js CONFIG"); + process.exit(1); +} + +const config = JSON.parse(fs.readFileSync(process.argv[2])); + +const buildingBlocks = { + 'ParkingSite': 'http://vocab.datex.org/terms#UrbanParkingSite', + 'pLabel': 'http://www.w3.org/2000/01/rdf-schema#label', + 'pNumberOfSpaces': 'http://vocab.datex.org/terms#parkingNumberOfSpaces', + 'pRdfType': 'http://www.w3.org/1999/02/22-rdf-syntax-ns#type' +}; + +// Create out/ and resources/ if necessary +if (!fs.existsSync('out')) { + fs.mkdirSync('out'); +} +if (!fs.existsSync('resources')) { + fs.mkdirSync('resources'); +} + +// Cleanup +let outdirs = fs.readdirSync('out'); +let resourcefiles = fs.readdirSync('resources'); +outdirs.forEach(dir => { + let files = fs.readdirSync(path.join('out', dir)); + files.forEach(file => { + fs.unlinkSync(path.join('out', dir, file), err => { + if (err) console.error(err); + }) + }); +}); +resourcefiles.forEach(file => { + fs.unlinkSync(path.join('resources', file), err => { + if (err) console.error(err); + }) +}); + +// Generate lpd config +config["file:output"] = 'out'; +config["file:output_meta_data"] = false; +config["file:extension"] = ''; +config["file:name_format"] = "UNIX"; +fs.writeFileSync('lpd_config.json', JSON.stringify(config)); + +// Generate data +let script = config['deploy:lpdgen_location'] + '/generator.js'; +let genProc = childProcess.fork(script, ['lpd_config.json']); +genProc.on('exit', code => { + console.log("Process exited with exit code", code); + + outdirs = fs.readdirSync('out'); + resourcefiles = fs.readdirSync('resources'); + + // Generate .env file + let env = ""; + let datasets = 'DATASETS="'; + outdirs.forEach(dir => { + let caps = dir.toUpperCase(); + env += caps + '_PUBLISH="http://' + dir + '.' + config["deploy:base_url"] + '"\n'; + datasets += caps + ','; + env += caps + '_PATH=oSoc\\Smartflanders\\Datasets\\Benchmark\\Benchmark' + '\n'; + }); + env += datasets.slice(0,-1) + '"\n'; + env += 'DATASETS_GATHER=""\n'; + env += 'RANGE_GATES_CONFIG="' + config["deploy:rangegate_config"] + '"\n'; + env += 'DATA_DIR="benchmark/out"\n'; + env += 'RESOURCE_DIR="benchmark/resources"\n'; + env += 'DEFAULT_GATHER_INTERVAL=' + config["time:time_per_file"] + '\n'; + env += 'BASE_PUBLISH="' + config["deploy:base_url"] + '"\n'; + + fs.writeFileSync('../.env.benchmark', env); + + // Generate resources: oldest timestamps, static data + outdirs.forEach(dir => { + let hasStaticData = false; + let staticData = {triples: [], prefixes: []}; + const files = fs.readdirSync(path.join('out', dir)); + files.forEach(file => { + let contents = fs.readFileSync(path.join('out', dir, file), "utf8"); + let foundStaticData = false; + let realTimeData = {triples: [], prefixes: []}; + let triples = N3.Parser().parse(contents); + triples.forEach(t => { + if ((t.predicate === buildingBlocks.pRdfType && t.object === buildingBlocks.ParkingSite) || + t.predicate === buildingBlocks.pLabel || t.predicate === buildingBlocks.pNumberOfSpaces) { + if (!hasStaticData) { + foundStaticData = true; + staticData.triples.push(t); + } + } else { + realTimeData.triples.push(t); + } + }); + if (foundStaticData) { + hasStaticData = true; + let filename = path.join('resources', dir + '_static_data'); + let writer = N3.Writer(); + writer.addTriples(staticData.triples); + writer.addPrefixes(staticData.prefixes); + writer.end((e, r) => { + fs.writeFileSync(filename, r) + }); + } + let writer = N3.Writer(); + writer.addTriples(realTimeData.triples); + writer.addPrefixes(realTimeData.prefixes); + writer.end((e, r) => { + fs.writeFileSync(path.join('out', dir, file), r) + }); + }) + }); + + // Deploy php -S +}); \ No newline at end of file diff --git a/benchmark/lpd_config.json b/benchmark/lpd_config.json new file mode 100644 index 0000000..8e055af --- /dev/null +++ b/benchmark/lpd_config.json @@ -0,0 +1 @@ +{"deploy:base_url":"example.com","deploy:rangegate_config":"1Y,1M,1W,1D","deploy:lpdgen_location":"../../lpdgenerator/bin","cities:number":5,"parkings:min_num_per_city":1,"parkings:max_num_per_city":8,"parkings:min_spaces":100,"parkings:max_spaces":300,"time:start":"2017-09-25T12:00:00","time:end":"2017-09-26T12:00:00","time:time_per_file":10800,"time:interval":30,"events:min_events":1,"events:max_events":3,"events:min_duration":1,"events:max_duration":8,"events:min_occupation":0.2,"events:max_occupation":0.8,"events:min_lambda":0.5,"events:max_lambda":2,"events:lambda_var":0.07,"events:min_sigma":0.05,"events:max_sigma":1,"events:sigma_var":0.04,"file:output":"out","file:output_meta_data":false,"file:extension":"","file:name_format":"UNIX"} \ No newline at end of file diff --git a/benchmark/package-lock.json b/benchmark/package-lock.json new file mode 100644 index 0000000..22fcb39 --- /dev/null +++ b/benchmark/package-lock.json @@ -0,0 +1,21 @@ +{ + "requires": true, + "lockfileVersion": 1, + "dependencies": { + "child_process": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/child_process/-/child_process-1.0.2.tgz", + "integrity": "sha1-sffn/HPSXn/R1FWtyU4UODAYK1o=" + }, + "fs": { + "version": "0.0.1-security", + "resolved": "https://registry.npmjs.org/fs/-/fs-0.0.1-security.tgz", + "integrity": "sha1-invTcYa23d84E/I4WLV+yq9eQdQ=" + }, + "n3": { + "version": "0.11.2", + "resolved": "https://registry.npmjs.org/n3/-/n3-0.11.2.tgz", + "integrity": "sha512-ICSiOmFLbZ4gI35+4H3e2vYGHDC944WZkCa1iVNRAx/mRZESEevQNFhfHaui/lhqynoZYvBVDNjM/2Tfd3TICQ==" + } + } +} diff --git a/benchmark/package.json b/benchmark/package.json new file mode 100644 index 0000000..7156faf --- /dev/null +++ b/benchmark/package.json @@ -0,0 +1,8 @@ +{ + "dependencies": { + "child_process": "^1.0.2", + "fs": "0.0.1-security", + "n3": "^0.11.2" + }, + "devDependencies": {} +} diff --git a/cron.php b/cron.php index b809ff2..0c01b84 100644 --- a/cron.php +++ b/cron.php @@ -22,36 +22,17 @@ } else if ($argv[1] === "debug") { acquire_data(); } -/** - * This function simply periodically saves the entire turtle file with the current ISO timestamp as filename - * + triples for timestamp and filename of previous file - */ + function acquire_data() { - $dotenv = new Dotenv(__DIR__); - $dotenv->load(); - $datasets = explode(',', $_ENV["DATASETS_GATHER"]); - $processors = array(); - foreach($datasets as $dataset) { - try { - $dotenv->required($dataset . "_PATH"); - $class = $_ENV[$dataset . "_PATH"]; - array_push($processors, new $class); - } catch (Exception $e) { - error_log("Invalid .env configuration: dataset " . $dataset . " was has no corresponding class path." - . " Please add the variable " . $dataset . "_PATH."); - } - } - foreach ($processors as $processor) { + $settings = \oSoc\Smartflanders\Settings::getInstance(); + foreach ($settings->getDatasetsGather() as $processor) { if ($processor->mustQuery()) { - $interval = 60*60*3; // 3 hour interval results in files of a few 100 KB - // TODO add out and resources directories to .env. Right now Dutch dataset class has dependency on this. - $fs = new Filesystem\FileWriter(__DIR__ . "/out", __DIR__ . "/resources", $interval, $processor); + $fs = new Filesystem\FileWriter(\oSoc\Smartflanders\Settings::getInstance(), $processor); echo "Fetching graph for " . $processor->getName() . "\n"; $graph = $processor->getDynamicGraph(); $now = time(); echo "Writing data for " . $processor->getName() . "\n"; $fs->writeToFile($now, $graph); - // Temporarily disabling range gates, semantically incorrect echo "Updating statistical summary for " . $processor->getName() . "\n"; $fs->updateStatisticalSummary($now, $graph); } diff --git a/datex_spec.xml b/datex_spec.xml deleted file mode 100644 index 059659a..0000000 --- a/datex_spec.xml +++ /dev/null @@ -1,78 +0,0 @@ - - - - - - - - Parkings per stad - - - **ISO TIME** - - - - **STAD** - - - - - - P07 Sint-Michiels - - - - - Sint-Michiels - - - 450 - - - - 1.0 - 2.0 - - - - - - - Sint-Michielsplein 8 9000 Gent - - - Tel.: 09 266 29 20 - - - - active - - - - 00:00:00.000+02:00 - 23:59:00.000+02:00 - - - - - - - ... - - - - - - - **ISO TIMESTAMP** - - 164 - - spacesAvailable - open - - ... - - - - \ No newline at end of file diff --git a/public/index.php b/public/index.php index 5fa8de3..410d95b 100644 --- a/public/index.php +++ b/public/index.php @@ -9,40 +9,9 @@ require __DIR__ . '/../vendor/autoload.php'; use oSoc\Smartflanders\Datasets; -use Dotenv\Dotenv; -$dotenv = new Dotenv(__DIR__ . '/../'); -$dotenv->load(); +$settings = Settings::getInstance(); -$processors = get_datasets_from_names(explode(',', $_ENV['DATASETS']), $dotenv); -$processors_gather = get_datasets_from_names(explode(',', $_ENV["DATASETS_GATHER"]), $dotenv); - -$nameToGP = []; -foreach($processors as $gp) { - $name = $gp->getName(); - $name_lower = strtolower($name); - $nameToGP[$name_lower] = $gp; -} - -$out_dirname = __DIR__ . "/../out"; -$res_dirname = __DIR__ . "/../resources"; -$second_interval = 60*60*3; // TODO store this in .env! - -$router = new Router($_SERVER['HTTP_HOST'], $out_dirname, $res_dirname, $second_interval, $nameToGP, $processors_gather); +$router = new Router($settings); $router->init(); -$router->run(); - -function get_datasets_from_names($names, $dotenv) { - $result = array(); - foreach($names as $dataset) { - try { - $dotenv->required($dataset . "_PATH"); - $class = $_ENV[$dataset . "_PATH"]; - array_push($result, new $class); - } catch (\Exception $e) { - error_log("Invalid .env configuration: dataset " . $dataset . " has no corresponding class path." - . " Please add the variable " . $dataset . "_PATH."); - } - } - return $result; -} \ No newline at end of file +$router->run(); \ No newline at end of file diff --git a/src/Datasets/Benchmark/Benchmark.php b/src/Datasets/Benchmark/Benchmark.php new file mode 100644 index 0000000..1e6d952 --- /dev/null +++ b/src/Datasets/Benchmark/Benchmark.php @@ -0,0 +1,58 @@ +publish_url = $publish; + $this->name = explode('.', $publish)[0]; + } + + public function getDynamicGraph() + { + return array(); + } + + public function getStaticGraph() + { + return array(); + } + + public function getName() + { + return $this->name; + } + + public function getBaseUrl() + { + return $this->publish_url; + } + + public function getRealTimeMaxAge() + { + return 30; + } + + public function mustQuery() + { + return true; + } + + public function setFetchUrl($url) + { + return; + } +} \ No newline at end of file diff --git a/src/Datasets/GentParking/GhentToRDF.php b/src/Datasets/GentParking/GhentToRDF.php index a459766..86fba89 100644 --- a/src/Datasets/GentParking/GhentToRDF.php +++ b/src/Datasets/GentParking/GhentToRDF.php @@ -7,7 +7,6 @@ namespace oSoc\Smartflanders\Datasets\GentParking; use oSoc\Smartflanders\Helpers; -use Dotenv; Class GhentToRDF implements Helpers\IGraphProcessor { @@ -19,14 +18,12 @@ private static $parkingURIs; private static $sameAs; - public function __construct() + public function __construct($publish) { - $dotenv = new Dotenv\Dotenv(__DIR__ . '/../../../'); - $dotenv->load(); $this->urls = [ self::STAT => "http://opendataportaalmobiliteitsbedrijf.stad.gent/datex2/v2/parkings/", self::DYN => "http://opendataportaalmobiliteitsbedrijf.stad.gent/datex2/v2/parkingsstatus", - self::BASE => $_ENV['GHENT_PUBLISH'] + self::BASE => $publish ]; } @@ -148,4 +145,6 @@ private static function preProcessing() return $graph; } + + public function setFetchUrl($url) {} } diff --git a/src/Datasets/Ixor/IxorLeuven.php b/src/Datasets/Ixor/IxorLeuven.php index 462381b..11c7af9 100644 --- a/src/Datasets/Ixor/IxorLeuven.php +++ b/src/Datasets/Ixor/IxorLeuven.php @@ -1,19 +1,9 @@ load(); - $fetch = $_ENV["IXOR_LEUVEN_FETCH"]; - $publish = $_ENV["IXOR_LEUVEN_PUBLISH"]; - parent::__construct($fetch, $publish); - } - public function getName() { return "Leuven"; diff --git a/src/Datasets/Ixor/IxorMechelen.php b/src/Datasets/Ixor/IxorMechelen.php index 0efda5b..6f0e4b5 100644 --- a/src/Datasets/Ixor/IxorMechelen.php +++ b/src/Datasets/Ixor/IxorMechelen.php @@ -1,19 +1,9 @@ load(); - $fetch = $_ENV["IXOR_MECHELEN_FETCH"]; - $publish = $_ENV["IXOR_MECHELEN_PUBLISH"]; - parent::__construct($fetch, $publish); - } - public function getName() { return "Mechelen"; diff --git a/src/Datasets/Ixor/IxorSintNiklaas.php b/src/Datasets/Ixor/IxorSintNiklaas.php index e9e876b..7b2b9d8 100644 --- a/src/Datasets/Ixor/IxorSintNiklaas.php +++ b/src/Datasets/Ixor/IxorSintNiklaas.php @@ -1,19 +1,9 @@ load(); - $fetch = $_ENV["IXOR_SINT-NIKLAAS_FETCH"]; - $publish = $_ENV["IXOR_SINT-NIKLAAS_PUBLISH"]; - parent::__construct($fetch, $publish); - } - public function getName() { return "Sint-Niklaas"; diff --git a/src/Datasets/Ixor/IxorToRDF.php b/src/Datasets/Ixor/IxorToRDF.php index a1ecadf..96d133e 100644 --- a/src/Datasets/Ixor/IxorToRDF.php +++ b/src/Datasets/Ixor/IxorToRDF.php @@ -9,9 +9,8 @@ abstract class IxorToRDF implements Helpers\IGraphProcessor private $fetch_url, $publish_url; private $authHeader; - public function __construct($fetch_url, $publish_url) + public function __construct($publish_url) { - $this->fetch_url = $fetch_url; $this->publish_url = $publish_url; $dotenv = new Dotenv\Dotenv(__DIR__ . '/../../../'); $dotenv->load(); @@ -73,4 +72,9 @@ public function mustQuery() { return true; } + + public function setFetchUrl($url) + { + $this->fetch_url = $url; + } } diff --git a/src/Datasets/Netherlands/NetherlandsToRDF.php b/src/Datasets/Netherlands/NetherlandsToRDF.php index 680b241..e0ee378 100644 --- a/src/Datasets/Netherlands/NetherlandsToRDF.php +++ b/src/Datasets/Netherlands/NetherlandsToRDF.php @@ -2,24 +2,18 @@ namespace oSoc\Smartflanders\Datasets\Netherlands; use oSoc\Smartflanders\Helpers; -use Dotenv; use \League\Flysystem\Adapter\Local; use \League\Flysystem\Filesystem; +use oSoc\Smartflanders\Settings; class NetherlandsToRDF implements Helpers\IGraphProcessor { - private $publish_url, $fetch_url, $res_fs; + private $publish_url, $fetch_url; - public function __construct() + public function __construct($publish) { - $dotenv = new Dotenv\Dotenv(__DIR__ . '/../../../'); - $dotenv->load(); - $this->publish_url = $_ENV['NEDERLAND_PUBLISH']; - $this->fetch_url = $_ENV['NEDERLAND_FETCH']; - // TODO this is a bad dependency. Resources directory should be added to .env. - $res_adapter = new Local(__DIR__ . '/../../../resources'); - $this->res_fs = new Filesystem($res_adapter); + $this->publish_url = $publish; } public function getDynamicGraph() @@ -97,20 +91,24 @@ public function getRealTimeMaxAge() public function mustQuery() { + $settings = Settings::getInstance(); + $res_adapter = new Local($settings->getResourcesDir()); + $res_fs = new Filesystem($res_adapter); + $filename = 'Nederland_last_measurement'; $now = time(); - if ($this->res_fs->has($filename)) { - $string_value = $this->res_fs->read($filename); + if ($res_fs->has($filename)) { + $string_value = $res_fs->read($filename); $int_value = intval($string_value); if ($now - $int_value > 30*60) { // Only query once every 30 minutes - $this->res_fs->put($filename, $now); + $res_fs->put($filename, $now); return true; } else { return false; } } - $this->res_fs->write($filename, $now); + $res_fs->write($filename, $now); return true; } @@ -127,4 +125,9 @@ private function getAccessibleParkings() { } return $accessible_parkings; } + + public function setFetchUrl($url) + { + $this->fetch_url = $url; + } } \ No newline at end of file diff --git a/src/Datasets/ParkoKortrijk/ParkoToRDF.php b/src/Datasets/ParkoKortrijk/ParkoToRDF.php index 2a29141..d8b61fd 100644 --- a/src/Datasets/ParkoKortrijk/ParkoToRDF.php +++ b/src/Datasets/ParkoKortrijk/ParkoToRDF.php @@ -3,18 +3,14 @@ namespace oSoc\Smartflanders\Datasets\ParkoKortrijk; use oSoc\Smartflanders\Helpers; -use Dotenv; class ParkoToRDF implements Helpers\IGraphProcessor { private $publish_url, $fetch_url; - public function __construct() + public function __construct($publish) { - $dotenv = new Dotenv\Dotenv(__DIR__ . '/../../../'); - $dotenv->load(); - $this->publish_url = $_ENV['PARKO_KORTRIJK_PUBLISH']; - $this->fetch_url = $_ENV['PARKO_KORTRIJK_FETCH']; + $this->publish_url = $publish; } public function getDynamicGraph() @@ -84,4 +80,9 @@ public function mustQuery() { return true; } + + public function setFetchUrl($url) + { + $this->fetch_url = $url; + } } diff --git a/src/Filesystem/FileReader.php b/src/Filesystem/FileReader.php index bd81a42..f404d17 100644 --- a/src/Filesystem/FileReader.php +++ b/src/Filesystem/FileReader.php @@ -183,7 +183,7 @@ private function getNextFileFromTimestamp($timestamp) { if ($this->out_fs->has($next_ts)) { return date("Y-m-d\TH:i:s", $next_ts); } else { - return $this->getNextFileFromTimestamp($next_ts + $this->second_interval); + return $this->getNextFileFromTimestamp($next_ts + $this->settings->getTimePerFile()); } } return false; @@ -193,7 +193,7 @@ private function getNextFileFromTimestamp($timestamp) { private function getPreviousFileFromTimestamp($timestamp) { $prev_ts = $this->getPreviousTimestampFromTimestamp($timestamp); if ($prev_ts) { - $prev_ts -= $this->second_interval; + $prev_ts -= $this->settings->getTimePerFile(); if ($this->out_fs->has($prev_ts)) { return date("Y-m-d\TH:i:s", $prev_ts); } else { diff --git a/src/Filesystem/FileSystemProcessor.php b/src/Filesystem/FileSystemProcessor.php index 88b3a22..ec46523 100644 --- a/src/Filesystem/FileSystemProcessor.php +++ b/src/Filesystem/FileSystemProcessor.php @@ -7,6 +7,7 @@ use \League\Flysystem\Adapter\Local; use \League\Flysystem\Filesystem; use oSoc\Smartflanders\Helpers\IGraphProcessor; +use oSoc\Smartflanders\Settings; use pietercolpaert\hardf\TriGWriter; @@ -14,25 +15,21 @@ protected $out_fs; protected $res_fs; protected $stat_fs; - protected $second_interval; protected $writer; protected $graph_processor; protected $static_data_filename; - protected $out_dirname; - protected $res_dirname; + protected $settings; const REFRESH_STATIC = false; - public function __construct($out_dirname, $res_dirname, $second_interval, IGraphProcessor $graph_processor) + public function __construct(Settings $settings, IGraphProcessor $graph_processor) { - $this->out_dirname = $out_dirname; - $this->res_dirname = $res_dirname; - $this->second_interval = $second_interval; + $this->settings = $settings; date_default_timezone_set("Europe/Brussels"); - $out_adapter = new Local($out_dirname . "/" . $graph_processor->getName()); + $out_adapter = new Local($this->settings->getOutDir() . "/" . $graph_processor->getName()); $this->out_fs = new Filesystem($out_adapter); - $res_adapter = new Local($res_dirname); + $res_adapter = new Local($this->settings->getResourcesDir()); $this->res_fs = new Filesystem($res_adapter); - $stat_adapter = new Local($out_dirname . "/" . $graph_processor->getName() . "/statistical"); + $stat_adapter = new Local($this->settings->getOutDir() . "/" . $graph_processor->getName() . "/statistical"); $this->stat_fs = new Filesystem($stat_adapter); $this->graph_processor = $graph_processor; $this->static_data_filename = $graph_processor->getName() . "_static_data.turtle"; @@ -51,7 +48,7 @@ public function getFilesForDay(\DateTime $day) { $start = $start - ($start % 60*60*24); // Round down to day $start = $this->getPreviousTimestampFromTimestamp($start); // Get valid timestamp $end = $start + 60*60*24; - for ($i = $start; $i < $end; $i += $this->second_interval) { + for ($i = $start; $i < $end; $i += $this->settings->getTimePerFile()) { if ($this->hasFile($i)) { array_push($result, $i); } @@ -59,16 +56,6 @@ public function getFilesForDay(\DateTime $day) { return $result; } - public function getSecondInterval() - { - return $this->second_interval; - } - - public function setSecondInterval($second_interval) - { - $this->second_interval = $second_interval; - } - // Get the last written page (closest to now) public function getLastPage() { return $this->getPreviousTimestampFromTimestamp(time()); @@ -85,7 +72,7 @@ public function getOldestTimestamp() { // Round a timestamp to its respective file timestamp protected function roundTimestamp($timestamp) { - $timestamp -= $timestamp % $this->second_interval; + $timestamp -= $timestamp % $this->settings->getTimePerFile(); return $timestamp; } @@ -98,7 +85,7 @@ public function getPreviousTimestampFromTimestamp($timestamp) { if ($this->out_fs->has($filename)) { return $timestamp; } - $timestamp -= $this->second_interval; + $timestamp -= $this->settings->getTimePerFile(); } } return false; @@ -108,7 +95,7 @@ public function getNextTimestampForTimestamp($timestamp) { $timestamp = $this->roundTimestamp($timestamp); $now = time(); while($timestamp < $now) { - $timestamp += $this->second_interval; + $timestamp += $this->settings->getTimePerFile(); $filename = $this->roundTimestamp($timestamp); if ($this->out_fs->has($filename)) { return $timestamp; @@ -122,10 +109,10 @@ public function hasFile($filename) { } public function getFileReader() { - return new FileReader($this->out_dirname, $this->res_dirname, $this->second_interval, $this->graph_processor); + return new FileReader($this->settings, $this->graph_processor); } public function getFileWriter() { - return new FileWriter($this->out_dirname, $this->res_dirname, $this->second_interval, $this->graph_processor); + return new FileWriter($this->settings, $this->graph_processor); } } diff --git a/src/Filesystem/FileWriter.php b/src/Filesystem/FileWriter.php index 216fbb8..2d32602 100644 --- a/src/Filesystem/FileWriter.php +++ b/src/Filesystem/FileWriter.php @@ -2,6 +2,7 @@ namespace oSoc\Smartflanders\Filesystem; +use oSoc\Smartflanders\Settings; use pietercolpaert\hardf\TriGWriter; use pietercolpaert\hardf\TriGParser; use pietercolpaert\hardf\Util; @@ -11,9 +12,9 @@ class FileWriter extends FileSystemProcessor { private $oldest_timestamp_filename; - public function __construct($out_dirname, $res_dirname, $second_interval, Helpers\IGraphProcessor $graph_processor) + public function __construct(Settings $settings, Helpers\IGraphProcessor $graph_processor) { - parent::__construct($out_dirname, $res_dirname, $second_interval, $graph_processor); + parent::__construct($settings, $graph_processor); $this->oldest_timestamp_filename = $this->graph_processor->getName() . "_oldest_timestamp"; } diff --git a/src/Helpers/IGraphProcessor.php b/src/Helpers/IGraphProcessor.php index 2df494b..3bca84d 100644 --- a/src/Helpers/IGraphProcessor.php +++ b/src/Helpers/IGraphProcessor.php @@ -9,4 +9,5 @@ public function getName(); public function getBaseUrl(); public function getRealTimeMaxAge(); public function mustQuery(); + public function setFetchUrl($url); } diff --git a/src/RangeGate/RangeGate.php b/src/RangeGate/RangeGate.php index 65f1d80..1aa55af 100644 --- a/src/RangeGate/RangeGate.php +++ b/src/RangeGate/RangeGate.php @@ -2,7 +2,6 @@ namespace oSoc\Smartflanders\RangeGate; -use Dotenv\Dotenv; use oSoc\Smartflanders\Filesystem\FileSystemProcessor; use oSoc\Smartflanders\Helpers\IGraphProcessor; use oSoc\Smartflanders\Helpers\TripleHelper; @@ -24,9 +23,6 @@ public function __construct($gatename, IGraphProcessor $dataset, FileSystemProce $this->dataset = $dataset; $this->fs = $fs; - $dotenv = new Dotenv(__DIR__ . '/../../'); - $dotenv->load(); - $this->intervalCalculator = new RangeGateIntervalCalculator($_ENV['RANGE_GATES_CONFIG'], $fs->getOldestTimestamp()); $this->baseUrl = $this->dataset->getBaseUrl() . '/rangegate/'; if ($this->gatename !== self::$ROOT_GATE) { diff --git a/src/RangeGate/RangeGateIntervalCalculator.php b/src/RangeGate/RangeGateIntervalCalculator.php index f6d5af7..20a2f8d 100644 --- a/src/RangeGate/RangeGateIntervalCalculator.php +++ b/src/RangeGate/RangeGateIntervalCalculator.php @@ -15,10 +15,6 @@ class RangeGateIntervalCalculator ); public function __construct($configString, $oldest_timestamp) { - // Round the oldest timestamp up to 1 day - /*$day = 60*60*24; - $rest = $oldest_timestamp % $day; - $this->oldest = $oldest_timestamp + $day - $rest - 2*60*60;*/ $this->oldest = \DateTime::createFromFormat('U', $oldest_timestamp); // Parse the configuration string @@ -47,13 +43,6 @@ public function isLegal($intervalString) { return false; } - // Determine if start is valid (must be a multiple of diff greater than oldest_timestamp) - //$start_relative = $interval[0] - $this->oldest; - /*$start_relative = $interval[0]->diff($this->oldest)->days; - var_dump($start_relative % $diff); - if ($start_relative % $diff !== 0) { - return false; - }*/ return true; } @@ -68,13 +57,6 @@ public function getRootSubRangeGates() { return $this->calculateSubRangeGates(-1, $this->oldest); } - /*private function dayDiff($interval) { - $from = new \DateTime(date('c', $interval[0])); - $to = new \DateTime(date('c', $interval[1])); - $diff = $from->diff($to); - return $diff->days; - }*/ - private function calculateSubRangeGates($level_index, \DateTime $lower_bound) { $result = array(); if ($level_index < count($this->levels)-1) { diff --git a/src/Router.php b/src/Router.php index 1b45358..783c8d5 100644 --- a/src/Router.php +++ b/src/Router.php @@ -3,58 +3,48 @@ namespace oSoc\Smartflanders; use Bramus; -use Dotenv\Dotenv; use oSoc\Smartflanders\RangeGate; class Router { private $router; - private $http_host; - private $out_dirname; - private $res_dirname; - private $second_interval; - private $nameToGP; - private $processors_gather; + private $settings; + private $nameToGP = array(); - public function __construct($http_host, $out_dirname, $res_dirname, $second_interval, $nameToGP, $processors_gather) { + public function __construct(Settings $settings) { $this->router = new Bramus\Router\Router(); $this->router->set404(function() {echo "Page not found.";}); - $this->http_host = $http_host; - $this->out_dirname = $out_dirname; - $this->res_dirname = $res_dirname; - $this->second_interval = $second_interval; - $this->nameToGP = $nameToGP; - $this->processors_gather = $processors_gather; + foreach($settings->getDatasets() as $gp) { + $name = $gp->getName(); + $name_lower = strtolower($name); + $this->nameToGP[$name_lower] = $gp; + } - $dotenv = new Dotenv(__DIR__ . '/../'); - $dotenv->load(); + $this->settings = $settings; } public function init() { - $out_dirname = $this->out_dirname; - $res_dirname = $this->res_dirname; - $second_interval = $this->second_interval; - $processors_gather = $this->processors_gather; + $settings = $this->settings; $found = false; - $dataset_name = explode('.', $this->http_host)[0]; + $dataset_name = explode('.', $_SERVER['HTTP_HOST'])[0]; $dataset = null; $fs = null; $calc = null; foreach($this->nameToGP as $name => $gp) { if ($name === $dataset_name) { $found = true; $dataset = $this->nameToGP[$dataset_name]; - $fs = new Filesystem\FileSystemProcessor($out_dirname, $res_dirname ,$second_interval, $dataset); + $fs = new Filesystem\FileSystemProcessor($this->settings, $dataset); } } $this->router->get('/parking', - function() use ($found, $dataset, $out_dirname, $res_dirname, $second_interval, $processors_gather) { + function() use ($settings, $found, $dataset, $dataset_name) { if ($found) { - View::view($dataset, $out_dirname, $res_dirname, $second_interval, $processors_gather); + View::view($settings, $dataset); } else { http_response_code(404); - die("Route not found: " . $dataset); + die("Dataset not found: " . $dataset_name); } } ); diff --git a/src/Settings.php b/src/Settings.php new file mode 100644 index 0000000..c5bdda0 --- /dev/null +++ b/src/Settings.php @@ -0,0 +1,110 @@ +has('.env.benchmark')) { + $this->dotenv = new Dotenv(__DIR__ . '/../', '.env.benchmark'); + } else { + $this->dotenv = new Dotenv(__DIR__ . '/../'); + } + + $this->dotenv->load(); + + $this->required(); + $this->parseDatasets(); + + $this->data_dir = __DIR__ . '/../' . $_ENV["DATA_DIR"]; + $this->resource_dir = __DIR__ . '/../' . $_ENV["RESOURCE_DIR"]; + $this->time_per_file = $_ENV["TIME_PER_FILE"]; + $this->range_gates_config = $_ENV["RANGE_GATES_CONFIG"]; + } + + function getOutDir() { + return $this->data_dir; + } + + function getResourcesDir() { + return $this->resource_dir; + } + + function getDatasets() { + return $this->datasets; + } + + function getDatasetsGather() { + return $this->datasets_gather; + } + + function getTimePerFile() { + return $this->time_per_file; + } + + function getRangeGatesConfig() { + return $this->range_gates_config; + } + + private function required() { + $this->dotenv->required('DATA_DIR'); + $this->dotenv->required('RESOURCE_DIR'); + $this->dotenv->required('DATASETS'); + $this->dotenv->required('DATASETS_GATHER'); + $this->dotenv->required('TIME_PER_FILE'); + } + + private function parseDatasets() { + $datasets = explode(',', $_ENV['DATASETS']); + $datasets_gather = explode(',', $_ENV['DATASETS_GATHER']); + + foreach($datasets as $dataset) { + array_push($this->datasets, $this->parseDataset($dataset)); + } + + foreach($datasets_gather as $dataset) { + array_push($this->datasets_gather, $this->parseDataset($dataset)); + } + } + + private function parseDataset($dataset) { + try { + $this->dotenv->required($dataset . "_PATH"); + $this->dotenv->required($dataset . "_PUBLISH"); + $class = $_ENV[$dataset . "_PATH"]; + $publish = $_ENV[$dataset . "_PUBLISH"]; + $processor = new $class($publish); + if (isset($_ENV[$dataset . "_FETCH"])) { + $processor->setFetchUrl($_ENV[$dataset . "_FETCH"]); + } + return $processor; + } catch (\Exception $e) { + error_log("Invalid .env configuration: dataset " . $dataset . " has no corresponding class path." + . " Please add the variable " . $dataset . "_PATH."); + } + return null; + } + + public static function getInstance() { + if (self::$instance === null) { + self::$instance = new Settings(); + } + return self::$instance; + } +} \ No newline at end of file diff --git a/src/View.php b/src/View.php index fbf9a14..32d2fe6 100644 --- a/src/View.php +++ b/src/View.php @@ -55,7 +55,8 @@ public static function viewRangeGate($rangegate) { } } - public static function view($graph_processor, $out_dirname, $res_dirname, $second_interval, $processors_gather) { + public static function view(Settings $settings, Helpers\IGraphProcessor $graph_processor) { + $processors_gather = $settings->getDatasetsGather(); if (in_array($graph_processor, $processors_gather)) { // This data is being gathered here, get the file // If no preferred content type is specified, prefer turtle @@ -65,7 +66,7 @@ public static function view($graph_processor, $out_dirname, $res_dirname, $secon $filename = null; - $fs = new Filesystem\FileSystemProcessor($out_dirname, $res_dirname ,$second_interval, $graph_processor); + $fs = new Filesystem\FileSystemProcessor($settings, $graph_processor); if (!isset($_GET['page']) && !isset($_GET['time'])) { $timestamp = $fs->getLastPage(); @@ -96,7 +97,7 @@ public static function view($graph_processor, $out_dirname, $res_dirname, $secon header('Location: ' . $graph_processor->getBaseUrl() . '?page=' . $filename); } else { // This is sloppy coding - $fileReader = new Filesystem\FileReader($out_dirname, $res_dirname ,$second_interval, $graph_processor); + $fileReader = new Filesystem\FileReader($settings, $graph_processor); $graphs = $fileReader->getFullyDressedGraphsFromFile($filename); $historic = true; if ((string)$filename === $fs->getLastPage()) { diff --git a/src/index-singular.php b/src/index-singular.php deleted file mode 100644 index 37d05ad..0000000 --- a/src/index-singular.php +++ /dev/null @@ -1,51 +0,0 @@ -getLastPage(); -} - -else if (isset($_GET['page'])) { - // If page name is provided, it must be exact - $filename = $_GET['page']; - if (!$fs->hasFile($filename)) { - http_response_code(404); - die("Page not found"); - } -} - -else if (isset($_GET['time'])) { - // If timestamp is provided, find latest file before timestamp - $filename = $fs->getClosestPage(strtotime($_GET['time'])); - if (!$filename) { - http_response_code(404); - die("Time not found"); - } -} - -if (!isset($_GET['page'])) { - header("Access-Control-Allow-Origin: *"); - header('Location: ' . $graph_processor->getBaseUrl() . '?page=' . $filename); -} else { - // This is sloppy coding - $fileReader = new Filesystem\FileReader($out_dirname, $res_dirname ,$second_interval, $graph_processor); - $graphs = $fileReader->getFullyDressedGraphsFromFile($filename); - $historic = true; - if ($filename === $fs->getLastPage()) { - $historic = false; - } - View::view($_SERVER['HTTP_ACCEPT'], $graphs, $historic, $graph_processor->getBaseUrl(), $graph_processor->getRealTimeMaxAge()); -} \ No newline at end of file diff --git a/test/TimeStampCalculatorTest.php b/test/TimeStampCalculatorTest.php deleted file mode 100644 index 7edc043..0000000 --- a/test/TimeStampCalculatorTest.php +++ /dev/null @@ -1,38 +0,0 @@ -assertEquals($calculators[0],$calculators[1]); - } - - /** - * @dataProvider prov - */ - public function testGetPrevious($calculators) { - $this->assertEquals($calculators[0],$calculators[2]); - } - - /** - * @dataProvider prov - */ - public function testGetFor($calculators) { - $this->assertEquals($calculators[0],$calculators[3]); - } - - public function prov() { - $res = array(); - for ($i = 5; $i <= 20; $i+=5) { - array_push($res, [new TimeStampCalculator($i*60)]); - } - return $res; - // for each array the methods will be called with the contents of this array as arguments - } -} \ No newline at end of file