Automatic deployment of generated data

.gitignore

@@ -1,5 +1,10 @@
+benchmark/node_modules/*
+benchmark/out/*
+benchmark/resources/*
+
 # Hide .env from repo
 .env
+.env.benchmark
 
 log/*
 sandbox.php

benchmark/config.json

@@ -0,0 +1,31 @@
+{
+  "deploy:base_url": "example.com",
+  "deploy:rangegate_config": "1Y,1M,1W,1D",
+
+  "cities:number": 5,
+
+  "parkings:min_num_per_city": 1,
+  "parkings:max_num_per_city": 8,
+  "parkings:min_spaces": 100,
+  "parkings:max_spaces": 300,
+
+  "time:start": "2017-09-25T12:00:00",
+  "time:end": "2017-09-26T12:00:00",
+  "time:time_per_file": 10800,
+  "time:interval": 30,
+
+  "events:min_events": 1,
+  "events:max_events": 3,
+  "events:min_duration": 1,
+  "events:max_duration": 8,
+  "events:min_occupation": 0.2,
+  "events:max_occupation": 0.8,
+
+  "events:min_lambda": 0.5,
+  "events:max_lambda": 2,
+  "events:lambda_var": 0.07,
+
+  "events:min_sigma": 0.05,
+  "events:max_sigma": 1,
+  "events:sigma_var": 0.04
+}

benchmark/deploy_benchmark.js

@@ -0,0 +1,122 @@
+const fs = require('fs');
+const childProcess = require('child_process');
+const path = require('path');
+const N3 = require('n3');
+const lpdgen = require('lpdgenerator');
+
+if (process.argv.length !== 3) {
+    console.error("Usage: node deploy_benchmark.js CONFIG");
+    process.exit(1);
+}
+
+const config = JSON.parse(fs.readFileSync(process.argv[2]));
+
+const buildingBlocks = {
+    'ParkingSite': 'http://vocab.datex.org/terms#UrbanParkingSite',
+    'pLabel': 'http://www.w3.org/2000/01/rdf-schema#label',
+    'pNumberOfSpaces': 'http://vocab.datex.org/terms#parkingNumberOfSpaces',
+    'pRdfType': 'http://www.w3.org/1999/02/22-rdf-syntax-ns#type'
+};
+
+// Create out/ and resources/ if necessary
+if (!fs.existsSync('out')) {
+    fs.mkdirSync('out');
+}
+if (!fs.existsSync('resources')) {
+    fs.mkdirSync('resources');
+}
+
+// Cleanup
+let outdirs = fs.readdirSync('out');
+let resourcefiles = fs.readdirSync('resources');
+outdirs.forEach(dir => {
+    let files = fs.readdirSync(path.join('out', dir));
+    files.forEach(file => {
+        fs.unlinkSync(path.join('out', dir, file), err => {
+            if (err) console.error(err);
+        })
+    });
+});
+resourcefiles.forEach(file => {
+    fs.unlinkSync(path.join('resources', file), err => {
+        if (err) console.error(err);
+    })
+});
+
+// Generate lpd config
+config["file:output"] = 'out';
+config["file:output_meta_data"] = false;
+config["file:extension"] = '';
+config["file:name_format"] = "UNIX";
+fs.writeFileSync('lpd_config.json', JSON.stringify(config));
+
+// Generate data
+let script = config['deploy:lpdgen_location'] + '/generator.js';
+let genProc = childProcess.fork(script, ['lpd_config.json']);
+genProc.on('exit', code => {
+    console.log("Process exited with exit code", code);
+
+    outdirs = fs.readdirSync('out');
+    resourcefiles = fs.readdirSync('resources');
+
+    // Generate .env file
+    let env = "";
+    let datasets = 'DATASETS="';
+    outdirs.forEach(dir => {
+        let caps = dir.toUpperCase();
+        env += caps + '_PUBLISH="http://' + dir + '.' + config["deploy:base_url"] + '"\n';
+        datasets += caps + ',';
+        env += caps + '_PATH=oSoc\\Smartflanders\\Datasets\\Benchmark\\Benchmark' + '\n';
+    });
+    env += datasets.slice(0,-1) + '"\n';
+    env += 'DATASETS_GATHER=""\n';
+    env += 'RANGE_GATES_CONFIG="' + config["deploy:rangegate_config"] + '"\n';
+    env += 'DATA_DIR="benchmark/out"\n';
+    env += 'RESOURCE_DIR="benchmark/resources"\n';
+    env += 'DEFAULT_GATHER_INTERVAL=' + config["time:time_per_file"] + '\n';
+    env += 'BASE_PUBLISH="' + config["deploy:base_url"] + '"\n';
+
+    fs.writeFileSync('../.env.benchmark', env);
+
+    // Generate resources: oldest timestamps, static data
+    outdirs.forEach(dir => {
+        let hasStaticData = false;
+        let staticData = {triples: [], prefixes: []};
+        const files = fs.readdirSync(path.join('out', dir));
+        files.forEach(file => {
+            let contents = fs.readFileSync(path.join('out', dir, file), "utf8");
+            let foundStaticData = false;
+            let realTimeData = {triples: [], prefixes: []};
+            let triples = N3.Parser().parse(contents);
+            triples.forEach(t => {
+                    if ((t.predicate === buildingBlocks.pRdfType && t.object === buildingBlocks.ParkingSite) ||
+                        t.predicate === buildingBlocks.pLabel || t.predicate === buildingBlocks.pNumberOfSpaces) {
+                        if (!hasStaticData) {
+                            foundStaticData = true;
+                            staticData.triples.push(t);
+                        }
+                    } else {
+                        realTimeData.triples.push(t);
+                    }
+                });
+            if (foundStaticData) {
+                hasStaticData = true;
+                let filename = path.join('resources', dir + '_static_data');
+                let writer = N3.Writer();
+                writer.addTriples(staticData.triples);
+                writer.addPrefixes(staticData.prefixes);
+                writer.end((e, r) => {
+                    fs.writeFileSync(filename, r)
+                });
+            }
+            let writer = N3.Writer();
+            writer.addTriples(realTimeData.triples);
+            writer.addPrefixes(realTimeData.prefixes);
+            writer.end((e, r) => {
+                fs.writeFileSync(path.join('out', dir, file), r)
+            });
+        })
+    });
+
+    // Deploy php -S
+});

benchmark/lpd_config.json

@@ -0,0 +1 @@
+{"deploy:base_url":"example.com","deploy:rangegate_config":"1Y,1M,1W,1D","deploy:lpdgen_location":"../../lpdgenerator/bin","cities:number":5,"parkings:min_num_per_city":1,"parkings:max_num_per_city":8,"parkings:min_spaces":100,"parkings:max_spaces":300,"time:start":"2017-09-25T12:00:00","time:end":"2017-09-26T12:00:00","time:time_per_file":10800,"time:interval":30,"events:min_events":1,"events:max_events":3,"events:min_duration":1,"events:max_duration":8,"events:min_occupation":0.2,"events:max_occupation":0.8,"events:min_lambda":0.5,"events:max_lambda":2,"events:lambda_var":0.07,"events:min_sigma":0.05,"events:max_sigma":1,"events:sigma_var":0.04,"file:output":"out","file:output_meta_data":false,"file:extension":"","file:name_format":"UNIX"}

benchmark/package.json

@@ -0,0 +1,8 @@
+{
+  "dependencies": {
+    "child_process": "^1.0.2",
+    "fs": "0.0.1-security",
+    "n3": "^0.11.2"
+  },
+  "devDependencies": {}
+}

cron.php

@@ -22,36 +22,17 @@
 } else if ($argv[1] === "debug") {
     acquire_data();
 }
-/**
- * This function simply periodically saves the entire turtle file with the current ISO timestamp as filename
- * + triples for timestamp and filename of previous file
- */
+
 function acquire_data() {
-    $dotenv = new Dotenv(__DIR__);
-    $dotenv->load();
-    $datasets = explode(',', $_ENV["DATASETS_GATHER"]);
-    $processors = array();
-    foreach($datasets as $dataset) {
-        try {
-            $dotenv->required($dataset . "_PATH");
-            $class = $_ENV[$dataset . "_PATH"];
-            array_push($processors, new $class);
-        } catch (Exception $e) {
-            error_log("Invalid .env configuration: dataset " . $dataset . " was has no corresponding class path."
-            . " Please add the variable " . $dataset . "_PATH.");
-        }
-    }
-    foreach ($processors as $processor) {
+    $settings = \oSoc\Smartflanders\Settings::getInstance();
+    foreach ($settings->getDatasetsGather() as $processor) {
         if ($processor->mustQuery()) {
-            $interval = 60*60*3; // 3 hour interval results in files of a few 100 KB
-            // TODO add out and resources directories to .env. Right now Dutch dataset class has dependency on this.
-            $fs = new Filesystem\FileWriter(__DIR__ . "/out", __DIR__ . "/resources", $interval, $processor);
+            $fs = new Filesystem\FileWriter(\oSoc\Smartflanders\Settings::getInstance(), $processor);
             echo "Fetching graph for " . $processor->getName() . "\n";
             $graph = $processor->getDynamicGraph();
             $now = time();
             echo "Writing data for " . $processor->getName() . "\n";
             $fs->writeToFile($now, $graph);
-            // Temporarily disabling range gates, semantically incorrect
             echo "Updating statistical summary for " . $processor->getName() . "\n";
             $fs->updateStatisticalSummary($now, $graph);
         }

public/index.php

@@ -9,40 +9,9 @@
 require __DIR__ . '/../vendor/autoload.php';
 
 use oSoc\Smartflanders\Datasets;
-use Dotenv\Dotenv;
 
-$dotenv = new Dotenv(__DIR__ . '/../');
-$dotenv->load();
+$settings = Settings::getInstance();
 
-$processors = get_datasets_from_names(explode(',', $_ENV['DATASETS']), $dotenv);
-$processors_gather = get_datasets_from_names(explode(',', $_ENV["DATASETS_GATHER"]), $dotenv);
-
-$nameToGP = [];
-foreach($processors as $gp) {
-    $name = $gp->getName();
-    $name_lower = strtolower($name);
-    $nameToGP[$name_lower] = $gp;
-}
-
-$out_dirname = __DIR__ . "/../out";
-$res_dirname = __DIR__ . "/../resources";
-$second_interval = 60*60*3; // TODO store this in .env!
-
-$router = new Router($_SERVER['HTTP_HOST'], $out_dirname, $res_dirname, $second_interval, $nameToGP, $processors_gather);
+$router = new Router($settings);
 $router->init();
-$router->run();
-
-function get_datasets_from_names($names, $dotenv) {
-    $result = array();
-    foreach($names as $dataset) {
-        try {
-            $dotenv->required($dataset . "_PATH");
-            $class = $_ENV[$dataset . "_PATH"];
-            array_push($result, new $class);
-        } catch (\Exception $e) {
-            error_log("Invalid .env configuration: dataset " . $dataset . " has no corresponding class path."
-                . " Please add the variable " . $dataset . "_PATH.");
-        }
-    }
-    return $result;
-}
+$router->run();