Updated files of wegenenverkeer vocabularies + extracting more terms + added extra logging for elastic

ddvlanck · ddvlanck · commit d3f1b2bc894a · 2021-02-10T15:50:36.000+01:00
diff --git a/files/wegenenverkeer-vocabularies.txt b/files/wegenenverkeer-vocabularies.txt
@@ -1,7 +1,6 @@
-https://raw.githubusercontent.com/Informatievlaanderen/OSLO-Generated/wegenenverkeer/report/doc/vocabularium/wegenenverkeer/abstracten/ontwerpdocument/v29/html-nj.json
-https://raw.githubusercontent.com/Informatievlaanderen/OSLO-Generated/wegenenverkeer/report/doc/vocabularium/wegenenverkeer/implementatieelement/ontwerpdocument/v29/html-nj.json
-https://raw.githubusercontent.com/Informatievlaanderen/OSLO-Generated/wegenenverkeer/report/doc/vocabularium/wegenenverkeer/installatie/ontwerpdocument/v29/html-nj.json
-https://raw.githubusercontent.com/Informatievlaanderen/OSLO-Generated/wegenenverkeer/report/doc/vocabularium/wegenenverkeer/levelofgeometry/ontwerpdocument/v29/html-nj.json
-https://raw.githubusercontent.com/Informatievlaanderen/OSLO-Generated/wegenenverkeer/report/doc/vocabularium/wegenenverkeer/levenscyclus/ontwerpdocument/v29/html-nj.json
-https://raw.githubusercontent.com/Informatievlaanderen/OSLO-Generated/wegenenverkeer/report/doc/vocabularium/wegenenverkeer/onderdeel/ontwerpdocument/v29/html-nj.json
-https://raw.githubusercontent.com/Informatievlaanderen/OSLO-Generated/wegenenverkeer/report/doc/vocabularium/wegenenverkeer/proefenmeting/ontwerpdocument/v29/html-nj.json
+https://raw.githubusercontent.com/Informatievlaanderen/OSLO-Generated/wegenenverkeer/report/doc/vocabularium/wegenenverkeer/abstracten/ontwerpdocument/1.0.0/html-nj.json
+https://raw.githubusercontent.com/Informatievlaanderen/OSLO-Generated/wegenenverkeer/report/doc/vocabularium/wegenenverkeer/implementatieelement/ontwerpdocument/1.0.0/html-nj.json
+https://raw.githubusercontent.com/Informatievlaanderen/OSLO-Generated/wegenenverkeer/report/doc/vocabularium/wegenenverkeer/installatie/ontwerpdocument/1.0.0/html-nj.json
+https://raw.githubusercontent.com/Informatievlaanderen/OSLO-Generated/wegenenverkeer/report/doc/vocabularium/wegenenverkeer/levenscyclus/ontwerpdocument/1.0.0/html-nj.json
+https://raw.githubusercontent.com/Informatievlaanderen/OSLO-Generated/wegenenverkeer/report/doc/vocabularium/wegenenverkeer/onderdeel/ontwerpdocument/1.0.0/html-nj.json
+https://raw.githubusercontent.com/Informatievlaanderen/OSLO-Generated/wegenenverkeer/report/doc/vocabularium/wegenenverkeer/proefenmeting/ontwerpdocument/1.0.0/html-nj.json
diff --git a/lib/Elastic.ts b/lib/Elastic.ts
@@ -1,5 +1,6 @@
 import {Client} from '@elastic/elasticsearch';
 const config = require('../config.json');
+const hash = require('object-hash');
 
 export class Elastic {
 
@@ -10,13 +11,13 @@ export class Elastic {
         this.checkElasticStatus()
     }
 
-
     async indexData(document, index) {
-        const body = document.flatMap(doc => [{index: {_index: index, _id: doc.id}}, doc]);
+        const body = document.flatMap(object => [{index: { _index: index, _id: this.createhash(object)}}, object]);
         const {body: bulkResponse} = await this.client.bulk({refresh: true, body});
 
         if (bulkResponse.errors) {
-            const erroredDocuments = []
+            console.log(`[Elasticsearch]: some errors occurred while adding the data:`)
+            const erroredDocuments = [];
             // The items array has the same order of the dataset we just indexed.
             // The presence of the `error` key indicates that the operation
             // that we did for the document has failed.
@@ -35,6 +36,8 @@ export class Elastic {
                 }
             })
             console.log(erroredDocuments)
+        } else {
+            console.log(`[Elasticsearch]: insert number of terms is: ${bulkResponse.items.length}`);
         }
     }
 
@@ -128,7 +131,7 @@ export class Elastic {
     private async checkIndexStatus(index: string) {
         return this.client.indices.exists({index: index})
     }
-    
+
     /* Mapping functions */
 
     private getVocabularyMapping() {
@@ -166,5 +169,9 @@ export class Elastic {
         }
     }
 
+    private createhash(object){
+        return hash(object);
+    }
+
 
 }
diff --git a/lib/Vocabulary.ts b/lib/Vocabulary.ts
@@ -3,15 +3,16 @@ import {VocabularyTerm} from "./Term";
 export class Vocabulary  {
 
     static createDocument(data){
-        //const name = data.metadata.uri;
+        const domain = data.metadata.uri;
         let terms = [];
-        // Get classes and properties from the vocabulary
+
+        // Get classes, properties and external terms from the vocabulary
         for(let object of data.classes){
             terms.push(new VocabularyTerm({
                 prefLabel: object.name['nl'],
                 id: object.uri,
                 definition: object.description['nl'],
-                context: data.metadata.uri
+                context: domain
             }));
         }
 
@@ -20,7 +21,16 @@ export class Vocabulary  {
                 prefLabel: object.name['nl'],
                 id: object.uri,
                 definition: object.description['nl'],
-                context: data.metadata.uri
+                context: domain
+            }));
+        }
+
+        for(let object of data.external_terms){
+            terms.push(new VocabularyTerm({
+                prefLabel: object.name['nl'],
+                id: object.uri,
+                definition: object.description['nl'],
+                context: domain
             }));
         }
 
diff --git a/package-lock.json b/package-lock.json
diff --git a/package.json b/package.json
@@ -23,6 +23,7 @@
     "commander": "^5.1.0",
     "fs": "0.0.1-security",
     "node-fetch": "^2.6.1",
+    "object-hash": "^2.1.1",
     "readline": "^1.3.0",
     "util": "^0.12.3"
   },

Original file line number	Diff line number	Diff line change
`@@ -1,5 +1,6 @@`
`1`	`1`	`import {Client} from '@elastic/elasticsearch';`
`2`	`2`	`const config = require('../config.json');`
	`3`	`+const hash = require('object-hash');`
`3`	`4`
`4`	`5`	`export class Elastic {`
`5`	`6`
`@@ -10,13 +11,13 @@ export class Elastic {`
`10`	`11`	`this.checkElasticStatus()`
`11`	`12`	`}`
`12`	`13`
`13`		`-`
`14`	`14`	`async indexData(document, index) {`
`15`		`- const body = document.flatMap(doc => [{index: {_index: index, _id: doc.id}}, doc]);`
	`15`	`+ const body = document.flatMap(object => [{index: { _index: index, _id: this.createhash(object)}}, object]);`
`16`	`16`	`const {body: bulkResponse} = await this.client.bulk({refresh: true, body});`
`17`	`17`
`18`	`18`	`if (bulkResponse.errors) {`
`19`		`- const erroredDocuments = []`
	`19`	+ console.log(`[Elasticsearch]: some errors occurred while adding the data:`)
	`20`	`+ const erroredDocuments = [];`
`20`	`21`	`// The items array has the same order of the dataset we just indexed.`
`21`	`22`	// The presence of the `error` key indicates that the operation
`22`	`23`	`// that we did for the document has failed.`
`@@ -35,6 +36,8 @@ export class Elastic {`
`35`	`36`	`}`
`36`	`37`	`})`
`37`	`38`	`console.log(erroredDocuments)`
	`39`	`+ } else {`
	`40`	+ console.log(`[Elasticsearch]: insert number of terms is: ${bulkResponse.items.length}`);
`38`	`41`	`}`
`39`	`42`	`}`
`40`	`43`
`@@ -128,7 +131,7 @@ export class Elastic {`
`128`	`131`	`private async checkIndexStatus(index: string) {`
`129`	`132`	`return this.client.indices.exists({index: index})`
`130`	`133`	`}`
`131`		`-`
	`134`	`+`
`132`	`135`	`/* Mapping functions */`
`133`	`136`
`134`	`137`	`private getVocabularyMapping() {`
`@@ -166,5 +169,9 @@ export class Elastic {`
`166`	`169`	`}`
`167`	`170`	`}`
`168`	`171`
	`172`	`+ private createhash(object){`
	`173`	`+ return hash(object);`
	`174`	`+ }`
	`175`	`+`
`169`	`176`
`170`	`177`	`}`