Skip to content

Commit 44b9d56

Browse files
ddvlanckddvlanck
committed
Configured upload of APs, auto create index and updates
1 parent 7ea6611 commit 44b9d56

File tree

6 files changed

+69
-40
lines changed

6 files changed

+69
-40
lines changed

‎bin/oslo-knowledge-graph.ts‎

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
import {Processor} from "../lib/Processor";
2+
import {Elastic} from "../lib/Elastic";
23
const program = require('commander');
34
const readline = require('readline');
45
const fs = require('fs');
@@ -39,14 +40,12 @@ if(type !== 'ap' && type !== 'voc'){
3940
process.exit(1);
4041
}
4142

42-
4343
processInput(bulkFile ? bulkFile : url, type, bulkFile ? true: false, update);
4444

4545
async function processInput(filename: string, type: string, bulk: boolean, update: boolean){
4646
let files = new Array<string>();
4747
const indexType = type === 'voc' ? config.VOCABULARY_INDEX : config.APPLICATION_PROFILE_INDEX;
4848

49-
5049
if(bulk){
5150
const stream = fs.createReadStream(filename);
5251
const rl = readline.createInterface({
@@ -60,5 +59,5 @@ async function processInput(filename: string, type: string, bulk: boolean, updat
6059
} else {
6160
files.push(filename);
6261
}
63-
new Processor(files, indexType)
62+
new Processor(files, indexType, update)
6463
}

‎files/applicationProfiles.txt‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
https://raw.githubusercontent.com/Informatievlaanderen/OSLO-Generated/test/report/doc/applicatieprofiel/DCAT-AP-VL/standaard/2019-06-13/html-nj.json
1+
https://raw.githubusercontent.com/Informatievlaanderen/OSLO-Generated/test/report/doc/applicatieprofiel/DCAT-AP-VL/erkendestandaard/2019-06-13/html-nj.json
22
https://raw.githubusercontent.com/Informatievlaanderen/OSLO-Generated/test/report/doc/applicatieprofiel/adressenregister/ontwerpdocument/niet-bepaald/html-nj.json
33
https://raw.githubusercontent.com/Informatievlaanderen/OSLO-Generated/test/report/doc/applicatieprofiel/bedrijventerrein/kandidaatstandaard/2020-04-23/html-nj.json
44
https://raw.githubusercontent.com/Informatievlaanderen/OSLO-Generated/test/report/doc/applicatieprofiel/begraafplaatsen/kandidaatstandaard/2020-06-18/html-nj.json
@@ -7,7 +7,7 @@ https://raw.githubusercontent.com/Informatievlaanderen/OSLO-Generated/test/repor
77
https://raw.githubusercontent.com/Informatievlaanderen/OSLO-Generated/test/report/doc/applicatieprofiel/besluit-mobiliteit/ontwerpdocument/2018-09-30/html-nj.json
88
https://raw.githubusercontent.com/Informatievlaanderen/OSLO-Generated/test/report/doc/applicatieprofiel/besluit-publicatie/ontwerpdocument/2020-01-06/html-nj.json
99
https://raw.githubusercontent.com/Informatievlaanderen/OSLO-Generated/test/report/doc/applicatieprofiel/besluit-subsidie/ontwerpstandaard/2019-12-02/html-nj.json
10-
https://raw.githubusercontent.com/Informatievlaanderen/OSLO-Generated/test/report/doc/applicatieprofiel/besluitvorming/erkendestandaard/2020-06-18/html-nj.json
10+
https://raw.githubusercontent.com/Informatievlaanderen/OSLO-Generated/test/report/doc/applicatieprofiel/besluitvorming/kandidaatstandaard/2020-03-31/html-nj.json
1111
https://raw.githubusercontent.com/Informatievlaanderen/OSLO-Generated/test/report/doc/applicatieprofiel/contactvoorkeuren/ontwerpdocument/2020-01-06/html-nj.json
1212
https://raw.githubusercontent.com/Informatievlaanderen/OSLO-Generated/test/report/doc/applicatieprofiel/cultureel-erfgoed-event/kandidaatstandaard/2020-07-17/html-nj.json
1313
https://raw.githubusercontent.com/Informatievlaanderen/OSLO-Generated/test/report/doc/applicatieprofiel/cultureel-erfgoed-object/kandidaatstandaard/2020-07-17/html-nj.json
@@ -20,7 +20,7 @@ https://raw.githubusercontent.com/Informatievlaanderen/OSLO-Generated/test/repor
2020
https://raw.githubusercontent.com/Informatievlaanderen/OSLO-Generated/test/report/doc/applicatieprofiel/generieke-terugmeldfaciliteit/ontwerpdocument/2020-01-06/html-nj.json
2121
https://raw.githubusercontent.com/Informatievlaanderen/OSLO-Generated/test/report/doc/applicatieprofiel/infrastructuurelementen/kandidaatstandaard/2020-06-18/html-nj.json
2222
https://raw.githubusercontent.com/Informatievlaanderen/OSLO-Generated/test/report/doc/applicatieprofiel/issue-tracking-voor-burgers-en-organisaties/ontwerpdocument/2020-01-06/html-nj.json
23-
https://raw.githubusercontent.com/Informatievlaanderen/OSLO-Generated/test/report/doc/applicatieprofiel/logies-basis/standaard/2019-03-14/html-nj.json
23+
https://raw.githubusercontent.com/Informatievlaanderen/OSLO-Generated/test/report/doc/applicatieprofiel/logies-basis/ontwerpstandaard/2019-03-14/html-nj.json
2424
https://raw.githubusercontent.com/Informatievlaanderen/OSLO-Generated/test/report/doc/applicatieprofiel/mandatendatabank/ontwerpdocument/2018-07-20/html-nj.json
2525
https://raw.githubusercontent.com/Informatievlaanderen/OSLO-Generated/test/report/doc/applicatieprofiel/mobiliteit-trips-en-aanbod/erkendestandaard/2020-04-23/html-nj.json
2626
https://raw.githubusercontent.com/Informatievlaanderen/OSLO-Generated/test/report/doc/applicatieprofiel/notificatie-basis/ontwerpdocument/2020-01-06/html-nj.json

‎lib/ApplicationProfile.ts‎

Lines changed: 6 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,10 @@
11
import {APTerm, ITerm, PropertyTerm} from "./Term";
22

3-
export class ApplicationProfile {
3+
export class ApplicationProfile {
44

5-
static createDocument(data){
5+
static createDocument(data) {
66
const name = data.metadata.uri;
7-
const baseuri = data.metadata.navigation.self;
7+
const baseUri = data.metadata.navigation.self;
88
let classes = []
99

1010
for (let object of data.classes) {
@@ -17,7 +17,7 @@ export class ApplicationProfile {
1717
id: property.uri,
1818
definition: property.description['nl'],
1919
context: name,
20-
fragmentIdentifier: baseuri + '#' + object.name['nl'].split(' ').join('%20') + '%3A' + property.name['nl'].split(' ').join('%20')
20+
fragmentIdentifier: `${baseUri}#${object.name['nl'].split(' ').join('%20')}%3A${property.name['nl'].split(' ').join('%20')}`
2121
}))
2222
}
2323

@@ -27,13 +27,11 @@ export class ApplicationProfile {
2727
prefLabel: object.name['nl'],
2828
id: object.uri,
2929
definition: object.description['nl'],
30-
properties: properties,
30+
classProperties: properties,
3131
context: name,
32-
fragmentIdentifier: baseuri + '#' + object.name['nl'].split(' ').join('%20')
32+
fragmentIdentifier:`${baseUri}#${object.name['nl'].split(' ').join('%20')}`
3333
}))
34-
3534
}
36-
3735
return classes;
3836
}
3937
}

‎lib/Elastic.ts‎

Lines changed: 49 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -7,14 +7,11 @@ export class Elastic {
77

88
constructor() {
99
this.client = this.getClient();
10-
//this.checkElasticStatus();
10+
this.checkElasticStatus()
1111
}
1212

13-
initElasticsearch() {
14-
//TODO
15-
}
1613

17-
async pushData(document, index) {
14+
async indexData(document, index) {
1815
const body = document.flatMap(doc => [{index: {_index: index, _id: doc.id}}, doc]);
1916
const {body: bulkResponse} = await this.client.bulk({refresh: true, body});
2017

@@ -41,6 +38,18 @@ export class Elastic {
4138
}
4239
}
4340

41+
async updateData(document, index){
42+
for(let object of document){
43+
await this.client.update({
44+
index: index,
45+
id: object.id,
46+
body: {
47+
doc: object
48+
}
49+
});
50+
}
51+
}
52+
4453
private getClient(): Client {
4554
return new Client({
4655
node: config.ELASTIC_ENDPOINT,
@@ -54,7 +63,7 @@ export class Elastic {
5463
});
5564
}
5665

57-
async checkElasticStatus() {
66+
private async checkElasticStatus() {
5867
await this.checkElasticsearchHealth();
5968
await this.checkIndicesHealth();
6069
}
@@ -70,14 +79,21 @@ export class Elastic {
7079

7180
private checkIndicesHealth() {
7281
this.checkIndexStatus(config.VOCABULARY_INDEX).then(response => {
73-
response.body === false ?
74-
console.log(`[Elastic]: index ${config.VOCABULARY_INDEX} does not exist in this elasticsearch instance. Please create it before running this program`) :
82+
if(response.body === false){
83+
console.log(`[Elastic]: index ${config.VOCABULARY_INDEX} does not exist in this elasticsearch instance. The index will be created now.`)
84+
this.createIndex(config.VOCABULARY_INDEX);
85+
} else {
7586
console.log(`[Elastic]: index ${config.VOCABULARY_INDEX} exists and is healthy.`)
87+
}
88+
7689
});
7790
this.checkIndexStatus(config.APPLICATION_PROFILE_INDEX).then(response => {
78-
response.body === false ?
79-
console.log(`[Elastic]: index ${config.APPLICATION_PROFILE_INDEX} does not exist in this elasticsearch instance. Please create it before running this program`) :
91+
if(response.body === false){
92+
console.log(`[Elastic]: index ${config.APPLICATION_PROFILE_INDEX} does not exist in this elasticsearch instance. The index will be created now.`)
93+
this.createIndex(config.APPLICATION_PROFILE_INDEX);
94+
} else {
8095
console.log(`[Elastic]: index ${config.APPLICATION_PROFILE_INDEX} exists and is healthy.`)
96+
}
8197
});
8298
}
8399

@@ -98,7 +114,7 @@ export class Elastic {
98114
});
99115
}
100116

101-
setMappingForIndex(index: string, mapping) {
117+
private setMappingForIndex(index: string, mapping) {
102118
this.client.indices.putMapping({
103119
index: index,
104120
body: mapping
@@ -112,7 +128,7 @@ export class Elastic {
112128
private async checkIndexStatus(index: string) {
113129
return this.client.indices.exists({index: index})
114130
}
115-
131+
116132
/* Mapping functions */
117133

118134
private getVocabularyMapping() {
@@ -126,8 +142,28 @@ export class Elastic {
126142
}
127143
}
128144

145+
129146
private getApplicationProfileMapping() {
130-
return {}
147+
return {
148+
properties: {
149+
prefLabel: {"type": "text"},
150+
id: {"type": "text"},
151+
context: {"type": "text"},
152+
definition: {"type": "text"},
153+
fragmentIdentifier: {"type": "text"},
154+
classProperties : {
155+
type: "nested",
156+
properties : {
157+
prefLabel: {"type": "text"},
158+
id: {"type": "text"},
159+
context: {"type": "text"},
160+
definition: {"type": "text"},
161+
fragmentIdentifier: {"type": "text"},
162+
}
163+
}
164+
165+
}
166+
}
131167
}
132168

133169

‎lib/Processor.ts‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -10,11 +10,13 @@ export class Processor {
1010
private readonly files: Array<string>;
1111
private elastic: Elastic;
1212
private filesWithError: Array<string>;
13+
private isUpdate: boolean;
1314

14-
constructor(files: Array<string>, type: string) {
15+
constructor(files: Array<string>, type: string, isUpdate) {
1516
this.files = files;
1617
this.elastic = new Elastic();
1718
this.filesWithError = new Array<string>();
19+
this.isUpdate = isUpdate
1820
this.processFiles(type);
1921
}
2022

@@ -33,12 +35,12 @@ export class Processor {
3335

3436
createVocabularyDocument(data){
3537
const document = Vocabulary.createDocument(data);
36-
this.elastic.pushData(document, config.VOCABULARY_INDEX)
38+
this.isUpdate === true ? this.elastic.updateData(document, config.VOCABULARY_INDEX) : this.elastic.indexData(document, config.VOCABULARY_INDEX);
3739
}
3840

3941
createApplicationProfileDocument(data){
4042
const document = ApplicationProfile.createDocument(data);
41-
this.elastic.pushData(document, config.APPLICATION_PROFILE_INDEX)
43+
this.isUpdate === true ? this.elastic.updateData(document, config.APPLICATION_PROFILE_INDEX) : this.elastic.indexData(document, config.APPLICATION_PROFILE_INDEX);
4244
}
4345

4446
printSummary(){

‎lib/Term.ts‎

Lines changed: 4 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@ export interface ITermSettings {
33
id: string,
44
definition: string,
55
context: string,
6-
properties?: Array<ITerm>
6+
classProperties?: Array<ITerm>
77
fragmentIdentifier?: string
88
}
99

@@ -24,14 +24,12 @@ export class VocabularyTerm implements ITerm {
2424
id: string;
2525
definition: string;
2626
prefLabel: string;
27-
termType: TermType;
2827
context: string;
2928

3029
constructor(settings: ITermSettings) {
3130
this.prefLabel = settings.prefLabel;
3231
this.id = settings.id;
3332
this.definition = settings.definition;
34-
this.termType = TermType.VOCABULARY_TERM;
3533
this.context = settings.context;
3634
}
3735
}
@@ -40,24 +38,22 @@ export class APTerm implements ITerm {
4038
id: string;
4139
definition: string;
4240
prefLabel: string;
43-
termType: TermType;
4441
context: string;
4542

4643
fragmentIdentifier: string;
4744

48-
properties: Array<ITerm>;
45+
classProperties: Array<ITerm>;
4946

5047
constructor(settings: ITermSettings) {
51-
if(!settings.properties){
48+
if(!settings.classProperties){
5249
console.error('[APTerm]: A properties array must be provided!') ;
5350
process.exit(1);
5451
}
55-
this.properties = settings.properties;
52+
this.classProperties = settings.classProperties;
5653
this.context = settings.context;
5754
this.prefLabel = settings.prefLabel;
5855
this.id = settings.id;
5956
this.definition = settings.definition;
60-
this.termType = TermType.APPLICATION_PROFILE_TERM;
6157
this.fragmentIdentifier = settings.fragmentIdentifier;
6258
}
6359
}
@@ -66,7 +62,6 @@ export class PropertyTerm implements ITerm {
6662
id: string;
6763
definition: string;
6864
prefLabel: string;
69-
termType: TermType;
7065
context: string;
7166

7267
fragmentIdentifier: string;
@@ -76,7 +71,6 @@ export class PropertyTerm implements ITerm {
7671
this.id = settings.id;
7772
this.definition = settings.definition;
7873
this.context = settings.context;
79-
this.termType = TermType.PROPERTY;
8074
this.fragmentIdentifier = settings.fragmentIdentifier
8175
}
8276
}

0 commit comments

Comments
 (0)