Feature rdf store experimental

bin/cli.ts

@@ -156,6 +156,12 @@ async function main() {
     });
   }
 
+  if (!quiet) {
+    client.on("error", (error) => {
+      console.error("Error", error);
+    });
+  }
+
   const reader = client.stream({ highWaterMark: 10 }).getReader();
   let el = await reader.read();
   let count = 0;

lib/client.ts

@@ -148,7 +148,6 @@
       config.shape ? shapeConfigStore : store,
       dereferencer,
       {
-        cbdDefaultGraph: config.onlyDefaultGraph,
         fetch: config.fetch,
       },
     ),
@@ -423,7 +422,17 @@
     fetch: fetch_f,
   });
   const url = resp.url;
-  const data = RdfStore.createDefault();
+  const data = new RdfStore<number>({
+    indexCombinations: [
+      [ 'graph', 'subject', 'predicate', 'object' ],
+      [ 'graph', 'predicate', 'object', 'subject' ],
+      [ 'graph', 'object', 'subject', 'predicate' ],
+    ],
+    indexConstructor: subOptions => new RdfStoreIndexNestedMapQuoted(subOptions),
+    dictionary: new TermDictionaryNumberRecordFullTerms(),
+    dataFactory: df,
+    termsCardinalitySets:  [ 'graph'] //enable quick overview of graphs using getGraph();
+  });
   await new Promise((resolve, reject) => {
     data.import(resp.data).on("end", resolve).on("error", reject);
   });

lib/memberManager.ts

@@ -44,6 +44,8 @@
   private extractor: CBDShapeExtractor;
   private shapeId?: Term;
 
+  private members: Array<Term>;
+
   private timestampPath?: Term;
   private isVersionOfPath?: Term;
 
@@ -67,7 +69,7 @@
   ) {
     const logger = log.extend("extract");
     const members = getObjects(page.data, this.ldesId, TREE.terms.member, null);
-
+    this.members = members;
     logger("%d members", members.length);
 
     const promises: Promise<Member | undefined | void>[] = [];
@@ -127,7 +129,17 @@
     return await this.extractor.extract(data, member, this.shapeId);
   }
 
+  private async extractMember(
+    member: Term,
+    data: RdfStore,
+  ): Promise<Member | undefined> {
+    const quads: Quad[] = await this.extractMemberQuads(member, data);
+
+    // let the extractor do it's thing
+    return await this.extractor.extract(data, member);
+  }
+
   private async extractMember(
     member: Term,
     data: RdfStore,
   ): Promise<Member | undefined> {

lib/pageFetcher.ts

@@ -3,10 +3,10 @@ import { Notifier } from "./utils";
 import { extractRelations, Relation } from "./page";
 import debug from "debug";
 import { SimpleRelation } from "./relation";
-import { RdfStore } from "rdf-stores";
+import { RdfStore, RdfStoreIndexNestedMapQuoted, TermDictionaryNumberRecordFullTerms } from "rdf-stores";
 import { DataFactory } from "rdf-data-factory";
 const log = debug("fetcher");
-const { namedNode } = new DataFactory();
+const df = new DataFactory();
 
 /**
  * target: url to fetch
@@ -124,22 +124,27 @@ export class Fetcher {
 
       logger("Cache for  %s %o", node.target, cache);
 
-      const data = RdfStore.createDefault();
-      let quadCount = 0;
+      const data = new RdfStore<number>({
+        indexCombinations: [
+          [ 'graph', 'object', 'predicate', 'subject' ],
+          [ 'graph', 'subject', 'predicate', 'object' ],
+          [ 'graph', 'predicate', 'object', 'subject' ],
+          [ 'graph', 'object', 'subject', 'predicate' ],
+        ],
+        indexConstructor: subOptions => new RdfStoreIndexNestedMapQuoted(subOptions),
+        dictionary: new TermDictionaryNumberRecordFullTerms(),
+        dataFactory: df,
+        termsCardinalitySets: ['graph'] //enable quick overview of graphs
+      });
+      logger("Start loading " + node.target + "into store");
       await new Promise((resolve, reject) => {
-        resp.data
-          .on("data", (quad) => {
-            data.addQuad(quad);
-            quadCount++;
-          })
-          .on("end", resolve)
+        data.import(resp.data).on("end", resolve)
           .on("error", reject);
       });
-
-      logger("Got data %s (%d quads)", node.target, quadCount);
+      logger("Imported data %s (%d quads)", node.target, data.size);
       for (let rel of extractRelations(
         data,
-        namedNode(resp.url),
+        df.namedNode(resp.url),
         this.loose,
         this.after,
         this.before,

package.json

@@ -32,7 +32,7 @@
     "n3": "^1.17.3",
     "rdf-data-factory": "^1.1.2",
     "rdf-dereference": "^2.2.0",
-    "rdf-stores": "^1.0.0"
+    "rdf-stores": "github:pietercolpaert/rdf-stores.js#feature-termcardinalities"
   },
   "devDependencies": {
     "@ajuvercr/js-runner": "^0.1.20",