Refactor and re-organize code structure

bin/cli.ts

@@ -1,11 +1,15 @@
 #!/usr/bin/env node
-import * as process from "process";
-import { Ordered, replicateLDES } from "../lib/client";
-import { intoConfig } from "../lib/config";
+import * as process from "process";;
 import { Command, Option } from "commander";
 import { Writer } from "n3";
-import { enhanced_fetch, FetchConfig, processConditionFile } from "../lib/utils";
-import { getLoggerFor } from "../lib/utils/logUtil";
+import { replicateLDES } from "../lib/client";
+import { intoConfig } from "../lib/config"
+import { enhanced_fetch } from "../lib/fetcher";
+import { processConditionFile } from "../lib/condition";
+import { getLoggerFor } from "../lib/utils";
+
+import type { Ordered } from "../lib/strategy";
+import type { FetchConfig } from "../lib/fetcher";
 
 const program = new Command();
 let paramURL: string = "";

lib/client.ts

@@ -1,39 +1,38 @@
-import { Config, intoConfig } from "./config";
-import { Fragment, Member } from "./page";
 import { RdfDereferencer, rdfDereferencer } from "rdf-dereference";
-import { FileStateFactory, NoStateFactory, StateFactory } from "./state";
+import { LDES, RDF, TREE } from "@treecg/types";
 import { CBDShapeExtractor } from "extract-cbd-shape";
 import { RdfStore } from "rdf-stores";
 import { DataFactory } from "rdf-data-factory";
-import { Term } from "@rdfjs/types";
+import { intoConfig } from "./config";
+import { handleConditions } from "./condition";
+import { FileStateFactory, NoStateFactory } from "./state";
+import { OrderedStrategy, UnorderedStrategy } from "./strategy";
+import {
+    ModulatorFactory,
+    Fetcher,
+    longPromise,
+    resetPromise,
+    Manager,
+    maybeVersionMaterialize,
+} from "./fetcher";
 import {
     extractMainNodeShape,
     getObjects,
-    handleConditions,
-    maybeVersionMaterialize,
-    ModulatorFactory,
-    Notifier,
     streamToArray,
+    getLoggerFor,
+    handleExit
 } from "./utils";
-import { LDES, RDF, TREE } from "@treecg/types";
-import { FetchedPage, Fetcher, longPromise, resetPromise } from "./pageFetcher";
-import { Manager } from "./memberManager";
-import { OrderedStrategy, StrategyEvents, UnorderedStrategy } from "./strategy";
-import { getLoggerFor } from "./utils/logUtil";
-import { handleExit } from "./exitHandler";
-
-export { intoConfig } from "./config";
-export { enhanced_fetch, extractMainNodeShape, retry_fetch } from "./utils";
-export * from "./condition/index";
-export type { Member, Page, Relation, Fragment } from "./page";
-export type { Config, ShapeConfig } from "./config";
+
+import type { Term } from "@rdfjs/types";
+import type { Config } from "./config";
+import type { StateFactory } from "./state";
+import type { Ordered, StrategyEvents } from "./strategy";
+import type { LDESInfo, Notifier, FetchedPage, Fragment, Member } from "./fetcher";
 
 const df = new DataFactory();
 
 type Controller = ReadableStreamDefaultController<Member>;
 
-export type Ordered = "ascending" | "descending" | "none";
-
 export function replicateLDES(
     config: Partial<Config> & { url: string },
     ordered: Ordered = "none",
@@ -43,13 +42,6 @@ export function replicateLDES(
     return new Client(intoConfig(config), ordered, dereferencer, streamId);
 }
 
-export type LDESInfo = {
-    shape: Term;
-    extractor: CBDShapeExtractor;
-    timestampPath?: Term;
-    versionOfPath?: Term;
-};
-
 async function getInfo(
     ldesId: Term,
     viewId: Term,
@@ -251,7 +243,6 @@ export class Client {
     async init(
         emit: (member: Member) => void,
         close: () => void,
-        factory: ModulatorFactory,
     ): Promise<void> {
         // Fetch the url
         const root = await fetchPage(
@@ -410,7 +401,7 @@ export class Client {
                     this.memberManager,
                     this.fetcher,
                     notifier,
-                    factory,
+                    this.modulatorFactory,
                     this.ordered,
                     this.config.polling,
                     this.config.pollInterval,
@@ -419,7 +410,7 @@ export class Client {
                     this.memberManager,
                     this.fetcher,
                     notifier,
-                    factory,
+                    this.modulatorFactory,
                     this.config.polling,
                     this.config.pollInterval,
                 );
@@ -455,7 +446,6 @@ export class Client {
                         resetPromise(emitted);
                     },
                     () => controller.close(),
-                    this.modulatorFactory,
                 );
             },
 

lib/condition/condition.ts

@@ -1,13 +1,18 @@
-import { Quad, Term } from "@rdfjs/types";
 import { NamedNode, Parser } from "n3";
-import { BasicLensM, Cont, extractShapes } from "rdf-lens";
+import { DataFactory } from "rdf-data-factory";
+import { BasicLensM, extractShapes, pred } from "rdf-lens";
 import { RdfStore } from "rdf-stores";
-import { Member } from "../page";
 import { TREE, XSD } from "@treecg/types";
+import { getLoggerFor, parseInBetweenRelation } from "../utils";
 import { SHAPES } from "./shapes";
-import { cbdEquals, Path } from "./range";
-import { getLoggerFor } from "../utils/logUtil";
-import { parseInBetweenRelation } from "../utils/inBetween";
+import { cbdEquals } from "./range";
+
+import type { Quad, Term } from "@rdfjs/types";
+import type { Cont } from "rdf-lens";
+import type { Member } from "../fetcher";
+import type { Path } from "./range";
+
+const df = new DataFactory();
 
 type RdfThing = {
     entry: Term;
@@ -470,3 +475,104 @@ export class MaxCountCondition implements Condition {
         }
     }
 }
+
+export async function processConditionFile(
+    conditionFile?: string,
+): Promise<Condition> {
+    let condition: Condition = empty_condition();
+
+    /* eslint-disable  @typescript-eslint/no-require-imports */
+    const fs =
+        typeof require === "undefined"
+            ? await import("fs/promises")
+            : require("fs/promises");
+
+    if (conditionFile) {
+        try {
+            condition = parse_condition(
+                await fs.readFile(conditionFile, { encoding: "utf8" }),
+                conditionFile,
+            );
+        } catch (ex) {
+            console.error(`Failed to read condition file: ${conditionFile}`);
+            throw ex;
+        }
+    }
+
+    return condition;
+}
+
+/**
+ * Function that handles any given condition, together with the "before" and "after" options,
+ * and builds the corresponding unified Condition.
+ */
+export function handleConditions(
+    condition: Condition,
+    defaultTimezone: string,
+    before?: Date,
+    after?: Date,
+    timestampPath?: Term,
+): Condition {
+    // Check if before and after conditions are defined and build corresponding Condition object
+    let handledCondition: Condition = empty_condition();
+    const toDateLiteral = (date: Date) => {
+        return df.literal(date.toISOString(), XSD.terms.dateTime);
+    };
+
+    if (before) {
+        if (!timestampPath) {
+            throw "Cannot apply 'before' or 'after' filters since the target LDES does not define a ldes:timestampPath predicate";
+        }
+
+        const predLens = pred(timestampPath);
+        const beforeCond = new LeafCondition({
+            relationType: TREE.terms.LessThanRelation,
+            value: toDateLiteral(before),
+            compareType: "date",
+            path: predLens,
+            pathQuads: { entry: timestampPath, quads: [] },
+            defaultTimezone,
+        });
+        if (after) {
+            const afterCond = new LeafCondition({
+                relationType: TREE.terms.GreaterThanRelation,
+                value: toDateLiteral(after),
+                compareType: "date",
+                path: predLens,
+                pathQuads: { entry: timestampPath, quads: [] },
+                defaultTimezone,
+            });
+            // Got bi-condition with before & after filters
+            handledCondition = new AndCondition({
+                items: [beforeCond, afterCond],
+            });
+        } else {
+            // Got condition with before filter only
+            handledCondition = beforeCond;
+        }
+    } else if (after) {
+        if (!timestampPath) {
+            throw "Cannot apply 'before' or 'after' filters since the target LDES does not define a ldes:timestampPath predicate";
+        }
+
+        const predLens = pred(timestampPath);
+        // Got condition with after filter only
+        handledCondition = new LeafCondition({
+            relationType: TREE.terms.GreaterThanRelation,
+            value: toDateLiteral(after),
+            compareType: "date",
+            path: predLens,
+            pathQuads: { entry: timestampPath, quads: [] },
+            defaultTimezone,
+        });
+    }
+
+    // See if condition file was defined too
+    if (!(condition instanceof EmptyCondition)) {
+        return new AndCondition({
+            items: [condition, handledCondition],
+        });
+    } else {
+        return handledCondition;
+    }
+}

lib/condition/index.ts

@@ -1 +1,3 @@
 export * from "./condition";
+export * from "./range";
+export * from "./shapes";

lib/condition/range.ts

@@ -1,8 +1,10 @@
-import { Quad, Term } from "@rdfjs/types";
 import { RdfStore } from "rdf-stores";
-import { getObjects } from "../utils";
 import { RDF, TREE } from "@treecg/types";
-import { Condition, Range } from "./condition";
+import { getObjects } from "../utils";
+import { Range } from "./condition";
+
+import type { Quad, Term } from "@rdfjs/types";
+import type { Condition } from "./condition";
 
 export type Path = {
     store: RdfStore;

lib/config.ts

@@ -1,5 +1,7 @@
-import { NamedNode, Quad } from "@rdfjs/types";
-import { Condition, empty_condition as emptyCondition } from "./condition";
+import { empty_condition as emptyCondition } from "./condition";
+
+import type { NamedNode, Quad } from "@rdfjs/types";
+import type { Condition } from "./condition";
 
 export interface ShapeConfig {
     quads: Quad[];