diff --git a/.gitignore b/.gitignore index e04f2a28a..493c2bf9a 100644 --- a/.gitignore +++ b/.gitignore @@ -141,6 +141,8 @@ untitled* # GeoJSON schema packages/schema/src/schema/geojson.json +# Schema pre-processing +packages/schema/temp-schema # Automatically generated file for processing diff --git a/packages/schema/package.json b/packages/schema/package.json index 31c5d68c8..9a448a4c1 100644 --- a/packages/schema/package.json +++ b/packages/schema/package.json @@ -29,7 +29,7 @@ "build:schema": "jlpm run build:processing && node ./cacheGeoJSONSchema.js && jlpm build:schema:js && jlpm build:schema:py", "build:processing": "python scripts/process.py", "build:schema:js": "echo 'Generating TypeScript types from schema...' && json2ts -i src/schema -o src/_interface --no-unknownAny --unreachableDefinitions --cwd ./src/schema && cd src/schema && node ../../schema.js", - "build:schema:py": "echo 'Generating Python types from schema...' && datamodel-codegen --input ./src/schema --output ../../python/jupytergis_core/jupytergis_core/schema/interfaces --output-model-type pydantic_v2.BaseModel --input-file-type jsonschema", + "build:schema:py": "echo 'Generating Python types from schema...' && node scripts/preprocess-schemas-for-python-type-generation.js && datamodel-codegen --input ./temp-schema --output ../../python/jupytergis_core/jupytergis_core/schema/interfaces --output-model-type pydantic_v2.BaseModel --input-file-type jsonschema && rm -rf ./temp-schema", "build:prod": "jlpm run clean && jlpm build:schema && jlpm run build:lib", "build:lib": "tsc -b", "build:dev": "jlpm run build", diff --git a/packages/schema/scripts/preprocess-schemas-for-python-type-generation.js b/packages/schema/scripts/preprocess-schemas-for-python-type-generation.js new file mode 100755 index 000000000..939001c5f --- /dev/null +++ b/packages/schema/scripts/preprocess-schemas-for-python-type-generation.js @@ -0,0 +1,116 @@ +#!/usr/bin/env node + +/** + * Preprocesses JSON schemas for Python type generation by replacing + * $ref paths which are relative to the schema root directory with paths which + * are relative to the file containing the $ref. + * + * This is a difference between how jsonschema-to-typescript and + * datamodel-codegen process $refs; I believe there is no way to write a $ref + * containing a path which is compatible with both unless our schemas are all + * in a flat directory structure. + */ + +const fs = require('fs'); +const path = require('path'); + +const schemaRoot = path.join(__dirname, '..', 'src', 'schema'); +const tempDir = path.join(__dirname, '..', 'temp-schema'); + +/* + * Rewrite `refValue`, if it contains a path, to be relative to `schemaDir` + * instead of `schemaRoot`. + */ +function updateRefPath(refValue, schemaDir, schemaRoot) { + // Handle $ref with optional fragment (e.g., "path/to/file.json#/definitions/something") + const [refPath, fragment] = refValue.split('#'); + + // Check if the referenced file exists + const absoluteRefPath = path.resolve(schemaRoot, refPath); + if (!fs.existsSync(absoluteRefPath)) { + throw new Error(`Referenced file does not exist: ${refPath} (resolved to ${absoluteRefPath})`); + } + + // Convert schemaRoot-relative path to schemaDir-relative path + const relativeToCurrentDir = path.relative(schemaDir, absoluteRefPath); + + // Just in case we're on Windows, replace backslashes. + const newRef = relativeToCurrentDir.replace(/\\/g, '/'); + + return fragment ? `${newRef}#${fragment}` : newRef; +} + +/* + * Recursively process `schema` (JSON content) to rewrite `$ref`s containing paths. + * + * Any path will be modified to be relative to `schemaDir` instead of relative + * to `schemaRoot`. + */ +function processSchema(schema, schemaDir, schemaRoot) { + if (Array.isArray(schema)) { + // Recurse! + return schema.map(item => processSchema(item, schemaDir, schemaRoot)); + } + + if (Object.prototype.toString.call(schema) !== '[object Object]') { + return schema; + } + + // `schema` is an "object": + const result = {}; + for (const [key, value] of Object.entries(schema)) { + if (key === '$ref' && typeof value === 'string' && !value.startsWith('#')) { + result[key] = updateRefPath(value, schemaDir, schemaRoot); + } else { + // Recurse! + result[key] = processSchema(value, schemaDir, schemaRoot); + } + } + + return result; +} + +/* + * Recursively rewrite schema files in `src` to `dest`. + * + * For each schema, rewrite the paths in JSONSchema `$ref`s to be relative to + * that schema's parent directory instead of `schemaRoot`. + */ +function preProcessSchemaDirectory(src, dest, schemaRoot) { + if (!fs.existsSync(dest)) { + fs.mkdirSync(dest, { recursive: true }); + } + + const children = fs.readdirSync(src, { withFileTypes: true }); + + for (const child of children) { + const srcChild = path.join(src, child.name); + const destChild = path.join(dest, child.name); + + if (child.isDirectory()) { + // Recurse! + preProcessSchemaDirectory(srcChild, destChild, schemaRoot); + } else if (child.isFile() && child.name.endsWith('.json')) { + // Process schema JSON to modify $ref paths + const content = fs.readFileSync(srcChild, 'utf8'); + const schema = JSON.parse(content); + const processedSchema = processSchema(schema, src, schemaRoot); + + fs.writeFileSync(destChild, JSON.stringify(processedSchema, null, 2)); + } else { + // There should be no non-JSON files in the schema directory! + throw new Error(`Non-JSON file detected in schema directory: ${child.parentPath}/${child.name}`); + } + } +} + +function preProcessSchemas() { + fs.rmSync(tempDir, { recursive: true, force: true }); + preProcessSchemaDirectory(schemaRoot, tempDir, schemaRoot); +} + +console.log(`Pre-processing JSONSchemas for Python type generation (writing to ${tempDir})...`) + +preProcessSchemas(); + +console.log('Schemas pre-processed for Python type generation.'); diff --git a/packages/schema/src/interfaces.ts b/packages/schema/src/interfaces.ts index 9249c71cd..d408c3843 100644 --- a/packages/schema/src/interfaces.ts +++ b/packages/schema/src/interfaces.ts @@ -27,7 +27,7 @@ import { SourceType, } from './_interface/project/jgis'; import { IRasterSource } from './_interface/project/sources/rasterSource'; -export { IGeoJSONSource } from './_interface/geoJsonSource'; +export { IGeoJSONSource } from './_interface/project/sources/geoJsonSource'; export type JgisCoordinates = { x: number; y: number }; diff --git a/packages/schema/src/schema/geoJsonSource.json b/packages/schema/src/schema/project/sources/geoJsonSource.json similarity index 100% rename from packages/schema/src/schema/geoJsonSource.json rename to packages/schema/src/schema/project/sources/geoJsonSource.json diff --git a/packages/schema/src/types.ts b/packages/schema/src/types.ts index 6a59a4c3c..b48a35312 100644 --- a/packages/schema/src/types.ts +++ b/packages/schema/src/types.ts @@ -1,7 +1,7 @@ export * from './_interface/project/jgis'; // Sources -export * from './_interface/geoJsonSource'; +export * from './_interface/project/sources/geoJsonSource'; export * from './_interface/project/sources/geoTiffSource'; export * from './_interface/project/sources/imageSource'; export * from './_interface/project/sources/rasterDemSource'; diff --git a/python/jupytergis_core/jupytergis_core/schema/__init__.py b/python/jupytergis_core/jupytergis_core/schema/__init__.py index 31ff42e2f..f25a838d9 100644 --- a/python/jupytergis_core/jupytergis_core/schema/__init__.py +++ b/python/jupytergis_core/jupytergis_core/schema/__init__.py @@ -10,7 +10,7 @@ from .interfaces.project.sources.vectorTileSource import IVectorTileSource # noqa from .interfaces.project.sources.rasterSource import IRasterSource # noqa -from .interfaces.geoJsonSource import IGeoJSONSource # noqa +from .interfaces.project.sources.geoJsonSource import IGeoJSONSource # noqa from .interfaces.project.sources.videoSource import IVideoSource # noqa from .interfaces.project.sources.imageSource import IImageSource # noqa from .interfaces.project.sources.geoTiffSource import IGeoTiffSource # noqa