feat: json, json-all generators

.c8rc.json

@@ -5,6 +5,7 @@
     "**/fixtures",
     "src/generators/legacy-html/assets",
     "src/generators/web/ui",
-    "**/*.d.ts"
+    "**/*.d.ts",
+    "tools/"
   ]
 }

.github/workflows/generate.yml

@@ -36,6 +36,8 @@ jobs:
             input: './node/doc/api/*.md'
           - target: web
             input: './node/doc/api/*.md'
+          - target: json
+            input: './node/doc/api/*.md'
           - target: llms-txt
             input: './node/doc/api/*.md'
       fail-fast: false

package.json

@@ -34,6 +34,8 @@
     "eslint-plugin-import-x": "^4.16.1",
     "eslint-plugin-jsdoc": "^61.1.12",
     "husky": "^9.1.7",
+    "json-schema-to-typescript": "^15.0.4",
+    "jsonschema": "^1.5.0",
     "lint-staged": "^16.2.6",
     "prettier": "3.6.2"
   },
@@ -58,6 +60,7 @@
     "globals": "^16.4.0",
     "hast-util-to-string": "^3.0.1",
     "hastscript": "^9.0.1",
+    "jsonc-parser": "^3.3.1",
     "lightningcss": "^1.30.1",
     "mdast-util-slice-markdown": "^2.0.1",
     "preact": "^10.27.2",

src/generators/index.mjs

@@ -3,6 +3,8 @@
 import addonVerify from './addon-verify/index.mjs';
 import apiLinks from './api-links/index.mjs';
 import astJs from './ast-js/index.mjs';
+import json from './json/index.mjs';
+import jsonAll from './json-all/index.mjs';
 import jsonSimple from './json-simple/index.mjs';
 import jsxAst from './jsx-ast/index.mjs';
 import legacyHtml from './legacy-html/index.mjs';
@@ -27,6 +29,8 @@ export const publicGenerators = {
   'orama-db': oramaDb,
   'llms-txt': llmsTxt,
   web,
+  json,
+  'json-all': jsonAll,
 };
 
 // These ones are special since they don't produce standard output,

src/generators/json-all/__tests__/index.test.mjs

@@ -0,0 +1,120 @@
+import assert from 'node:assert';
+import { join } from 'node:path';
+import { test } from 'node:test';
+
+import { Validator } from 'jsonschema';
+import { SemVer } from 'semver';
+
+import { BASE_URL } from '../../../constants.mjs';
+import createGenerator from '../../../generators.mjs';
+import createMarkdownLoader from '../../../loaders/markdown.mjs';
+import createMarkdownParser from '../../../parsers/markdown.mjs';
+import { SCHEMA_FILENAME } from '../constants.mjs';
+import jsonAll from '../index.mjs';
+import { generateJsonSchema } from '../util/generateJsonSchema.mjs';
+
+const FIXTURES_DIR = join(
+  import.meta.dirname,
+  '..',
+  'json',
+  '__tests__',
+  'fixtures'
+);
+
+const loader = createMarkdownLoader();
+const parser = createMarkdownParser();
+
+test('generator output complies with json schema', async () => {
+  const validator = new Validator();
+
+  const version = 'v1.2.3';
+
+  /**
+   * @type {object}
+   */
+  const schema = generateJsonSchema(version);
+
+  const input = [
+    join(FIXTURES_DIR, 'text-doc.md'),
+    join(FIXTURES_DIR, 'module.md'),
+  ];
+  const files = await loader.loadFiles(input);
+  const docs = await parser.parseApiDocs(files);
+
+  const { runGenerators } = createGenerator(docs);
+
+  const result = await runGenerators({
+    generators: ['json-all'],
+    input,
+    output: undefined,
+    version: new SemVer('v1.2.3'),
+    releases: [],
+    gitRef: 'a'.repeat(40),
+    threads: 1,
+    typeMap: {},
+  });
+
+  assert.ok(validator.validate(result, schema).valid);
+});
+
+test('combines json correctly', async () => {
+  const version = 'v1.2.3';
+
+  /**
+   * @type {Array<import('../../json/generated.d.ts').NodeJsAPIDocumentationSchema>}
+   */
+  const jsonOutput = [
+    {
+      $schema: `https://nodejs.org/docs/${version}/api/node-doc-schema.json`,
+      source: 'doc/api/some-module.md',
+      type: 'module',
+      '@name': 'Some Module',
+      '@module': 'node:module',
+      classes: [],
+    },
+    {
+      $schema: `https://nodejs.org/docs/${version}/api/node-doc-schema.json`,
+      source: 'doc/api/some-module.md',
+      type: 'text',
+      '@name': 'Some Module',
+      description: 'asdf',
+      text: [
+        {
+          type: 'text',
+          '@name': 'asdf',
+          description: 'bla bla bla',
+        },
+      ],
+    },
+  ];
+
+  const result = await jsonAll.generate(jsonOutput, {
+    version: new SemVer(version),
+  });
+
+  assert.deepStrictEqual(result, {
+    $schema: `${BASE_URL}docs/${version}/api/${SCHEMA_FILENAME}`,
+    modules: [
+      {
+        type: 'module',
+        '@name': 'Some Module',
+        '@module': 'node:module',
+        classes: [],
+      },
+    ],
+    text: [
+      {
+        type: 'text',
+        '@name': 'Some Module',
+        description: 'asdf',
+        text: [
+          {
+            type: 'text',
+            '@name': 'asdf',
+            description: 'bla bla bla',
+          },
+        ],
+      },
+    ],
+  });
+});

src/generators/json-all/constants.mjs

@@ -0,0 +1 @@
+export const SCHEMA_FILENAME = 'node-doc-all-schema.json';

src/generators/json-all/index.mjs

@@ -0,0 +1,78 @@
+'use strict';
+
+import { writeFile } from 'node:fs/promises';
+import { join } from 'node:path';
+
+import { SCHEMA_FILENAME } from './constants.mjs';
+import { BASE_URL } from '../../constants.mjs';
+import { generateJsonSchema } from './util/generateJsonSchema.mjs';
+
+/**
+ * This generator is responsible for collecting the JSON output generated by
+ * the `json` generator into a single JSON file.
+ *
+ * @typedef {Array<ApiDocMetadataEntry>} Input
+ *
+ * @type {GeneratorMetadata<Input, object>}
+ */
+export default {
+  name: 'json-all',
+
+  // This should be kept in sync with the JSON schema version for this
+  // generator AND the `json` generator
+  version: '2.0.0',
+
+  description:
+    'This generator is responsible for collecting the JSON output generated by the `json` generator into a single JSON file.',
+
+  dependsOn: 'json',
+
+  /**
+   * Generates a JSON file.
+   *
+   * @param {Input} input
+   * @param {Partial<GeneratorOptions>} param1
+   * @returns {Promise<object>}
+   */
+  async generate(input, { version, output }) {
+    const versionString = `v${version.toString()}`;
+
+    const generatedValue = {
+      $schema: `${BASE_URL}docs/${versionString}/api/${SCHEMA_FILENAME}`,
+      modules: [],
+      text: [],
+    };
+
+    const propertiesToIgnore = ['$schema', 'source'];
+
+    input.forEach(section => {
+      const copiedSection = {};
+
+      Object.keys(section).forEach(key => {
+        if (!propertiesToIgnore.includes(key)) {
+          copiedSection[key] = section[key];
+        }
+      });
+
+      switch (section.type) {
+        case 'module':
+          generatedValue.modules.push(copiedSection);
+          break;
+        case 'text':
+          generatedValue.text.push(copiedSection);
+          break;
+        default:
+          throw new TypeError(`unsupported root section type ${section.type}`);
+      }
+    });
+
+    if (output) {
+      const schema = generateJsonSchema(versionString);
+
+      // Write the parsed JSON schema to the output directory
+      await writeFile(join(output, SCHEMA_FILENAME), JSON.stringify(schema));
+    }
+
+    return generatedValue;
+  },
+};

src/generators/json-all/util/generateJsonSchema.mjs

@@ -0,0 +1,30 @@
+'use strict';
+
+import { BASE_URL } from '../../../constants.mjs';
+import { SCHEMA_FILENAME } from '../constants.mjs';
+import jsonAll from '../index.mjs';
+
+/**
+ * @param {string} version
+ */
+export function generateJsonSchema(version) {
+  const jsonSchemaUrl = `${BASE_URL}/docs/${version}/api/${SCHEMA_FILENAME}`;
+
+  return {
+    $schema: 'http://json-schema.org/draft-07/schema#',
+    $id: `nodejs-api-doc-all@v${jsonAll.version}`,
+    title: 'Node.js API Documentation Schema (All)',
+    readOnly: true,
+
+    properties: {
+      modules: {
+        type: 'array',
+        items: { $ref: `${jsonSchemaUrl}/#/definitions/Module` },
+      },
+      text: {
+        type: 'array',
+        items: { $ref: `${jsonSchemaUrl}/#/definitions/Text` },
+      },
+    },
+  };
+}

src/generators/json/__tests__/fixtures/module.md

@@ -0,0 +1,73 @@
+# Module
+
+<!-- introduced_in=v20.0.0 -->
+<!-- source_link=lib/module.js -->
+
+> Stability: 2 - Stable
+
+Lorem ipsum dolor sit amet, consectetur adipiscing elit. Nulla aliquet est sit amet nisi iaculis ornare. Donec eu orci condimentum, semper quam ut, luctus nisl. Aliquam vel erat efficitur, egestas justo a, accumsan mauris. Quisque scelerisque enim rhoncus ornare iaculis. Sed commodo, sem ac accumsan posuere, sapien magna eleifend massa, sit amet rhoncus sem diam quis nulla. Donec feugiat elit a tellus hendrerit congue quis bibendum urna. Pellentesque mollis tincidunt tortor, in lobortis ex fringilla a.
+
+Nulla sagittis pulvinar lacus et venenatis. Morbi eget varius sem, a rutrum velit. Vestibulum placerat vehicula felis. Proin et porttitor augue. Sed feugiat urna sit amet euismod posuere. Phasellus quis pretium velit. Integer porta ut velit ac consectetur. Pellentesque venenatis leo et volutpat dapibus. Nam consectetur quam venenatis feugiat tempus. Curabitur semper fringilla felis.
+
+Donec sed est at nulla lacinia suscipit id id neque. Cras quis risus a risus mattis consectetur ut ut ex. Cras non metus a orci facilisis laoreet. Sed eu nisi sit amet mauris iaculis placerat. Nullam interdum, lectus sed convallis tristique, nibh nibh ullamcorper leo, non vehicula metus elit a nisl. Donec eros felis, ornare eget tempor nec, egestas mattis elit. Integer semper purus at risus pretium, in tristique ipsum tristique.
+
+```mjs
+import { something } from 'node:something';
+```
+
+## text section
+
+<!-- YAML
+changes:
+  - version: v3.0.0
+    pr-url: https://github.com/nodejs/node/pull/1234
+    description: asdf
+-->
+
+Lorem ipsum dolor sit amet, consectetur adipiscing elit. Nulla aliquet est sit amet nisi iaculis ornare. Donec eu orci condimentum, semper quam ut, luctus nisl. Aliquam vel erat efficitur, egestas justo a, accumsan mauris. Quisque scelerisque enim rhoncus ornare iaculis. Sed commodo, sem ac accumsan posuere, sapien magna eleifend massa, sit amet rhoncus sem diam quis nulla. Donec feugiat elit a tellus hendrerit congue quis bibendum urna. Pellentesque mollis tincidunt tortor, in lobortis ex fringilla a.
+
+Nulla sagittis pulvinar lacus et venenatis. Morbi eget varius sem, a rutrum velit. Vestibulum placerat vehicula felis. Proin et porttitor augue. Sed feugiat urna sit amet euismod posuere. Phasellus quis pretium velit. Integer porta ut velit ac consectetur. Pellentesque venenatis leo et volutpat dapibus. Nam consectetur quam venenatis feugiat tempus. Curabitur semper fringilla felis.
+
+Donec sed est at nulla lacinia suscipit id id neque. Cras quis risus a risus mattis consectetur ut ut ex. Cras non metus a orci facilisis laoreet. Sed eu nisi sit amet mauris iaculis placerat. Nullam interdum, lectus sed convallis tristique, nibh nibh ullamcorper leo, non vehicula metus elit a nisl. Donec eros felis, ornare eget tempor nec, egestas mattis elit. Integer semper purus at risus pretium, in tristique ipsum tristique.
+
+## Class: `Something`
+
+<!-- YAML
+added:
+  - v15.7.0
+  - v14.18.0
+changes:
+  - version:
+    - v18.0.0
+    - v16.17.0
+    pr-url: https://github.com/nodejs/node/pull/1234
+    description: No longer experimental.
+-->
+
+Lorem ipsum dolor sit amet, consectetur adipiscing elit. Nulla aliquet est sit amet nisi iaculis ornare. Donec eu orci condimentum, semper quam ut, luctus nisl. Aliquam vel erat efficitur, egestas justo a, accumsan mauris. Quisque scelerisque enim rhoncus ornare iaculis. Sed commodo, sem ac accumsan posuere, sapien magna eleifend massa, sit amet rhoncus sem diam quis nulla. Donec feugiat elit a tellus hendrerit congue quis bibendum urna. Pellentesque mollis tincidunt tortor, in lobortis ex fringilla a.
+
+### `new something.Something([sources[, options]])`
+
+- `sources` {string\[]|object} something something bla bla bla
+- `options` {object} something something bla bla bla
+  - `optionA` {boolean} asdf
+  - `bla` qwerty
+
+### `something.doThing()`
+
+- Returns: {Promise}
+
+Lorem ipsum dolor sit amet, consectetur adipiscing elit. Nulla aliquet est sit amet nisi iaculis ornare. Donec eu orci condimentum, semper quam ut, luctus nisl. Aliquam vel erat efficitur, egestas justo a, accumsan mauris. Quisque scelerisque enim rhoncus ornare iaculis. Sed commodo, sem ac accumsan posuere, sapien magna eleifend massa, sit amet rhoncus sem diam quis nulla. Donec feugiat elit a tellus hendrerit congue quis bibendum urna. Pellentesque mollis tincidunt tortor, in lobortis ex fringilla a.
+
+### Static method: `something.doStaticThing()`
+
+- Returns: {Promise}
+
+Lorem ipsum dolor sit amet, consectetur adipiscing elit. Nulla aliquet est sit amet nisi iaculis ornare. Donec eu orci condimentum, semper quam ut, luctus nisl. Aliquam vel erat efficitur, egestas justo a, accumsan mauris. Quisque scelerisque enim rhoncus ornare iaculis. Sed commodo, sem ac accumsan posuere, sapien magna eleifend massa, sit amet rhoncus sem diam quis nulla. Donec feugiat elit a tellus hendrerit congue quis bibendum urna. Pellentesque mollis tincidunt tortor, in lobortis ex fringilla a.
+
+### Event: `somethingHappened`
+
+- `thing` {string} thing that happened
+- `how` {string|object} bla
+
+Lorem ipsum dolor sit amet, consectetur adipiscing elit. Nulla aliquet est sit amet nisi iaculis ornare. Donec eu orci condimentum, semper quam ut, luctus nisl. Aliquam vel erat efficitur, egestas justo a, accumsan mauris. Quisque scelerisque enim rhoncus ornare iaculis. Sed commodo, sem ac accumsan posuere, sapien magna eleifend massa, sit amet rhoncus sem diam quis nulla. Donec feugiat elit a tellus hendrerit congue quis bibendum urna. Pellentesque mollis tincidunt tortor, in lobortis ex fringilla a.