Add value plugin for inline JSON and YAML data in markdown fence blocks, schema definitions for inline data elements, and compiler support

docs/dist/v1/chartifact.compiler.umd.js

@@ -1057,6 +1057,10 @@ var __publicField = (obj, key, value) => __defNormalProp(obj, typeof key !== "sy
           inlineDataMd = tickWrap(`dsv delimiter:${delimiter} variableId:${ds_raw}`, content);
           break;
         }
+        case "json": {
+          inlineDataMd = tickWrap(`json ${ds_raw}`, content);
+          break;
+        }
         default: {
           console.warn(`Unsupported inline data format: ${dataSource.format}, type is ${typeof dataSource.content}`);
           break;

docs/dist/v1/chartifact.editor.umd.js

@@ -1041,6 +1041,10 @@ var __publicField = (obj, key, value) => __defNormalProp(obj, typeof key !== "sy
           inlineDataMd = tickWrap(`dsv delimiter:${delimiter} variableId:${ds_raw}`, content);
           break;
         }
+        case "json": {
+          inlineDataMd = tickWrap(`json ${ds_raw}`, content);
+          break;
+        }
         default: {
           console.warn(`Unsupported inline data format: ${dataSource.format}, type is ${typeof dataSource.content}`);
           break;

docs/dist/v1/chartifact.host.umd.js

@@ -1041,6 +1041,10 @@ var __publicField = (obj, key, value) => __defNormalProp(obj, typeof key !== "sy
           inlineDataMd = tickWrap(`dsv delimiter:${delimiter} variableId:${ds_raw}`, content);
           break;
         }
+        case "json": {
+          inlineDataMd = tickWrap(`json ${ds_raw}`, content);
+          break;
+        }
         default: {
           console.warn(`Unsupported inline data format: ${dataSource.format}, type is ${typeof dataSource.content}`);
           break;

packages/compiler/src/loader.ts

@@ -18,15 +18,42 @@ export function addStaticDataLoaderToSpec(vegaScope: VegaScope, dataSource: Data
 
     if (dataSource.type === 'inline' && dataSource.format === 'json') {
 
-        const newData: ValuesData = {
-            name: dataSourceName,
-            values: dataSource.content as object[],
-            transform: dataSource.dataFrameTransformations || [],
-        };
-        spec.signals.push(dataAsSignal(dataSourceName));
-
-        //real data goes to the beginning of the data array
-        spec.data.unshift(newData);
+        // Check if content is already an object array
+        if (Array.isArray(dataSource.content) && typeof dataSource.content[0] === 'object') {
+            const newData: ValuesData = {
+                name: dataSourceName,
+                values: dataSource.content as object[],
+                transform: dataSource.dataFrameTransformations || [],
+            };
+            spec.signals.push(dataAsSignal(dataSourceName));
+
+            //real data goes to the beginning of the data array
+            spec.data.unshift(newData);
+        } else if (typeof dataSource.content === 'string') {
+            // Content is JSON string - need to output as json data fence block
+            const content = dataSource.content;
+            let ds_raw = dataSourceName;
+
+            if (dataSource.dataFrameTransformations) {
+                ds_raw += '_raw';
+
+                const newData: SourceData = {
+                    name: dataSourceName,
+                    source: ds_raw,
+                    transform: dataSource.dataFrameTransformations || [],
+                };
+                spec.signals.push(dataAsSignal(dataSourceName));
+
+                spec.data.unshift(newData);
+
+                //add a placeholder data since the transform depends on it
+                spec.data.unshift({
+                    name: ds_raw
+                });
+            }
+
+            inlineDataMd = tickWrap(`json data ${ds_raw}`, content);
+        }
 
     } else if (typeof dataSource.content === 'string' || (Array.isArray(dataSource.content) && typeof dataSource.content[0] === 'string')) {
 

packages/markdown/src/factory.ts

@@ -156,6 +156,14 @@ export function create() {
             else {
                 const infoWords = info.split(/\s+/);
                 if (infoWords.length > 0) {
+                    // Special case for "json data" - treat as single plugin name
+                    if (infoWords[0] === 'json' && infoWords[1] === 'data') {
+                        const jsonDataPlugin = findPlugin('json-data');
+                        if (jsonDataPlugin) {
+                            return jsonDataPlugin;
+                        }
+                    }
+
                     const pluginPrefix = findPluginByPrefix(infoWords[0]);
                     if (pluginPrefix) {
                         return pluginPrefix;

packages/markdown/src/plugins/index.ts

@@ -13,6 +13,7 @@ import { dsvPlugin } from './dsv.js';
 import { googleFontsPlugin } from './google-fonts.js';
 import { dropdownPlugin } from './dropdown.js';
 import { imagePlugin } from './image.js';
+import { jsonDataPlugin } from './json-data.js';
 import { mermaidPlugin } from './mermaid.js';
 import { numberPlugin } from './number.js';
 import { placeholdersPlugin } from './placeholders.js';
@@ -34,6 +35,7 @@ export function registerNativePlugins() {
     registerMarkdownPlugin(googleFontsPlugin);
     registerMarkdownPlugin(dropdownPlugin);
     registerMarkdownPlugin(imagePlugin);
+    registerMarkdownPlugin(jsonDataPlugin);
     registerMarkdownPlugin(mermaidPlugin);
     registerMarkdownPlugin(numberPlugin);
     registerMarkdownPlugin(placeholdersPlugin);

packages/markdown/src/plugins/interfaces.ts

@@ -7,6 +7,7 @@ export { CsvSpec } from './csv.js';
 export { DropdownSpec } from './dropdown.js';
 export { DsvSpec } from './dsv.js';
 export { ImageSpec } from './image.js';
+export { JsonSpec } from './json-data.js';
 export { MermaidSpec } from './mermaid.js';
 export { NumberSpec } from './number.js';
 export { PresetsSpec } from './presets.js';
@@ -25,6 +26,7 @@ export type PluginNames =
     'dsv' |
     'image' |
     'google-fonts' |
+    'json-data' |
     'mermaid' |
     'number' |
     'placeholders' |

packages/markdown/src/plugins/json-data.ts

@@ -0,0 +1,216 @@
+/**
+* Copyright (c) Microsoft Corporation.
+* Licensed under the MIT License.
+*/
+
+import { Batch, IInstance, Plugin, RawFlaggableSpec } from '../factory.js';
+import { sanitizedScriptTag, sanitizeHtmlComment } from '../sanitize.js';
+import { pluginClassName } from './util.js';
+import { PluginNames } from './interfaces.js';
+import { SpecReview } from 'common';
+import { parseVariableId } from './dsv.js';
+
+interface JsonInstance {
+    id: string;
+    spec: JsonSpec;
+    data: object[];
+}
+
+export interface JsonSpec {
+    variableId: string;
+    wasDefaultId?: boolean;
+}
+
+function inspectJsonSpec(spec: JsonSpec): RawFlaggableSpec<JsonSpec> {
+    const result: RawFlaggableSpec<JsonSpec> = {
+        spec,
+        hasFlags: false,
+        reasons: []
+    };
+
+    // Flag if we had to use defaults
+    if (spec.wasDefaultId) {
+        result.hasFlags = true;
+        result.reasons.push('No variable ID specified - using default');
+    }
+
+    return result;
+}
+
+const pluginName: PluginNames = 'json-data';
+const className = pluginClassName(pluginName);
+
+export const jsonDataPlugin: Plugin<JsonSpec> = {
+    name: pluginName,
+    fence: (token, index) => {
+        const content = token.content.trim();
+        const info = token.info.trim();
+
+        // Parse the fence info - expect "json data variableId" format
+        const parts = info.split(/\s+/);
+
+        // Require "json data" prefix
+        if (parts.length < 2 || parts[0] !== 'json' || parts[1] !== 'data') {
+            // This fence is not for json-data plugin
+            return '';
+        }
+
+        // Check for variable ID
+        let variableId: string;
+        let wasDefaultId = false;
+
+        if (parts.length >= 3) {
+            // Format: json data variableId
+            variableId = parts[2];
+        } else {
+            // Default variable ID if not provided
+            variableId = `jsonData${index}`;
+            wasDefaultId = true;
+        }
+
+        // Use script tag with application/json type instead of pre tag
+        const scriptElement = sanitizedScriptTag(content, {
+            id: `${pluginName}-${index}`,
+            class: className,
+            'data-variable-id': variableId,
+            'data-was-default-id': wasDefaultId.toString()
+        });
+
+        return scriptElement.outerHTML;
+    },
+    hydrateSpecs: (renderer, errorHandler) => {
+        const flagged: SpecReview<JsonSpec>[] = [];
+        const containers = renderer.element.querySelectorAll(`.${className}`);
+
+        for (const [index, container] of Array.from(containers).entries()) {
+            try {
+                const variableId = container.getAttribute('data-variable-id');
+                const wasDefaultId = container.getAttribute('data-was-default-id') === 'true';
+
+                if (!variableId) {
+                    errorHandler(new Error('No variable ID found'), pluginName, index, 'parse', container);
+                    continue;
+                }
+
+                const spec: JsonSpec = { variableId, wasDefaultId };
+                const flaggableSpec = inspectJsonSpec(spec);
+
+                const f: SpecReview<JsonSpec> = { 
+                    approvedSpec: null, 
+                    pluginName, 
+                    containerId: container.id 
+                };
+
+                if (flaggableSpec.hasFlags) {
+                    f.blockedSpec = flaggableSpec.spec;
+                    f.reason = flaggableSpec.reasons?.join(', ') || 'Unknown reason';
+                } else {
+                    f.approvedSpec = flaggableSpec.spec;
+                }
+
+                flagged.push(f);
+            } catch (e) {
+                errorHandler(e instanceof Error ? e : new Error(String(e)), pluginName, index, 'parse', container);
+            }
+        }
+
+        return flagged;
+    },
+    hydrateComponent: async (renderer, errorHandler, specs) => {
+        const { signalBus } = renderer;
+        const jsonInstances: JsonInstance[] = [];
+
+        for (let index = 0; index < specs.length; index++) {
+            const specReview = specs[index];
+            if (!specReview.approvedSpec) {
+                continue;
+            }
+
+            const container = renderer.element.querySelector(`#${specReview.containerId}`);
+            if (!container) {
+                errorHandler(new Error('Container not found'), pluginName, index, 'init', null);
+                continue;
+            }
+
+            try {
+                const content = container.textContent?.trim();
+                if (!content) {
+                    errorHandler(new Error('No JSON content found'), pluginName, index, 'parse', container);
+                    continue;
+                }
+
+                const spec: JsonSpec = specReview.approvedSpec;
+
+                // Parse JSON content
+                let data: object[];
+                try {
+                    const parsed = JSON.parse(content);
+                    // Ensure data is an array
+                    if (Array.isArray(parsed)) {
+                        data = parsed;
+                    } else {
+                        // If it's a single object, wrap it in an array
+                        data = [parsed];
+                    }
+                } catch (jsonError) {
+                    errorHandler(
+                        new Error(`Invalid JSON: ${jsonError instanceof Error ? jsonError.message : String(jsonError)}`), 
+                        pluginName, 
+                        index, 
+                        'parse', 
+                        container
+                    );
+                    continue;
+                }
+
+                const jsonInstance: JsonInstance = { 
+                    id: `${pluginName}-${index}`, 
+                    spec, 
+                    data 
+                };
+                jsonInstances.push(jsonInstance);
+
+                // Add a safe comment before the container to show that data was loaded
+                const comment = sanitizeHtmlComment(`JSON data loaded: ${data.length} rows for variable '${spec.variableId}'`);
+                container.insertAdjacentHTML('beforebegin', comment);
+
+            } catch (e) {
+                errorHandler(e instanceof Error ? e : new Error(String(e)), pluginName, index, 'parse', container);
+            }
+        }
+
+        const instances = jsonInstances.map((jsonInstance): IInstance => {
+            const { spec, data } = jsonInstance;
+
+            const initialSignals = [{
+                name: spec.variableId,
+                value: data,
+                priority: 1,
+                isData: true,
+            }];
+
+            return {
+                ...jsonInstance,
+                initialSignals,
+                beginListening() {
+                    // JSON data is static, but we broadcast it when listening begins
+                    const batch: Batch = {
+                        [spec.variableId]: {
+                            value: data,
+                            isData: true,
+                        },
+                    };
+                    signalBus.broadcast(jsonInstance.id, batch);
+                },
+                getCurrentSignalValue: () => {
+                    return data;
+                },
+                destroy: () => {
+                    // No cleanup needed for JSON data
+                },
+            };
+        });
+
+        return instances;
+    },
+};

packages/markdown/src/sanitize.ts

@@ -26,11 +26,7 @@ export function sanitizedHTML(tagName: string, attributes: { [key: string]: stri
 
     if (precedeWithScriptTag) {
         // Create a script tag that precedes the main element
-        const scriptElement = domDocument.createElement('script');
-        scriptElement.setAttribute('type', 'application/json');
-        // Only escape the dangerous sequence that could break out of script tag
-        const safeContent = content.replace(/<\/script>/gi, '<\\/script>');
-        scriptElement.innerHTML = safeContent;
+        const scriptElement = sanitizedScriptTag(content);
 
         // Return script tag followed by empty element
         return scriptElement.outerHTML + element.outerHTML;
@@ -43,6 +39,30 @@ export function sanitizedHTML(tagName: string, attributes: { [key: string]: stri
     return element.outerHTML;
 }
 
+export function sanitizedScriptTag(content: string, attributes?: { [key: string]: string }): HTMLScriptElement {
+    if (!domDocument) {
+        throw new Error('No DOM Document available. Please set domDocument using setDomDocument.');
+    }
+
+    const scriptElement = domDocument.createElement('script');
+
+    // Set default type to application/json
+    scriptElement.setAttribute('type', 'application/json');
+
+    // Set additional attributes if provided
+    if (attributes) {
+        Object.keys(attributes).forEach(key => {
+            scriptElement.setAttribute(key, attributes[key]);
+        });
+    }
+
+    // Only escape the dangerous sequence that could break out of script tag
+    const safeContent = content.replace(/<\/script>/gi, '<\\/script>');
+    scriptElement.innerHTML = safeContent;
+
+    return scriptElement;
+}
+
 export function sanitizeHtmlComment(content: string) {
 
     // First escape the content safely