Skip to content

Commit 50aed7e

Browse files
Copilotdanmarshall
Copilot
and
danmarshall
committed
Add fetch plugin implementation and example
Co-authored-by: danmarshall <11507384+danmarshall@users.noreply.github.com>
1 parent b148a47 commit 50aed7e

4 files changed

Lines changed: 312 additions & 0 deletions

File tree

‎packages/markdown/src/plugins/fetch.ts‎

Lines changed: 264 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,264 @@
1+
/**
2+
* Copyright (c) Microsoft Corporation.
3+
* Licensed under the MIT License.
4+
*/
5+
6+
import { read } from 'vega';
7+
import { Batch, IInstance, Plugin, RawFlaggableSpec } from '../factory.js';
8+
import { sanitizedHTML, sanitizeHtmlComment } from '../sanitize.js';
9+
import { pluginClassName } from './util.js';
10+
import { PluginNames } from './interfaces.js';
11+
import { SpecReview } from 'common';
12+
import { DynamicUrl } from './url.js';
13+
14+
interface FetchInstance {
15+
id: string;
16+
spec: FetchSpec;
17+
data: object[] | null;
18+
dynamicUrl: DynamicUrl | null;
19+
container: Element;
20+
errorHandler: (error: Error) => void;
21+
}
22+
23+
export interface FetchSpec {
24+
variableId: string;
25+
url: string;
26+
format?: 'json' | 'csv' | 'tsv' | 'dsv';
27+
delimiter?: string;
28+
}
29+
30+
function inspectFetchSpec(spec: FetchSpec): RawFlaggableSpec<FetchSpec> {
31+
const result: RawFlaggableSpec<FetchSpec> = {
32+
spec,
33+
hasFlags: false,
34+
reasons: []
35+
};
36+
37+
// Check for http:// (should use https://)
38+
if (spec.url.includes('http://') && !spec.url.includes('{{')) {
39+
result.hasFlags = true;
40+
result.reasons.push('URL uses http:// instead of https://');
41+
}
42+
43+
return result;
44+
}
45+
46+
const pluginName: PluginNames = 'fetch' as PluginNames;
47+
const className = pluginClassName(pluginName);
48+
49+
export const fetchPlugin: Plugin<FetchSpec> = {
50+
name: pluginName,
51+
fence: (token, index) => {
52+
const info = token.info.trim();
53+
const parts = info.split(/\s+/);
54+
55+
// Parse format: fetch url:https://example.com/data.json format:json variableId:myData
56+
let url = '';
57+
let format: 'json' | 'csv' | 'tsv' | 'dsv' = 'json';
58+
let delimiter = ',';
59+
let variableId = `fetchData${index}`;
60+
61+
for (let i = 1; i < parts.length; i++) {
62+
const part = parts[i];
63+
if (part.startsWith('url:')) {
64+
url = part.slice(4);
65+
} else if (part.startsWith('format:')) {
66+
format = part.slice(7) as 'json' | 'csv' | 'tsv' | 'dsv';
67+
} else if (part.startsWith('delimiter:')) {
68+
delimiter = part.slice(10);
69+
if (delimiter === '\\t') delimiter = '\t';
70+
if (delimiter === '\\n') delimiter = '\n';
71+
if (delimiter === '\\r') delimiter = '\r';
72+
} else if (part.startsWith('variableId:')) {
73+
variableId = part.slice(11);
74+
} else if (i === 1 && !part.includes(':')) {
75+
// First parameter without prefix is the URL
76+
url = part;
77+
}
78+
}
79+
80+
return sanitizedHTML('div', {
81+
id: `${pluginName}-${index}`,
82+
class: className,
83+
style: 'display:none',
84+
'data-variable-id': variableId,
85+
'data-url': url,
86+
'data-format': format,
87+
'data-delimiter': delimiter
88+
}, '', false);
89+
},
90+
hydrateSpecs: (renderer, errorHandler) => {
91+
const flagged: SpecReview<FetchSpec>[] = [];
92+
const containers = renderer.element.querySelectorAll(`.${className}`);
93+
94+
for (const [index, container] of Array.from(containers).entries()) {
95+
try {
96+
const variableId = container.getAttribute('data-variable-id');
97+
const url = container.getAttribute('data-url');
98+
const format = (container.getAttribute('data-format') || 'json') as 'json' | 'csv' | 'tsv' | 'dsv';
99+
const delimiter = container.getAttribute('data-delimiter') || ',';
100+
101+
if (!variableId) {
102+
errorHandler(new Error('No variable ID found'), pluginName, index, 'parse', container);
103+
continue;
104+
}
105+
106+
if (!url) {
107+
errorHandler(new Error('No URL found'), pluginName, index, 'parse', container);
108+
continue;
109+
}
110+
111+
const spec: FetchSpec = { variableId, url, format, delimiter };
112+
const flaggableSpec = inspectFetchSpec(spec);
113+
114+
const f: SpecReview<FetchSpec> = {
115+
approvedSpec: null,
116+
pluginName,
117+
containerId: container.id
118+
};
119+
120+
if (flaggableSpec.hasFlags) {
121+
f.blockedSpec = flaggableSpec.spec;
122+
f.reason = flaggableSpec.reasons?.join(', ') || 'Unknown reason';
123+
} else {
124+
f.approvedSpec = flaggableSpec.spec;
125+
}
126+
127+
flagged.push(f);
128+
} catch (e) {
129+
errorHandler(e instanceof Error ? e : new Error(String(e)), pluginName, index, 'parse', container);
130+
}
131+
}
132+
133+
return flagged;
134+
},
135+
hydrateComponent: async (renderer, errorHandler, specs) => {
136+
const { signalBus } = renderer;
137+
const fetchInstances: FetchInstance[] = [];
138+
139+
for (let index = 0; index < specs.length; index++) {
140+
const specReview = specs[index];
141+
if (!specReview.approvedSpec) {
142+
continue;
143+
}
144+
145+
const container = renderer.element.querySelector(`#${specReview.containerId}`);
146+
if (!container) {
147+
errorHandler(new Error('Container not found'), pluginName, index, 'init', null);
148+
continue;
149+
}
150+
151+
const spec: FetchSpec = specReview.approvedSpec;
152+
153+
const fetchInstance: FetchInstance = {
154+
id: `${pluginName}-${index}`,
155+
spec,
156+
data: null,
157+
dynamicUrl: null,
158+
container,
159+
errorHandler: (error) => {
160+
errorHandler(error, pluginName, index, 'fetch', container, spec.url);
161+
}
162+
};
163+
164+
fetchInstances.push(fetchInstance);
165+
}
166+
167+
const instances = fetchInstances.map((fetchInstance): IInstance => {
168+
const { spec, id } = fetchInstance;
169+
170+
// Function to fetch data from URL
171+
const fetchData = async (url: string) => {
172+
if (!url || url.includes('{{')) {
173+
// URL still has unresolved variables
174+
return;
175+
}
176+
177+
try {
178+
const response = await fetch(url);
179+
if (!response.ok) {
180+
throw new Error(`HTTP error! status: ${response.status}`);
181+
}
182+
183+
let data: object[];
184+
185+
if (spec.format === 'json') {
186+
const jsonData = await response.json();
187+
data = Array.isArray(jsonData) ? jsonData : [jsonData];
188+
} else {
189+
const text = await response.text();
190+
// Use vega's read function to parse CSV/TSV/DSV
191+
const formatType = spec.format === 'dsv' ? 'dsv' : spec.format;
192+
const delimiter = spec.format === 'dsv' ? spec.delimiter :
193+
spec.format === 'tsv' ? '\t' : ',';
194+
data = read(text, { type: formatType, delimiter });
195+
}
196+
197+
fetchInstance.data = data;
198+
199+
// Broadcast the data
200+
const batch: Batch = {
201+
[spec.variableId]: {
202+
value: data,
203+
isData: true,
204+
},
205+
};
206+
signalBus.broadcast(id, batch);
207+
208+
// Add a comment to show data was loaded
209+
const comment = sanitizeHtmlComment(`Fetch data loaded: ${data.length} rows for variable '${spec.variableId}' from ${url}`);
210+
fetchInstance.container.insertAdjacentHTML('afterbegin', comment);
211+
212+
} catch (e) {
213+
fetchInstance.errorHandler(e instanceof Error ? e : new Error(String(e)));
214+
}
215+
};
216+
217+
// Check if URL has variables
218+
if (spec.url.includes('{{')) {
219+
// Dynamic URL - set up DynamicUrl handler
220+
fetchInstance.dynamicUrl = new DynamicUrl(spec.url, (resolvedUrl) => {
221+
fetchData(resolvedUrl);
222+
});
223+
} else {
224+
// Static URL - fetch immediately
225+
fetchData(spec.url);
226+
}
227+
228+
const signalNames = Object.keys(fetchInstance.dynamicUrl?.signals || {});
229+
230+
return {
231+
id,
232+
initialSignals: signalNames.map(name => ({
233+
name,
234+
value: null,
235+
priority: -1,
236+
isData: false,
237+
})),
238+
receiveBatch: async (batch, from) => {
239+
fetchInstance.dynamicUrl?.receiveBatch(batch);
240+
},
241+
beginListening() {
242+
// If we have data already, broadcast it
243+
if (fetchInstance.data) {
244+
const batch: Batch = {
245+
[spec.variableId]: {
246+
value: fetchInstance.data,
247+
isData: true,
248+
},
249+
};
250+
signalBus.broadcast(id, batch);
251+
}
252+
},
253+
getCurrentSignalValue: () => {
254+
return fetchInstance.data;
255+
},
256+
destroy: () => {
257+
// No cleanup needed
258+
},
259+
};
260+
});
261+
262+
return instances;
263+
},
264+
};

‎packages/markdown/src/plugins/index.ts‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@ import { commentPlugin } from './comment.js';
1010
import { cssPlugin } from './css.js';
1111
import { csvPlugin } from './csv.js';
1212
import { dsvPlugin } from './dsv.js';
13+
import { fetchPlugin } from './fetch.js';
1314
import { googleFontsPlugin } from './google-fonts.js';
1415
import { dropdownPlugin } from './dropdown.js';
1516
import { imagePlugin } from './image.js';
@@ -31,6 +32,7 @@ export function registerNativePlugins() {
3132
registerMarkdownPlugin(cssPlugin);
3233
registerMarkdownPlugin(csvPlugin);
3334
registerMarkdownPlugin(dsvPlugin);
35+
registerMarkdownPlugin(fetchPlugin);
3436
registerMarkdownPlugin(googleFontsPlugin);
3537
registerMarkdownPlugin(dropdownPlugin);
3638
registerMarkdownPlugin(imagePlugin);

‎packages/markdown/src/plugins/interfaces.ts‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ export { CheckboxSpec } from './checkbox.js';
66
export { CsvSpec } from './csv.js';
77
export { DropdownSpec } from './dropdown.js';
88
export { DsvSpec } from './dsv.js';
9+
export { FetchSpec } from './fetch.js';
910
export { ImageSpec } from './image.js';
1011
export { MermaidSpec } from './mermaid.js';
1112
export { NumberSpec } from './number.js';
@@ -23,6 +24,7 @@ export type PluginNames =
2324
'checkbox' |
2425
'dropdown' |
2526
'dsv' |
27+
'fetch' |
2628
'image' |
2729
'google-fonts' |
2830
'mermaid' |

‎packages/web-deploy/json/features/2.3b.fetch-plugin.idoc.json‎

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,44 @@
1+
{
2+
"$schema": "../../../../docs/schema/idoc_v1.json",
3+
"title": "Feature: Fetch Plugin",
4+
"variables": [
5+
{
6+
"variableId": "json_url",
7+
"type": "string",
8+
"initialValue": "https://vega.github.io/editor/data/barley.json"
9+
}
10+
],
11+
"groups": [
12+
{
13+
"groupId": "main",
14+
"elements": [
15+
"## Fetch Plugin with Dynamic URL Variables",
16+
"The fetch plugin allows you to load data from a URL. You can use variables to dynamically change data sources, just like the example in 2.3.dynamic-url-variables.idoc.md, but without requiring a Vega spec.",
17+
"",
18+
"Use a single variable for the entire URL (no URL encoding is applied) or construct URLs from multiple segments (e.g., `{{host}}/{{path}}`). When using multiple segments, each is encoded using `encodeURIComponent`.",
19+
"",
20+
"Select a data source:",
21+
{
22+
"type": "dropdown",
23+
"variableId": "json_url",
24+
"options": [
25+
"https://vega.github.io/editor/data/barley.json",
26+
"https://vega.github.io/editor/data/cars.json"
27+
]
28+
},
29+
"",
30+
"```fetch url:{{json_url}} format:json variableId:fetchedData```",
31+
"",
32+
{
33+
"type": "tabulator",
34+
"dataSourceName": "fetchedData",
35+
"tabulatorOptions": {
36+
"autoColumns": true,
37+
"layout": "fitColumns",
38+
"maxHeight": "300px"
39+
}
40+
}
41+
]
42+
}
43+
]
44+
}

0 commit comments

Comments
 (0)