Skip to content

Commit 5219a70

Browse files
authored
Backfill baseline locators into discovered docs; add Datadog specs (#68)
1 parent 2ff7fa5 commit 5219a70

3 files changed

Lines changed: 190 additions & 10 deletions

File tree

‎sources/openapi-manual.json‎

Lines changed: 37 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -10,9 +10,44 @@
1010
"description": "The full Cloudflare API — manage DNS, Workers, R2, KV, Zero Trust, CDN/cache, WAF, and the rest of the Cloudflare platform.",
1111
"openapiVer": "3.0.0",
1212
"origin": "https://raw.githubusercontent.com/cloudflare/api-schemas/main/openapi.json",
13-
"categories": ["cloud", "developer_tools"],
13+
"categories": [
14+
"cloud",
15+
"developer_tools"
16+
],
1417
"updated": "2026-06-29T00:00:00.000Z",
1518
"added": "2026-06-29T00:00:00.000Z"
19+
},
20+
{
21+
"provider": "datadoghq.com",
22+
"providerName": "Datadog",
23+
"openapiVer": "3.0.0",
24+
"categories": [
25+
"monitoring",
26+
"developer_tools"
27+
],
28+
"updated": "2026-08-27T00:00:00.000Z",
29+
"added": "2026-08-27T00:00:00.000Z",
30+
"service": "v2",
31+
"versionKey": "v2",
32+
"title": "Datadog API v2",
33+
"description": "Datadog's current API surface — metrics, logs, monitors, dashboards, incidents, security, and the rest of the observability platform.",
34+
"origin": "https://raw.githubusercontent.com/DataDog/datadog-api-client-typescript/master/.generator/schemas/v2/openapi.yaml"
35+
},
36+
{
37+
"provider": "datadoghq.com",
38+
"providerName": "Datadog",
39+
"openapiVer": "3.0.0",
40+
"categories": [
41+
"monitoring",
42+
"developer_tools"
43+
],
44+
"updated": "2026-08-27T00:00:00.000Z",
45+
"added": "2026-08-27T00:00:00.000Z",
46+
"service": "v1",
47+
"versionKey": "v1",
48+
"title": "Datadog API v1",
49+
"description": "Datadog's v1 API — the original endpoints for metrics, events, monitors, dashboards, and downtimes still in wide use.",
50+
"origin": "https://raw.githubusercontent.com/DataDog/datadog-api-client-typescript/master/.generator/schemas/v1/openapi.yaml"
1651
}
1752
]
18-
}
53+
}

‎worker/discovery-doc.test.ts‎

Lines changed: 78 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -42,4 +42,82 @@ describe("discoveryDoc", () => {
4242
test("returns null for genuinely unknown domains", async () => {
4343
await expect(discoveryDoc(envWith({}), origin, "missing.com")).resolves.toBeNull();
4444
});
45+
46+
const storedDoc = (domain: string, surfaces: unknown[]) => ({
47+
result: { version: 3, domain, summary: "", credentials: {}, surfaces },
48+
discoveredAt: "2026-07-05T19:00:00.000Z",
49+
model: "test",
50+
});
51+
52+
test("backfills a stored surface's missing spec from a lone baseline candidate", async () => {
53+
const doc = await discoveryDoc(
54+
envWith({
55+
kv: {
56+
"notionish.com": JSON.stringify(
57+
storedDoc("notionish.com", [
58+
{ type: "http", url: "https://api.notionish.com", slug: "rest-api" },
59+
{ type: "mcp", url: "https://mcp.notionish.com/mcp", slug: "mcp" },
60+
]),
61+
),
62+
},
63+
baseline: {
64+
version: 3,
65+
domain: "notionish.com",
66+
surfaces: [{ type: "http", spec: "https://spec.example/openapi.json", url: "https://api.notionish.com" }],
67+
},
68+
}),
69+
origin,
70+
"notionish.com",
71+
);
72+
73+
expect(doc?.surfaces).toHaveLength(2);
74+
expect(doc?.surfaces?.[0]).toMatchObject({ slug: "rest-api", spec: "https://spec.example/openapi.json" });
75+
});
76+
77+
test("appends baseline spec surfaces when the stored type has none and the match is ambiguous", async () => {
78+
const doc = await discoveryDoc(
79+
envWith({
80+
kv: {
81+
"datadogish.com": JSON.stringify(
82+
storedDoc("datadogish.com", [{ type: "http", url: "https://api.datadogish.com/api/", slug: "http-api" }]),
83+
),
84+
},
85+
baseline: {
86+
version: 3,
87+
domain: "datadogish.com",
88+
surfaces: [
89+
{ type: "http", spec: "https://spec.example/v2.yaml", slug: "api-v2" },
90+
{ type: "http", spec: "https://spec.example/v1.yaml", slug: "api-v1" },
91+
],
92+
},
93+
}),
94+
origin,
95+
"datadogish.com",
96+
);
97+
98+
expect(doc?.surfaces).toHaveLength(3);
99+
expect(doc?.surfaces?.map((s: { slug?: string }) => s.slug)).toEqual(["http-api", "api-v2", "api-v1"]);
100+
});
101+
102+
test("leaves a type alone when a stored surface already carries its locator", async () => {
103+
const stored = [{ type: "http", spec: "https://stored.example/spec.json", slug: "kept" }];
104+
const doc = await discoveryDoc(
105+
envWith({
106+
kv: { "twilioish.com": JSON.stringify(storedDoc("twilioish.com", stored)) },
107+
baseline: {
108+
version: 3,
109+
domain: "twilioish.com",
110+
surfaces: [
111+
{ type: "http", spec: "https://other.example/a.json", slug: "a" },
112+
{ type: "http", spec: "https://other.example/b.json", slug: "b" },
113+
],
114+
},
115+
}),
116+
origin,
117+
"twilioish.com",
118+
);
119+
120+
expect(doc?.surfaces).toHaveLength(1);
121+
expect(doc?.surfaces?.[0]).toMatchObject({ slug: "kept", spec: "https://stored.example/spec.json" });
122+
});
45123
});

‎worker/discovery-doc.ts‎

Lines changed: 75 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -28,23 +28,90 @@ const stripSdkSurfaces = (doc: DiscoveryDocWithSurfaces): DiscoveryDocWithSurfac
2828
surfaces: doc.surfaces.filter((s) => !isSdkNotCli(s)),
2929
});
3030

31-
/** The domain page's render source: durable discovery result first, then the
32-
* prerendered baseline discovery JSON. */
31+
type SurfaceLike = {
32+
type?: string;
33+
url?: string;
34+
spec?: string;
35+
slug?: string;
36+
};
37+
38+
/** The machine locator per surface type — what a consumer needs to actually
39+
* connect (an http surface's docs page or base URL is not it). */
40+
const locatorField = (type: string | undefined): "spec" | "url" | null =>
41+
type === "http" || type === "graphql" ? "spec" : type === "mcp" ? "url" : null;
42+
43+
// GraphQL's connectable locator is really the endpoint; treat a surface as
44+
// machine-reachable when either the endpoint or the SDL pointer is present.
45+
const hasLocator = (s: SurfaceLike): boolean =>
46+
s.type === "graphql" ? !!(s.url ?? s.spec) : !!s[locatorField(s.type) ?? "url"];
47+
48+
/**
49+
* Live-discovered docs sometimes drop the machine locators the static catalog
50+
* carries (the LLM found the API but not its spec URL). Backfill from the
51+
* baseline, conservatively, per surface type:
52+
* - any stored surface of the type already has a locator → leave the type alone;
53+
* - exactly one stored surface and one locator-bearing baseline candidate →
54+
* fill the stored surface's missing `spec`/`url`;
55+
* - otherwise append the baseline's locator-bearing surfaces (slug-deduped),
56+
* so registry-known specs stay reachable alongside the discovered surface.
57+
*/
58+
const backfillBaselineLocators = (
59+
stored: DiscoveryDocWithSurfaces,
60+
baseline: DiscoveryDocWithSurfaces,
61+
): DiscoveryDocWithSurfaces => {
62+
const surfaces: SurfaceLike[] = [...stored.surfaces];
63+
const takenSlugs = new Set(surfaces.map((s) => s.slug).filter(Boolean));
64+
for (const type of ["http", "graphql", "mcp"]) {
65+
const storedOfType = surfaces.filter((s) => s.type === type);
66+
if (storedOfType.some(hasLocator)) continue;
67+
const candidates = (baseline.surfaces as SurfaceLike[]).filter(
68+
(s) => s.type === type && hasLocator(s),
69+
);
70+
if (candidates.length === 0) continue;
71+
if (storedOfType.length === 1 && candidates.length === 1) {
72+
const target = storedOfType[0]!;
73+
const source = candidates[0]!;
74+
if (!target.spec && source.spec) target.spec = source.spec;
75+
if (!target.url && source.url) target.url = source.url;
76+
continue;
77+
}
78+
for (const candidate of candidates) {
79+
if (candidate.slug && takenSlugs.has(candidate.slug)) continue;
80+
if (candidate.slug) takenSlugs.add(candidate.slug);
81+
surfaces.push(candidate);
82+
}
83+
}
84+
return { ...stored, surfaces: surfaces as DiscoveryDocWithSurfaces["surfaces"] };
85+
};
86+
87+
const baselineDoc = async (
88+
env: Env,
89+
origin: string,
90+
canonical: string,
91+
): Promise<DiscoveryDocWithSurfaces | null> => {
92+
const res = await env.ASSETS.fetch(`${origin}/disc/${encodeURIComponent(canonical)}.json`);
93+
if (!res.ok) return null;
94+
const baseline = (await res.json()) as DiscoverData;
95+
return hasSurfaceArray(baseline) ? baseline : null;
96+
};
97+
98+
/** The domain page's render source: durable discovery result first (with
99+
* missing machine locators backfilled from the baseline), then the prerendered
100+
* baseline discovery JSON. */
33101
export async function discoveryDoc(env: Env, origin: string, domain: string): Promise<DiscoverData | null> {
34102
const canonical = canonicalDomain(domain);
35103
try {
36104
const raw = await discoveryKvGet(env, canonical);
37105
if (raw) {
38106
const stored = JSON.parse(raw) as { result?: DiscoverData; discoveredAt?: string };
39107
if (hasSurfaceArray(stored.result)) {
40-
return stripSdkSurfaces({ ...stored.result, discoveredAt: stored.result.discoveredAt ?? stored.discoveredAt });
108+
const doc = { ...stored.result, discoveredAt: stored.result.discoveredAt ?? stored.discoveredAt };
109+
const baseline = await baselineDoc(env, origin, canonical);
110+
return stripSdkSurfaces(baseline ? backfillBaselineLocators(doc, baseline) : doc);
41111
}
42112
}
43-
const res = await env.ASSETS.fetch(`${origin}/disc/${encodeURIComponent(canonical)}.json`);
44-
if (res.ok) {
45-
const baseline = (await res.json()) as DiscoverData;
46-
if (hasSurfaceArray(baseline)) return stripSdkSurfaces(baseline);
47-
}
113+
const baseline = await baselineDoc(env, origin, canonical);
114+
if (baseline) return stripSdkSurfaces(baseline);
48115
} catch {
49116
/* unavailable or malformed discovery data */
50117
}

0 commit comments

Comments
 (0)