diff --git a/README.md b/README.md index 35e9001..46c3af3 100644 --- a/README.md +++ b/README.md @@ -11,7 +11,8 @@ Content lives as Markdown in your repo and is served **at request time** — par - **Instant production content** — GitHub-sourced content pinned to a commit SHA, ISR-cached HTML, revalidated on push by a GitHub webhook (`/api/revalidate`). - **Versioned previews** — any branch (`/tree/:branch`) or commit (`/blob/:sha`) can be previewed through versioned URLs. - Docs UI built with [Nuxt UI](https://ui.nuxt.com): sidebar navigation, search (`⌘K`), TOC, prev/next links, version history panel. -- SEO & AEO out of the box: sitemap, robots, canonical URLs, OG images (Satori), JSON-LD, `llms.txt` / `llms-full.txt`, raw markdown mirrors (`/raw/**`), RSS, MCP server (`/mcp`), Agent Skills discovery (`/.well-known/skills/`). +- SEO out of the box: sitemap, robots, canonical URLs, OG images (Satori), JSON-LD, RSS. +- Markdown for agents through [nuxt-agent-discovery](https://github.com/benjamincanac/nuxt-agent-discovery): content negotiation on every page URL, raw markdown mirrors (`/raw/**`), `llms.txt` / `llms-full.txt`, `sitemap.md`, `/openapi.json`, `/.well-known/api-catalog`, an MCP server (`/mcp`) with its server card, Agent Skills discovery (`/.well-known/skills/`). ## Quick start @@ -44,13 +45,13 @@ From there, the docs cover everything: ## Agent Skills -Drop skills into a `skills/` directory at the app root and the layer serves them at `/.well-known/skills/`, following the [Agent Skills Discovery RFC](https://github.com/cloudflare/agent-skills-discovery-rfc) (v0.1). Users install them with: +Drop skills into a `skills/` directory at the app root and [nuxt-agent-discovery](https://github.com/benjamincanac/nuxt-agent-discovery) serves them at `/.well-known/skills/`, following the [Agent Skills Discovery RFC](https://github.com/cloudflare/agent-skills-discovery-rfc) (v0.1). Users install them with: ```bash npx skills add https://your-docs-domain.com ``` -Each skill is a directory with a `SKILL.md` whose frontmatter includes a `description`; `name` defaults to the directory name. Skills are scanned at build time from the filesystem (they ship with the app, not with GitHub-sourced content), so a skill change needs a redeploy. Override the directory with `comarkDocs.skills.dir`. +Each skill is a directory with a `SKILL.md` whose frontmatter includes a `description`; `name` defaults to the directory name. Skills are scanned at build time from the filesystem (they ship with the app, not with GitHub-sourced content), so a skill change needs a redeploy. Override the directory with `agentDiscovery.skills.dir`. ## Keyboard shortcuts diff --git a/app/components/docs/DocsPageAsideLinks.vue b/app/components/docs/DocsPageAsideLinks.vue index 387423f..7424c47 100644 --- a/app/components/docs/DocsPageAsideLinks.vue +++ b/app/components/docs/DocsPageAsideLinks.vue @@ -13,7 +13,8 @@ const { copy: copyLink } = useClipboard() const copying = ref(false) const site = useSiteConfig() -const mdPath = computed(() => `/raw${route.path}.md`) +const { rawPrefix } = useRuntimeConfig().public.agentDiscovery +const mdPath = computed(() => `${rawPrefix}${route.path}.md`) const mdUrl = computed(() => `${site.url}${mdPath.value}`) const { github, docs } = useAppConfig() diff --git a/app/utils/navigation.ts b/app/utils/navigation.ts index dc718d0..c2fa995 100644 --- a/app/utils/navigation.ts +++ b/app/utils/navigation.ts @@ -22,7 +22,6 @@ function walk(items: NavigationItem[], path: string): boolean { return false } -// Shared with the server-side `/raw/**` mirror (server/routes/raw/[...slug].md.get.ts). export { findFirstLeaf } from '../../utils/first-leaf' export interface BreadcrumbItem { diff --git a/modules/config.ts b/modules/config.ts index 35aaa70..30621eb 100644 --- a/modules/config.ts +++ b/modules/config.ts @@ -1,6 +1,7 @@ import { existsSync, readdirSync } from 'node:fs' -import { defineNuxtModule, useLogger } from '@nuxt/kit' +import { addServerPlugin, createResolver, defineNuxtModule, useLogger } from '@nuxt/kit' import { defu } from 'defu' +import type { ModuleOptions as AgentDiscoveryOptions } from 'nuxt-agent-discovery' import { resolveContentDir } from '../utils/content-dir' import { getGitBranch, getGitEnv, getGitRoot, getLocalGitInfo } from '../utils/git' import { LAYER_ICON_COLLECTIONS } from '../utils/icons' @@ -26,12 +27,8 @@ export interface ComarkDocsOptions { /** GitHub repos (`owner/name`) `/api/code-explorer` may read. Defaults to the content repo only. */ allowRepos?: string[] } + /** @deprecated Use `agentDiscovery.skills` instead. */ skills?: { - /** - * Directory, relative to the app root, scanned at build time for Agent Skills. - * Each subdirectory with a `SKILL.md` is published at `/.well-known/skills/`. - * @default 'skills' - */ dir?: string } } @@ -49,12 +46,13 @@ export default defineNuxtModule({ // Untyped view: `site` (nuxt-site-config) and `appConfig` aren't typed until app.config is generated. const nuxtOptions = nuxt.options as typeof nuxt.options & { - site?: { url?: string; name?: string } + site?: { url?: string; name?: string; description?: string } appConfig: Record } - // Static module defaults live in the layer's nuxt.config: seeding them here makes module order - // load-bearing. Only build-time discoveries (git, env, the consumer's content dir) belong below. + // This module is listed first in the layer's nuxt.config, so what is seeded below (`site`, `mcp`, + // `agentDiscovery`) is in place before the modules that read it at setup. Static defaults still belong in + // nuxt.config; only build-time discoveries (git, env, the consumer's content dir) are resolved here. const url = inferSiteURL() const meta = await getPackageJsonMetadata(rootDir) @@ -111,6 +109,8 @@ export default defineNuxtModule({ githubToken: '', webhookSecret: '', bypassToken: '', + // Feeds the OpenAPI document. + version: meta.version || '0.0.0', github: { owner: gitInfo?.owner || '', repo: gitInfo?.name || '', @@ -144,11 +144,38 @@ export default defineNuxtModule({ } }) - const mcpOptions = (nuxt.options as { mcp?: { name?: string; version?: string } }).mcp - ;(nuxt.options as { mcp?: { name?: string; version?: string } }).mcp = defu(mcpOptions, { + const rawMcpOptions = (nuxt.options as { mcp?: false | { name?: string; version?: string; route?: string } }).mcp + const mcp = defu(rawMcpOptions || undefined, { name: `${siteName} Docs`, version: '1.0.0', }) + ;(nuxt.options as { mcp?: typeof mcp }).mcp = mcp + + // What nuxt-agent-discovery cannot know: the MCP server card describing the toolkit's endpoint under the + // same name, and the deprecated `comarkDocs.skills` alias. + if (options.skills) { + logger.warn('`comarkDocs.skills` is deprecated. Move it to `agentDiscovery.skills` in nuxt.config.ts.') + } + const agentDiscovery = (nuxt.options as { agentDiscovery?: AgentDiscoveryOptions }).agentDiscovery + ;(nuxt.options as { agentDiscovery?: AgentDiscoveryOptions }).agentDiscovery = defu(agentDiscovery, { + discovery: { + mcpServerCard: + rawMcpOptions === false + ? false + : { + endpoint: mcp.route || '/mcp', + name: mcp.name, + version: mcp.version, + ...(nuxtOptions.site?.description ? { description: nuxtOptions.site.description } : {}), + }, + }, + ...(options.skills ? { skills: options.skills } : {}), + }) as AgentDiscoveryOptions + + // `llms.txt` sections come from the content navigation at request time. Registered here rather than + // scanned from `server/plugins/` so the hook runs ahead of the nuxt-agent-discovery bridge (see the plugin). + const { resolve } = createResolver(import.meta.url) + addServerPlugin(resolve('./runtime/server/plugins/llms')) // ISR rules here (not `$production`) so they merge cleanly across npm layers; content sections need a redeploy. if (!nuxt.options.dev && options.isr !== false) { @@ -167,6 +194,7 @@ export default defineNuxtModule({ // Global content indexes, purged by the push webhook on content changes. '/llms.txt': { isr }, '/llms-full.txt': { isr }, + '/sitemap.md': { isr }, '/rss.xml': { isr }, // Fetched on every page hydration (see app.vue) and parses every doc body, so cache it. '/api/content/blob/*/search-sections': { isr: true }, diff --git a/modules/markdown-rewrite.ts b/modules/markdown-rewrite.ts deleted file mode 100644 index 463a1a0..0000000 --- a/modules/markdown-rewrite.ts +++ /dev/null @@ -1,36 +0,0 @@ -import { readFile, writeFile } from 'node:fs/promises' -import { defineNuxtModule, useLogger } from '@nuxt/kit' -import { resolve } from 'pathe' -import { buildMarkdownRewriteRoutes } from '../utils/markdown-rewrite' - -const logger = useLogger('comark-docs') - -/** - * Serve raw markdown to agents on the *page* URLs: `Accept: text/markdown` (or a curl user-agent) on - * `/getting-started/installation` 307-redirects to `/raw/getting-started/installation.md`, and `/` - * to `/llms.txt`. Implemented as Vercel routing-layer redirects written into - * `.vercel/output/config.json` after Nitro compiles. Redirects (not rewrites) are required: the ISR - * cache is keyed on the request path only and ignores `Vary`, so a rewrite would let the HTML and - * markdown variants of the same URL poison each other's cache entry. - */ -export default defineNuxtModule({ - meta: { - name: 'comark-docs/markdown-rewrite', - }, - setup(_options, nuxt) { - nuxt.hooks.hook('nitro:init', (nitro) => { - if (nitro.options.dev || !nitro.options.preset.includes('vercel')) return - - nitro.hooks.hook('compiled', async () => { - const configPath = resolve(nitro.options.output.dir, 'config.json') - const config = JSON.parse(await readFile(configPath, 'utf8')) - - const routes = buildMarkdownRewriteRoutes() - config.routes.unshift(...routes) - - await writeFile(configPath, JSON.stringify(config, null, 2), 'utf8') - logger.info(`Injected ${routes.length} markdown content-negotiation routes into ${configPath}`) - }) - }) - }, -}) diff --git a/modules/runtime/server/plugins/llms.ts b/modules/runtime/server/plugins/llms.ts new file mode 100644 index 0000000..60a5ffa --- /dev/null +++ b/modules/runtime/server/plugins/llms.ts @@ -0,0 +1,74 @@ +import type { NavigationItem } from 'comark-content' +import type { NitroApp } from 'nitropack/types' +import type { LLMsSection } from 'nuxt-llms' +import { useAppConfig } from 'nitropack/runtime' + +/** + * Builds `llms.txt` from the content navigation: one section per top-level directory, in sidebar + * order, plus the `docs.llms.links` extras. Registered from modules/config.ts rather than scanned from + * `server/plugins/` so it runs ahead of the nuxt-agent-discovery bridge, which leaves sections + * that carry links alone apart from rewriting every page link to its raw markdown twin, and renders + * `llms-full.txt` from the same content adapter. + */ +export default defineNitroPlugin((nitroApp: NitroApp) => { + nitroApp.hooks.hook('llms:generate', async (event, options) => { + const site = getSiteConfig(event) + const appConfig = useAppConfig(event) + const siteName = appConfig.seo?.siteName || site.name || options.title || '' + + options.title ||= siteName + options.description ||= appConfig.docs?.llms?.description || site.description || '' + + // A consumer declaring `llms.sections` with `navigation` selectors owns the sections; the bridge + // resolves those. Otherwise the intro goes ahead of the "Documentation Sets" entry nuxt-llms seeds. + if (!options.sections.some((section) => 'navigation' in section)) { + const content = await getProdContent() + const navigation = await content.navigation() + options.sections.unshift(documentationSection(navigation, siteName)) + options.sections.push(...navigationSections(navigation)) + } + + const extraLinks = (appConfig.docs?.llms?.links ?? []) as LLMsSection['links'] + if (extraLinks?.length) { + options.sections.push({ title: 'Optional', links: extraLinks }) + } + }) +}) + +/** The landing page and the top-level pages that belong to no section. */ +function documentationSection(navigation: NavigationItem[], siteName: string): LLMsSection { + return { + title: 'Documentation', + description: 'Every page below is available as raw markdown. Fetch any URL directly.', + links: [ + { title: 'Landing page', description: `Overview of ${siteName}`, href: '/' }, + ...pageLinks(navigation.filter((item) => !item.children?.length)), + ], + } +} + +/** One section per top-level directory, carrying its navigation description. */ +function navigationSections(navigation: NavigationItem[]): LLMsSection[] { + const sections: LLMsSection[] = [] + for (const item of navigation) { + if (!item.children?.length) continue + const links = pageLinks([item]) + if (links.length) sections.push({ title: item.title, description: item.description, links }) + } + return sections +} + +/** Every page in the subtree, depth first, linked on its page URL. */ +function pageLinks(items: NavigationItem[]): NonNullable { + const links: NonNullable = [] + const collect = (entries: NavigationItem[]) => { + for (const entry of entries) { + if (entry.page !== false && entry.path && entry.path !== '/') { + links.push({ title: entry.title, description: entry.description, href: entry.path }) + } + if (entry.children?.length) collect(entry.children) + } + } + collect(items) + return links +} diff --git a/modules/skills/index.ts b/modules/skills/index.ts deleted file mode 100644 index a901709..0000000 --- a/modules/skills/index.ts +++ /dev/null @@ -1,49 +0,0 @@ -import { addPrerenderRoutes, addServerHandler, createResolver, defineNuxtModule, useLogger } from '@nuxt/kit' -import { defu } from 'defu' -import { join } from 'pathe' -import { scanSkills } from './utils' -import type { ComarkDocsOptions } from '../config' - -const logger = useLogger('comark-docs') - -export default defineNuxtModule({ - meta: { - name: 'comark-docs/skills', - }, - async setup(_options, nuxt) { - const comarkDocs = (nuxt.options as typeof nuxt.options & { comarkDocs?: ComarkDocsOptions }).comarkDocs - const skillsDir = join(nuxt.options.rootDir, comarkDocs?.skills?.dir || 'skills') - - const { catalog, warnings } = await scanSkills(skillsDir) - for (const warning of warnings) logger.warn(warning) - if (!catalog.length) return - - logger.info(`Found ${catalog.length} agent skill${catalog.length > 1 ? 's' : ''}: ${catalog.map((s) => s.name).join(', ')}`) - nuxt.options.runtimeConfig.skills = { catalog } - - const { resolve } = createResolver(import.meta.url) - const handler = resolve('./runtime/server/routes/skills-files') - - nuxt.hook('nitro:config', (nitroConfig) => { - nitroConfig.serverAssets ||= [] - nitroConfig.serverAssets.push({ baseName: 'skills', dir: skillsDir }) - }) - - const prerenderRoutes = ['/.well-known/skills', '/.well-known/skills/', '/.well-known/skills/index.json'] - for (const skill of catalog) { - for (const file of skill.files) { - prerenderRoutes.push(`/.well-known/skills/${skill.name}/${file}`) - } - } - addPrerenderRoutes(prerenderRoutes) - - if (!nuxt.options.dev && comarkDocs?.isr !== false) { - nuxt.options.routeRules = defu(nuxt.options.routeRules, { - '/.well-known/skills/**': { isr: true }, - }) as typeof nuxt.options.routeRules - } - - addServerHandler({ route: '/.well-known/skills', handler }) - addServerHandler({ route: '/.well-known/skills/**', handler }) - }, -}) diff --git a/modules/skills/runtime/server/routes/skills-files.ts b/modules/skills/runtime/server/routes/skills-files.ts deleted file mode 100644 index ff90bfe..0000000 --- a/modules/skills/runtime/server/routes/skills-files.ts +++ /dev/null @@ -1,53 +0,0 @@ -import { resolveSkillFilePath, type SkillEntry } from '../../../utils' - -const PREFIX = '/.well-known/skills/' -const CONTENT_TYPES: Record = { - '.md': 'text/markdown; charset=utf-8', - '.json': 'application/json; charset=utf-8', - '.yaml': 'text/yaml; charset=utf-8', - '.yml': 'text/yaml; charset=utf-8', - '.txt': 'text/plain; charset=utf-8', - '.py': 'text/plain; charset=utf-8', - '.sh': 'text/plain; charset=utf-8', - '.js': 'text/javascript; charset=utf-8', - '.ts': 'text/plain; charset=utf-8', -} - -function contentType(path: string): string { - const dot = path.lastIndexOf('.') - return (dot === -1 ? undefined : CONTENT_TYPES[path.slice(dot)]) || 'application/octet-stream' -} - -export default defineEventHandler(async (event) => { - const url = getRequestURL(event) - const idx = url.pathname.indexOf(PREFIX) - const filePath = idx === -1 ? '' : decodeURIComponent(url.pathname.slice(idx + PREFIX.length)) - const { skills } = useRuntimeConfig(event) - - if (!filePath || filePath === 'index.json') { - setHeader(event, 'content-type', 'application/json') - setHeader(event, 'cache-control', 'public, max-age=3600') - return { skills: skills.catalog } - } - - const resolved = resolveSkillFilePath(filePath) - if (!resolved) { - throw createError({ statusCode: 400, statusMessage: 'Bad Request' }) - } - - const catalog = skills.catalog as SkillEntry[] - const skill = catalog.find((entry) => entry.name === resolved.skillName) - if (!skill || !skill.files.includes(resolved.relativeFile)) { - throw createError({ statusCode: 404, statusMessage: 'Not Found' }) - } - - const storagePath = `${resolved.skillName}/${resolved.relativeFile}` - const content = await useStorage('assets:skills').getItemRaw(storagePath) - if (!content) { - throw createError({ statusCode: 404, statusMessage: 'Not Found' }) - } - - setHeader(event, 'content-type', contentType(storagePath)) - setHeader(event, 'cache-control', 'public, max-age=3600') - return content -}) diff --git a/modules/skills/test/skills.test.ts b/modules/skills/test/skills.test.ts deleted file mode 100644 index 3a1246b..0000000 --- a/modules/skills/test/skills.test.ts +++ /dev/null @@ -1,124 +0,0 @@ -import { mkdir, mkdtemp, symlink, writeFile } from 'node:fs/promises' -import { tmpdir } from 'node:os' -import { join } from 'pathe' -import { describe, expect, it } from 'vitest' -import { resolveSkillFilePath, scanSkills } from '../utils' - -async function skillsRoot(): Promise { - return mkdtemp(join(tmpdir(), 'comark-skills-')) -} - -async function writeSkill(root: string, name: string, skillMd: string, extra: Record = {}) { - const dir = join(root, name) - await mkdir(dir, { recursive: true }) - await writeFile(join(dir, 'SKILL.md'), skillMd) - for (const [rel, body] of Object.entries(extra)) { - const path = join(dir, rel) - await mkdir(join(path, '..'), { recursive: true }) - await writeFile(path, body) - } -} - -describe('scanSkills', () => { - it('catalogues a valid skill with supporting files, SKILL.md first', async () => { - const root = await skillsRoot() - await writeSkill( - root, - 'my-product', - '---\nname: my-product\ndescription: >\n Build apps with My Product.\n---\n', - { - 'references/api.md': '# API\n', - 'scripts/setup.sh': '#!/bin/sh\n', - } - ) - - const { catalog, warnings } = await scanSkills(root) - expect(warnings).toEqual([]) - expect(catalog).toEqual([ - { - name: 'my-product', - description: 'Build apps with My Product.\n', - files: ['SKILL.md', 'references/api.md', 'scripts/setup.sh'], - }, - ]) - }) - - it('defaults name to the directory when frontmatter omits it', async () => { - const root = await skillsRoot() - await writeSkill(root, 'create-project', '---\ndescription: Scaffold a project.\n---\n') - expect((await scanSkills(root)).catalog[0]?.name).toBe('create-project') - }) - - it('skips skills without a description, with an invalid name, or with a name/dir mismatch', async () => { - const root = await skillsRoot() - await writeSkill(root, 'no-desc', '---\nname: no-desc\n---\n') - await writeSkill(root, 'BadName', '---\nname: BadName\ndescription: Nope.\n---\n') - await writeSkill(root, 'mismatch', '---\nname: other\ndescription: Nope.\n---\n') - await writeSkill(root, 'ok-skill', '---\ndescription: Fine.\n---\n') - - const { catalog, warnings } = await scanSkills(root) - expect(catalog.map((s) => s.name)).toEqual(['ok-skill']) - expect(warnings).toHaveLength(3) - }) - - it('omits hidden files from the catalog', async () => { - const root = await skillsRoot() - await writeSkill(root, 'my-skill', '---\ndescription: Hidden files stay private.\n---\n', { - '.secret': 'nope', - 'refs/.cache': 'nope', - }) - expect((await scanSkills(root)).catalog[0]?.files).toEqual(['SKILL.md']) - }) - - it('returns an empty catalog when the directory is missing', async () => { - expect(await scanSkills(join(tmpdir(), 'comark-skills-missing'))).toEqual({ - catalog: [], - warnings: [], - }) - }) - - it('skips a skill whose SKILL.md is a directory, without aborting the scan', async () => { - const root = await skillsRoot() - await mkdir(join(root, 'broken', 'SKILL.md'), { recursive: true }) - await writeSkill(root, 'ok-skill', '---\ndescription: Fine.\n---\n') - - const { catalog, warnings } = await scanSkills(root) - expect(catalog.map((s) => s.name)).toEqual(['ok-skill']) - expect(warnings.some((w) => w.includes('broken') && w.includes('not a file'))).toBe(true) - }) - - it('does not list a symlink that points outside the skill directory', async () => { - const root = await skillsRoot() - const outside = await mkdtemp(join(tmpdir(), 'comark-skills-outside-')) - await writeFile(join(outside, 'secret.md'), 'leaked') - await writeSkill(root, 'my-skill', '---\ndescription: Fine.\n---\n', { - 'references/api.md': '# API\n', - }) - await symlink(join(outside, 'secret.md'), join(root, 'my-skill', 'leaked.md')) - await symlink(outside, join(root, 'my-skill', 'escape')) - - expect((await scanSkills(root)).catalog[0]?.files).toEqual(['SKILL.md', 'references/api.md']) - }) -}) - -describe('resolveSkillFilePath', () => { - it('normalises in-skill `.` / `..` segments', () => { - expect(resolveSkillFilePath('my-skill/refs/../SKILL.md')).toEqual({ - skillName: 'my-skill', - relativeFile: 'SKILL.md', - }) - expect(resolveSkillFilePath('my-skill/./references/api.md')).toEqual({ - skillName: 'my-skill', - relativeFile: 'references/api.md', - }) - }) - - it('rejects paths that escape or have no file', () => { - expect(resolveSkillFilePath('../etc/passwd')).toBeNull() - expect(resolveSkillFilePath('my-skill/../../etc/passwd')).toBeNull() - expect(resolveSkillFilePath('/etc/passwd')).toBeNull() - expect(resolveSkillFilePath('my-skill/SKILL.md\0.png')).toBeNull() - expect(resolveSkillFilePath('my-skill')).toBeNull() - expect(resolveSkillFilePath('')).toBeNull() - }) -}) diff --git a/modules/skills/utils/index.ts b/modules/skills/utils/index.ts deleted file mode 100644 index 8a7d017..0000000 --- a/modules/skills/utils/index.ts +++ /dev/null @@ -1,117 +0,0 @@ -import { lstat, readdir, readFile, stat } from 'node:fs/promises' -import { load as parseYaml } from 'js-yaml' -import { isAbsolute, join, normalize } from 'pathe' - -export interface SkillEntry { - name: string - description: string - files: string[] -} - -export interface ScanSkillsResult { - catalog: SkillEntry[] - warnings: string[] -} - -const SKILL_NAME_REGEX = /^[a-z0-9](?:[a-z0-9-]*[a-z0-9])?$/ -const MAX_NAME_LENGTH = 64 - -function skillNameError(name: string, dirName: string): string | null { - if (name.length > MAX_NAME_LENGTH) return `Skill "${name}" exceeds ${MAX_NAME_LENGTH} character limit` - if (!SKILL_NAME_REGEX.test(name) || name.includes('--')) { - return `Skill name "${name}" does not match the Agent Skills naming spec` - } - if (name !== dirName) return `Skill name "${name}" does not match directory name "${dirName}"` - return null -} - -function parseSkillFrontmatter(content: string): { name?: string; description?: string } | null { - const match = content.match(/^---\r?\n([\s\S]*?)\r?\n---/) - if (!match?.[1]) return null - try { - const parsed = parseYaml(match[1]) - if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed)) return null - const record = parsed as Record - return { - name: typeof record.name === 'string' ? record.name : undefined, - description: typeof record.description === 'string' ? record.description : undefined, - } - } catch { - return null - } -} - -/** Normalise a `/.well-known/skills/` relative path into `{skill}/{file}`. */ -export function resolveSkillFilePath(filePath: string): { skillName: string; relativeFile: string } | null { - if (!filePath || filePath.includes('\0')) return null - const n = normalize(filePath.replace(/\\/g, '/')) - if (!n || isAbsolute(n) || n === '..' || n.startsWith('../')) return null - const i = n.indexOf('/') - if (i <= 0 || i === n.length - 1) return null - return { skillName: n.slice(0, i), relativeFile: n.slice(i + 1) } -} - -async function listFilesRecursively(dir: string, base = ''): Promise { - const files: string[] = [] - const entries = await readdir(dir, { withFileTypes: true }) - for (const entry of entries) { - if (entry.name.startsWith('.') || entry.isSymbolicLink()) continue - const relPath = base ? `${base}/${entry.name}` : entry.name - if (entry.isDirectory()) files.push(...(await listFilesRecursively(join(dir, entry.name), relPath))) - else if (entry.isFile()) files.push(relPath) - } - return files -} - -/** Scan a `skills/` directory and return a discovery catalog (v0.1 `.well-known/skills` shape). */ -export async function scanSkills(skillsDir: string): Promise { - const catalog: SkillEntry[] = [] - const warnings: string[] = [] - - const rootStat = await stat(skillsDir).catch(() => null) - if (!rootStat?.isDirectory()) return { catalog, warnings } - - for (const entry of await readdir(skillsDir, { withFileTypes: true })) { - if (!entry.isDirectory() || entry.isSymbolicLink()) continue - - const skillDir = join(skillsDir, entry.name) - const skillMdPath = join(skillDir, 'SKILL.md') - const mdStat = await lstat(skillMdPath).catch(() => null) - if (!mdStat) continue - if (!mdStat.isFile()) { - warnings.push(`Skipping skill "${entry.name}": SKILL.md is not a file`) - continue - } - - let content: string - try { - content = await readFile(skillMdPath, 'utf-8') - } catch { - warnings.push(`Skipping skill "${entry.name}": could not read SKILL.md`) - continue - } - - const frontmatter = parseSkillFrontmatter(content) - if (!frontmatter?.description?.trim()) { - warnings.push(`Skipping skill "${entry.name}": missing description in SKILL.md frontmatter`) - continue - } - - const name = frontmatter.name || entry.name - const nameError = skillNameError(name, entry.name) - if (nameError) { - warnings.push(nameError) - continue - } - - const files = await listFilesRecursively(skillDir) - catalog.push({ - name, - description: frontmatter.description, - files: ['SKILL.md', ...files.filter((f) => f !== 'SKILL.md').sort()], - }) - } - - catalog.sort((a, b) => a.name.localeCompare(b.name)) - return { catalog, warnings } -} diff --git a/nuxt.config.ts b/nuxt.config.ts index ee65cd7..660cdd1 100644 --- a/nuxt.config.ts +++ b/nuxt.config.ts @@ -1,11 +1,19 @@ import { resolveModulePath } from 'exsolve' import { defineNuxtConfig } from 'nuxt/config' +import { createResolver } from 'nuxt/kit' import { layerIconCollections } from './utils/icons' +const { resolve } = createResolver(import.meta.url) + export default defineNuxtConfig({ compatibilityDate: '2026-06-09', devtools: { enabled: true }, modules: [ + // The layer's own modules go first: what config.ts seeds (`site`, `mcp`, `agentDiscovery`) has to be in + // place before the modules that read it at setup. Nuxt queues a layer's `modules` before its scanned + // `modules/` dir and dedupes by file path, so the extension is what keeps these from installing twice. + resolve('./modules/config.ts'), + resolve('./modules/css.ts'), '@nuxt/ui', '@comark/nuxt', '@nuxtjs/robots', @@ -14,11 +22,31 @@ export default defineNuxtConfig({ 'nuxt-og-image', '@nuxtjs/mcp-toolkit', 'nuxt-llms', + 'nuxt-agent-discovery', ], ignore: ['content/**'], ui: { content: true, prose: true }, sitemap: { - sources: ['/api/__sitemap__/urls'], exclude: ['/tree/**', '/blob/**', '/pr/**'] + // Content is the source of truth: the app sources would only add the prerendered skill files. + excludeAppSources: true, + sources: ['/api/__sitemap__/urls'], + exclude: ['/tree/**', '/blob/**', '/pr/**'], + }, + // Markdown for agents: content negotiation on every page, `/raw/**`, the `llms.txt` bridge, `sitemap.md`, + // the api-catalog, the MCP server card and Agent Skills. Every page negotiates, since the content sections + // are only known at request time. The server card is seeded in modules/config.ts, where the site name is. + agentDiscovery: { + // comark sites build their own content instance, so the adapter is a file rather than auto-detected. + source: resolve('./server/utils/agent-source.ts'), + // Versioned previews serve HTML only, and `/logos` is a layer page with no document behind it. + excludePrefixes: { extend: ['/tree/', '/blob/', '/pr/', '/logos'] }, + discovery: { + // `server/routes/openapi.json.get.ts`, the one document the module cannot know about. + links: [ + { href: '/openapi.json', rel: 'service-desc', type: 'application/vnd.oai.openapi+json', title: 'OpenAPI document: every route this site serves to agents', anchor: '/' }, + { href: '/rss.xml', rel: 'alternate', type: 'application/rss+xml', title: 'RSS feed of the documentation' }, + ], + }, }, ogImage: { zeroRuntime: false }, icon: { @@ -45,7 +73,10 @@ export default defineNuxtConfig({ routeRules: { '/llms.txt': { prerender: false }, '/llms-full.txt': { prerender: false }, + '/openapi.json': { prerender: true }, }, + // MCP tool handlers reach the request through `useEvent()`. + experimental: { asyncContext: true }, vercel: { config: { bypassToken: process.env.VERCEL_BYPASS_TOKEN, diff --git a/package.json b/package.json index fe53c21..b15c22e 100644 --- a/package.json +++ b/package.json @@ -63,6 +63,7 @@ "exsolve": "^1.1.1", "js-yaml": "^5.3.0", "motion-v": "^2.4.0", + "nuxt-agent-discovery": "^0.5.0", "nuxt-llms": "^0.2.0", "nuxt-og-image": "^6.7.8", "nuxt-seo-utils": "^8.4.2", diff --git a/playground/content/1.getting-started/3.configuration.md b/playground/content/1.getting-started/3.configuration.md index 69f9d25..e86fc9d 100644 --- a/playground/content/1.getting-started/3.configuration.md +++ b/playground/content/1.getting-started/3.configuration.md @@ -36,9 +36,6 @@ export default defineNuxtConfig({ codeExplorer: { allowRepos: ['my-org/examples'], }, - skills: { - dir: 'skills', - }, }, }) ``` @@ -48,12 +45,28 @@ export default defineNuxtConfig({ | `isr` | `300` | ISR expiration in seconds for the generated route rules. Set to `false` to disable them entirely. | | `contentDir` | inferred | Content directory **relative to the repository root** (an app in `docs/` becomes `docs/content`). Inferred from the local git checkout. | | `codeExplorer.allowRepos` | `[]` | Extra `owner/repo` entries the [CodeExplorer component](/writing/components#codeexplorer) may fetch from. The content repository is always allowed. | -| `skills.dir` | `'skills'` | Directory scanned for [Agent Skills](https://agentskills.io) served at `/.well-known/skills/`. | ::warning A build that can't see `.git` (a shallow Docker build, an exported tarball) can't infer `contentDir` and assumes the app is the repository root. If your app lives in a subdirectory, set `comarkDocs.contentDir` (or `NUXT_DOCS_CONTENT_DIR`) explicitly — the build warns when it has to guess. :: +### agentDiscovery options + +[Markdown for agents](/concepts/architecture#markdown-for-agents) is [nuxt-agent-discovery](https://github.com/benjamincanac/nuxt-agent-discovery), configured under `agentDiscovery`. The layer points it at the content instance, keeps versioned previews out of negotiation and seeds the MCP server card. Everything else is the module's defaults, and the option you are most likely to touch is the [Agent Skills](https://agentskills.io) directory: + +```ts [nuxt.config.ts] +export default defineNuxtConfig({ + extends: ['comark-docs'], + agentDiscovery: { + skills: { + dir: 'skills', + }, + }, +}) +``` + +Each subdirectory holding a `SKILL.md` with a `description` is published at `/.well-known/skills/`. Skills are scanned at build time from the app, not from GitHub-sourced content, so a skill change needs a redeploy. + ## app.config.ts Branding and navigation live in [`app.config.ts`](https://nuxt.com/docs/guide/directory-structure/app-config). Everything is optional — this playground's own config is a good starting point: diff --git a/playground/content/3.concepts/1.architecture.md b/playground/content/3.concepts/1.architecture.md index 56455cb..adf5c82 100644 --- a/playground/content/3.concepts/1.architecture.md +++ b/playground/content/3.concepts/1.architecture.md @@ -73,16 +73,13 @@ Without the webhook, the site still updates: ISR entries expire on their own aft ## Markdown for agents -Every production documentation page is mirrored as raw Markdown at `/raw/.md` ([versioned previews](/concepts/versioned-previews) serve HTML only). The mirrors carry the same ISR caching as the HTML pages. +Every production documentation page is mirrored as raw Markdown at `/raw/.md` ([versioned previews](/concepts/versioned-previews) serve HTML only). The mirrors carry the same ISR caching as the HTML pages. This part of the site is [nuxt-agent-discovery](https://github.com/benjamincanac/nuxt-agent-discovery), reading the same `comark-content` instance that renders the HTML. -On Vercel, agents don't need to know the mirror URLs. The layer injects redirects into the build output: +Agents don't need to know the mirror URLs. A page URL answers Markdown when the request sends `Accept: text/markdown`, appends `.md` to the path, or comes from a known agent user agent (ClaudeBot, GPTBot, PerplexityBot and the rest of the [ai.robots.txt](https://github.com/ai-robots-txt/ai.robots.txt) list). `/` answers the landing page followed by a *Resources for Agents* list of the discovery documents. -- A request for any page URL with `Accept: text/markdown` (or a curl user-agent) gets a 307 redirect to its `/raw/**` mirror. -- A request for `/` gets a 307 redirect to `/llms.txt`. +On Vercel the negotiation runs at the edge, before the ISR cache sees the request. A cached page 307-redirects to its `/raw/**` mirror rather than being rewritten: the ISR cache is keyed on the request path alone and ignores `Vary`, so serving both variants under one URL would let them overwrite each other's cache entry. Follow redirects when fetching, e.g. `curl -L`. In development the same negotiation runs in a Nitro middleware and answers in place. -A redirect (rather than a rewrite) is required because the ISR cache is keyed on the request path alone and ignores `Vary` — a rewrite would let the HTML and Markdown variants of the same URL overwrite each other's cache entry. With redirects, each path caches exactly one variant. Follow redirects when fetching, e.g. `curl -L`. - -A request for a page that doesn't exist returns a real HTTP 404 with a short Markdown body pointing at `/llms.txt`, `/llms-full.txt`, and the sitemap, so an agent that guesses a URL wrong can recover. +A request for a page that doesn't exist returns a real HTTP 404 with a short Markdown body pointing at the discovery documents, so an agent that guesses a URL wrong can recover. The same documents are advertised in a `Link` header on `/` and in `/.well-known/api-catalog`: `/llms.txt` and `/llms-full.txt`, `/sitemap.md` (every page, grouped by section), `/openapi.json`, the MCP server card at `/.well-known/mcp/server-card.json`, and the [Agent Skills](/getting-started/configuration#agentdiscovery-options) index. `robots.txt` allows the same agent list. ## No redeploys for content diff --git a/playground/content/4.deployment/1.vercel.md b/playground/content/4.deployment/1.vercel.md index f1021ed..7482ee7 100644 --- a/playground/content/4.deployment/1.vercel.md +++ b/playground/content/4.deployment/1.vercel.md @@ -97,7 +97,7 @@ Replace `contentSha` with another full commit SHA to move the pin. Delete the it ## ISR behavior -The layer generates ISR route rules for every top-level content section, the landing page, previews, and the machine-readable routes (`/raw/**`, `/llms.txt`, `/rss.xml`). Pages expire after 300 seconds by default, or immediately when the webhook purges them. +The layer generates ISR route rules for every top-level content section, the landing page, previews, and the machine-readable routes (`/raw/**`, `/llms.txt`, `/sitemap.md`, `/rss.xml`). Pages expire after 300 seconds by default, or immediately when the webhook purges them. Tune or disable this with [`comarkDocs.isr`](/getting-started/configuration#nuxtconfigts-comarkdocs-options) in `nuxt.config.ts`. Your own `routeRules` take precedence over the generated ones. diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 40b81d2..e574e9b 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -98,6 +98,9 @@ importers: motion-v: specifier: ^2.4.0 version: 2.4.0(@vueuse/core@14.4.0(vue@3.5.41(typescript@6.0.3)))(vue@3.5.41(typescript@6.0.3)) + nuxt-agent-discovery: + specifier: ^0.5.0 + version: 0.5.0(0c059d5f95a64ae6a0dd943b0684c648) nuxt-llms: specifier: ^0.2.0 version: 0.2.0(magic-string@1.2.3)(magicast@0.5.4)(oxc-parser@0.143.0)(rolldown@1.2.5)(unplugin@3.3.0(esbuild@0.28.2)(rolldown@1.2.5)(rollup@4.62.5)(vite@8.2.2(@types/node@26.2.0)(esbuild@0.28.2)(jiti@2.7.0)(terser@5.50.0)(yaml@2.9.0))) @@ -4966,6 +4969,29 @@ packages: nth-check@2.1.1: resolution: {integrity: sha512-lqjrjmaOoAnWfMmBPL+XNnynZh2+swxiX3WUE0s4yEHI6m+AwrK2UZOimIRl3X/4QctVqS8AiZjFqyOGrMXb/w==} + nuxt-agent-discovery@0.5.0: + resolution: {integrity: sha512-FTEnPiMdVliNk3rc8fAKgg/x6mPokmaePug367tEhnDQUAJ0TM4ShMicTxNQWVTdUXRCKPlGU8DX2wj+5MaHHQ==} + peerDependencies: + '@nuxt/content': ^3.0.0 + '@nuxtjs/mcp-toolkit': '>=0.19.0' + '@nuxtjs/robots': ^6.0.0 + '@nuxtjs/sitemap': ^8.0.0 + comark: ^0.6.0 + nuxt-llms: '>=0.1.0' + peerDependenciesMeta: + '@nuxt/content': + optional: true + '@nuxtjs/mcp-toolkit': + optional: true + '@nuxtjs/robots': + optional: true + '@nuxtjs/sitemap': + optional: true + comark: + optional: true + nuxt-llms: + optional: true + nuxt-llms@0.2.0: resolution: {integrity: sha512-GoEW00x8zaZ1wS0R0aOYptt3b54JEaRwlyVtuAiQoH51BwYdjN5/3+00/+4wi39M5cT4j5XcnGwOxJ7v4WVb9A==} @@ -11869,6 +11895,26 @@ snapshots: dependencies: boolbase: 1.0.0 + nuxt-agent-discovery@0.5.0(0c059d5f95a64ae6a0dd943b0684c648): + dependencies: + '@nuxt/kit': 4.5.2(magic-string@1.2.3)(magicast@0.5.4)(oxc-parser@0.143.0)(rolldown@1.2.5)(unplugin@3.3.0(esbuild@0.28.2)(rolldown@1.2.5)(rollup@4.62.5)(vite@8.2.2(@types/node@26.2.0)(esbuild@0.28.2)(jiti@2.7.0)(terser@5.50.0)(yaml@2.9.0))) + defu: 6.1.7 + pathe: 2.0.3 + ufo: 1.6.4 + yaml: 2.9.0 + optionalDependencies: + '@nuxtjs/mcp-toolkit': 0.18.1(@vue/compiler-sfc@3.5.41)(h3@1.15.11)(magic-string@1.2.3)(magicast@0.5.4)(oxc-parser@0.143.0)(rolldown@1.2.5)(rollup@4.62.5)(supports-color@10.2.2)(unplugin@3.3.0(esbuild@0.28.2)(rolldown@1.2.5)(rollup@4.62.5)(vite@8.2.2(@types/node@26.2.0)(esbuild@0.28.2)(jiti@2.7.0)(terser@5.50.0)(yaml@2.9.0)))(vite@8.2.2(@types/node@26.2.0)(esbuild@0.28.2)(jiti@2.7.0)(terser@5.50.0)(yaml@2.9.0))(vue@3.5.41(typescript@6.0.3))(zod@4.4.3) + '@nuxtjs/robots': 6.2.0(3bc4744619a624c5169bc56293f3c985) + '@nuxtjs/sitemap': 8.5.0(3bc4744619a624c5169bc56293f3c985) + comark: 0.6.2(beautiful-mermaid@1.1.3)(rangi@2.2.0)(shiki@4.4.3) + nuxt-llms: 0.2.0(magic-string@1.2.3)(magicast@0.5.4)(oxc-parser@0.143.0)(rolldown@1.2.5)(unplugin@3.3.0(esbuild@0.28.2)(rolldown@1.2.5)(rollup@4.62.5)(vite@8.2.2(@types/node@26.2.0)(esbuild@0.28.2)(jiti@2.7.0)(terser@5.50.0)(yaml@2.9.0))) + transitivePeerDependencies: + - magic-string + - magicast + - oxc-parser + - rolldown + - unplugin + nuxt-llms@0.2.0(magic-string@1.2.3)(magicast@0.5.4)(oxc-parser@0.143.0)(rolldown@1.2.5)(unplugin@3.3.0(esbuild@0.28.2)(rolldown@1.2.5)(rollup@4.62.5)(vite@8.2.2(@types/node@26.2.0)(esbuild@0.28.2)(jiti@2.7.0)(terser@5.50.0)(yaml@2.9.0))): dependencies: '@nuxt/kit': 4.5.2(magic-string@1.2.3)(magicast@0.5.4)(oxc-parser@0.143.0)(rolldown@1.2.5)(unplugin@3.3.0(esbuild@0.28.2)(rolldown@1.2.5)(rollup@4.62.5)(vite@8.2.2(@types/node@26.2.0)(esbuild@0.28.2)(jiti@2.7.0)(terser@5.50.0)(yaml@2.9.0))) diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index 669fb7c..a29c888 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -18,6 +18,7 @@ minimumReleaseAgeExclude: - '@comark/vue@0.6.0 || 0.6.1 || 0.6.2' - comark@0.6.0 || 0.6.1 || 0.6.2 - comark-content@0.3.0 + - nuxt-agent-discovery@0.5.0 overrides: # Keep the workspace on a single h3 major (v1), matching comark-content. diff --git a/server/api/assistant.post.ts b/server/api/assistant.post.ts index 1963ce1..15bcd24 100644 --- a/server/api/assistant.post.ts +++ b/server/api/assistant.post.ts @@ -10,6 +10,7 @@ import { import { gateway } from '@ai-sdk/gateway' import { z } from 'zod' import type { NavigationItem } from 'comark-content' +import { getAgentDocument } from '#agent-discovery' /** Keep the context bounded on a public endpoint: only the tail of long conversations is forwarded. */ const MAX_MESSAGES = 20 @@ -96,8 +97,10 @@ ${pageIndex(navigation)}`, path: z.string().describe('Page path, e.g. /getting-started/installation'), }), execute: async ({ path }) => { - const markdown = await renderPageMarkdown(content, path.startsWith('/') ? path : `/${path}`) - return markdown ?? `Page not found: ${path}. Use a path from the page index.` + const document = await getAgentDocument(event, path.startsWith('/') ? path : `/${path}`) + if (!document) return `Page not found: ${path}. Use a path from the page index.` + if ('redirect' in document) return `${path} is a section, not a page. Read ${document.redirect} instead.` + return document.markdown }, }), }, diff --git a/server/api/revalidate.post.ts b/server/api/revalidate.post.ts index 51de3d5..4a7451d 100644 --- a/server/api/revalidate.post.ts +++ b/server/api/revalidate.post.ts @@ -150,8 +150,9 @@ export default defineEventHandler(async (event) => { // URL the browser loads (`…/_payload.json?`). const buildId = useRuntimeConfig(event).app.buildId - // Any content change invalidates the llms indexes, the feed, and the body-derived search index. - const paths = new Set(['/llms.txt', '/llms-full.txt', '/rss.xml', '/api/content/search-sections']) + // Any content change invalidates the llms indexes, the markdown sitemap, the feed, and the body-derived + // search index. + const paths = new Set(['/llms.txt', '/llms-full.txt', '/sitemap.md', '/rss.xml', '/api/content/search-sections']) for (const f of changedFiles) { const pageUrl = pageUrlForPath(f) if (pageUrl) { diff --git a/server/mcp/tools/get-page.ts b/server/mcp/tools/get-page.ts index ade0a71..d9caf7e 100644 --- a/server/mcp/tools/get-page.ts +++ b/server/mcp/tools/get-page.ts @@ -1,25 +1,33 @@ import { z } from 'zod' import { withLeadingSlash } from 'ufo' +import { useEvent } from 'nitropack/runtime' +import { getAgentDocument } from '#agent-discovery' export default defineMcpTool({ description: - 'Read a documentation page as markdown. Pass the page path from list-pages, e.g. /getting-started/installation.', + 'Read a documentation page as markdown. Pass the page path from list-pages, e.g. /getting-started/installation. Pass sections to keep only the named `##` headings of a long page.', inputSchema: { path: z.string().describe('Page path, e.g. /getting-started/installation'), + sections: z.array(z.string()).optional().describe('Titles of the `##` sections to keep, e.g. ["Setup"]'), }, - handler: async ({ path }) => { - const content = await getProdContent() - - const markdown = await renderPageMarkdown(content, withLeadingSlash(path)) - if (!markdown) { + handler: async ({ path, sections }) => { + // The same document `/raw/.md` serves, resolved in-process. + const document = await getAgentDocument(useEvent(), withLeadingSlash(path), { sections }) + if (!document) { return { content: [{ type: 'text', text: `Page not found: ${path}. Use list-pages to see available paths.` }], isError: true, } } + if ('redirect' in document) { + return { + content: [{ type: 'text', text: `${path} is a section, not a page. Read ${document.redirect} instead.` }], + isError: true, + } + } return { - content: [{ type: 'text', text: markdown }], + content: [{ type: 'text', text: document.markdown }], } }, }) diff --git a/server/mcp/tools/list-pages.ts b/server/mcp/tools/list-pages.ts index 2251b2b..faaeb32 100644 --- a/server/mcp/tools/list-pages.ts +++ b/server/mcp/tools/list-pages.ts @@ -1,24 +1,16 @@ -import type { NavigationItem } from 'comark-content' +import { useEvent } from 'nitropack/runtime' +import { listAgentPages } from '#agent-discovery' export default defineMcpTool({ description: 'List every page of the documentation with its path, title, and description. Use get-page to read a page.', handler: async () => { - const content = await getProdContent() - const navigation = await content.navigation() + const pages = await listAgentPages(useEvent()) - const lines: string[] = [] - const collect = (items: NavigationItem[], section?: string) => { - for (const item of items) { - if (item.page !== false && item.path) { - lines.push( - `${item.path} — ${item.title}${item.description ? `: ${item.description}` : ''}${section ? ` (${section})` : ''}` - ) - } - if (item.children?.length) collect(item.children, section ?? item.title) - } - } - collect(navigation) + const lines = pages.map( + (page) => + `${page.route} — ${page.title ?? page.route}${page.description ? `: ${page.description}` : ''}${page.section ? ` (${page.section})` : ''}` + ) return { content: [{ type: 'text', text: lines.join('\n') }], diff --git a/server/plugins/llms.ts b/server/plugins/llms.ts deleted file mode 100644 index f7de9a9..0000000 --- a/server/plugins/llms.ts +++ /dev/null @@ -1,138 +0,0 @@ -import { appendHeader } from 'h3' -import type { NavigationItem } from 'comark-content' -import { useAppConfig } from 'nitropack/runtime' -import type { NitroApp } from 'nitropack/types' -import { joinURL } from 'ufo' -import type { LLMsSection } from 'nuxt-llms' - -export default defineNitroPlugin((nitroApp: NitroApp) => { - const prerenderPaths = new Set() - - nitroApp.hooks.hook('llms:generate', async (event, options) => { - const content = await getProdContent() - const navigation = await content.navigation() - const site = getSiteConfig(event) - const appConfig = useAppConfig(event) - const siteName = appConfig.seo?.siteName || site.name || options.title || '' - const siteUrl = options.domain || site.url || '/' - - options.title ||= siteName - options.description ||= appConfig.docs?.llms?.description || site.description || '' - - const introLinks = [ - { - title: 'Landing page', - description: `Overview of ${siteName}`, - href: documentLink('/', siteUrl, prerenderPaths), - }, - ...standaloneLinks(navigation, siteUrl, prerenderPaths), - ] - - options.sections.unshift({ - title: 'Documentation', - description: 'Every page below is available as raw markdown. Fetch any URL directly.', - links: introLinks, - }) - - for (const item of navigation) { - if (!item.children?.length) continue - - const links = sectionLinks(item, siteUrl, prerenderPaths) - if (!links.length) continue - - options.sections.push({ - title: item.title, - description: item.description, - links, - }) - } - - const extraLinks = (appConfig.docs?.llms?.links ?? []) as LLMsSection['links'] - if (extraLinks?.length) { - options.sections.push({ - title: 'Optional', - links: extraLinks, - }) - } - }) - - nitroApp.hooks.hook('llms:generate:full', async (_event, _options, contents) => { - const content = await getProdContent() - const navigation = await content.navigation() - - const paths: string[] = [] - const collect = (items: NavigationItem[]) => { - for (const item of items) { - if (item.page !== false && item.path && item.path !== '/') paths.push(item.path) - if (item.children?.length) collect(item.children) - } - } - collect(navigation) - - const pages = await Promise.all( - [...new Set(paths)].map((path) => renderPageMarkdown(content, path)) - ) - contents.push(...pages.filter((page): page is string => Boolean(page))) - }) - - if (['nitro-prerender', 'nitro-dev'].includes(import.meta.preset as string)) { - nitroApp.hooks.hook('beforeResponse', (event) => { - if (event.path === '/') { - appendHeader(event, 'x-nitro-prerender', Array.from(prerenderPaths)) - } - }) - } -}) - -function standaloneLinks( - navigation: NavigationItem[], - siteUrl: string, - prerenderPaths: Set -): NonNullable { - return navigation - .filter((item) => !item.children?.length && item.page !== false && item.path && item.path !== '/') - .map((item) => ({ - title: item.title, - description: item.description, - href: documentLink(item.path, siteUrl, prerenderPaths), - })) -} - -function sectionLinks( - item: NavigationItem, - siteUrl: string, - prerenderPaths: Set -): NonNullable { - const links: NonNullable = [] - - const collect = (items: NavigationItem[]) => { - for (const entry of items) { - if (entry.page !== false && entry.path && entry.path !== '/') { - links.push({ - title: entry.title, - description: entry.description, - href: documentLink(entry.path, siteUrl, prerenderPaths), - }) - } - if (entry.children?.length) collect(entry.children) - } - } - - if (item.page !== false && item.path && item.path !== '/') { - links.push({ - title: item.title, - description: item.description, - href: documentLink(item.path, siteUrl, prerenderPaths), - }) - } - - if (item.children?.length) collect(item.children) - - return links -} - -function documentLink(path: string, domain: string, prerenderPaths: Set) { - const href = joinURL(domain, rawUrlForPage(path)) - prerenderPaths.add(rawUrlForPage(path)) - return href -} diff --git a/server/routes/openapi.json.get.ts b/server/routes/openapi.json.get.ts new file mode 100644 index 0000000..ed99e1c --- /dev/null +++ b/server/routes/openapi.json.get.ts @@ -0,0 +1,24 @@ +import { agentDiscoveryOpenApi, getAgentSiteUrl } from '#agent-discovery' + +/** + * The routes an agent can call on this site: the markdown twin of every page, the discovery documents + * and the MCP endpoint, described by nuxt-agent-discovery from the same route config that serves them. + */ +export default defineEventHandler((event) => { + const config = useRuntimeConfig(event) + const discovery = agentDiscoveryOpenApi(event) + const siteName = config.public.agentDiscovery?.siteName || 'Documentation' + + return { + openapi: '3.1.0', + info: { + title: siteName, + description: `Markdown representations of every ${siteName} page, the documents agents discover the site through, and its MCP endpoint.`, + version: config.docs.version, + }, + servers: [{ url: getAgentSiteUrl(event) }], + tags: discovery.tags, + paths: discovery.paths, + components: discovery.components, + } +}) diff --git a/server/routes/raw/[...slug].md.get.ts b/server/routes/raw/[...slug].md.get.ts deleted file mode 100644 index a871320..0000000 --- a/server/routes/raw/[...slug].md.get.ts +++ /dev/null @@ -1,26 +0,0 @@ -import { findFirstLeaf } from '../../../utils/first-leaf' - -export default defineEventHandler(async (event) => { - const slug = getRouterParams(event)['slug.md'] - if (!slug?.endsWith('.md')) { - return notFoundMarkdown(event, event.path) - } - - const content = await getProdContent() - - const path = pagePathFromRawSlug(slug) - const markdown = await renderPageMarkdown(content, path) - if (!markdown) { - // A directory without an index page (e.g. /raw/getting-started.md) redirects to the - // mirror of its first navigation page — same behaviour as the HTML pages. - const firstLeaf = findFirstLeaf(await content.navigation(), path) - if (firstLeaf) { - return sendRedirect(event, `/raw${firstLeaf}.md`, 302) - } - return notFoundMarkdown(event, path) - } - - setHeader(event, 'Content-Type', 'text/markdown; charset=utf-8') - setHeader(event, 'Vary', 'Accept') - return markdown -}) diff --git a/server/utils/agent-source.ts b/server/utils/agent-source.ts new file mode 100644 index 0000000..0ec9df9 --- /dev/null +++ b/server/utils/agent-source.ts @@ -0,0 +1,8 @@ +import { createComarkSource } from '#agent-discovery/comark' + +/** + * Content adapter behind nuxt-agent-discovery: the raw markdown route, `sitemap.md`, the `llms.txt` + * bridge and the MCP helpers all read production content through it. Versioned previews (`/tree`, + * `/blob`, `/pr`) are excluded from negotiation in nuxt.config.ts and keep serving HTML. + */ +export default createComarkSource(() => getProdContent()) diff --git a/server/utils/markdown.ts b/server/utils/markdown.ts deleted file mode 100644 index f432289..0000000 --- a/server/utils/markdown.ts +++ /dev/null @@ -1,14 +0,0 @@ -import type { DocsContent } from './content' -import { renderMarkdown } from 'comark/render' - -export async function renderPageMarkdown(content: DocsContent, path: string): Promise { - const item = await content.get(path) - if (!item || item.meta.kind !== 'document') return null - - return await renderMarkdown({ nodes: item.nodes, frontmatter: item.data }) -} - -/** Page path → raw markdown URL (`/` → `/raw/index.md`). */ -export function rawUrlForPage(path: string): string { - return path === '/' ? '/raw/index.md' : `/raw/${path.replace(/^\//, '')}.md` -} diff --git a/server/utils/not-found.ts b/server/utils/not-found.ts deleted file mode 100644 index 6165ed2..0000000 --- a/server/utils/not-found.ts +++ /dev/null @@ -1,34 +0,0 @@ -import type { H3Event } from 'h3' -import { setHeader, setResponseStatus } from 'h3' - -/** - * A 404 with a short markdown body instead of the app shell or a JSON error: agents that land on a - * missing page get pointers to the machine-readable indexes so they can recover instead of guessing. - */ -export function notFoundMarkdown(event: H3Event, path?: string): string { - setResponseStatus(event, 404, 'Page not found') - setHeader(event, 'Content-Type', 'text/markdown; charset=utf-8') - setHeader(event, 'Vary', 'Accept') - - return [ - '# Page not found', - '', - path ? `\`${path}\` does not exist on this site.` : 'This page does not exist on this site.', - '', - 'Where to look next:', - '', - '- [/llms.txt](/llms.txt) — index of every documentation page, with raw markdown links', - '- [/llms-full.txt](/llms-full.txt) — the full documentation as a single markdown file', - '- [/raw/index.md](/raw/index.md) — the landing page as markdown', - '- [/sitemap.xml](/sitemap.xml) — sitemap of the rendered pages', - '', - 'Every documentation page is mirrored as raw markdown at `/raw/.md`.', - '', - ].join('\n') -} - -/** `/raw/**` slug (`getting-started/installation.md`) → content path (`/getting-started/installation`). */ -export function pagePathFromRawSlug(slug: string): string { - const stripped = slug.replace(/\.md$/, '') - return stripped === 'index' ? '/' : `/${stripped}` -} diff --git a/test/markdown-rewrite.test.ts b/test/markdown-rewrite.test.ts deleted file mode 100644 index 107845c..0000000 --- a/test/markdown-rewrite.test.ts +++ /dev/null @@ -1,81 +0,0 @@ -import { describe, expect, it } from 'vitest' -import { buildMarkdownRewriteRoutes, type VercelRoute } from '../utils/markdown-rewrite' - -// Vercel resolves `$n` in `headers.Location` from the capture groups of `src` — replicate that to -// assert on the final redirect target rather than on regex internals. -function redirect(routes: VercelRoute[], path: string): VercelRoute & { location: string } | null { - for (const route of routes) { - const match = path.match(new RegExp(route.src)) - if (!match) continue - const location = (route.headers?.Location ?? '').replace(/\$(\d+)/g, (_, n) => match[Number(n)] ?? '') - return { ...route, location } - } - return null -} - -const routes = buildMarkdownRewriteRoutes() - -describe('buildMarkdownRewriteRoutes', () => { - it('pairs every redirect with an Accept matcher and a curl matcher', () => { - expect(routes.length % 2).toBe(0) - const conditions = routes.map((route) => route.has?.[0]) - expect(conditions.filter((c) => c?.key === 'accept').length).toBe(routes.length / 2) - expect(conditions.filter((c) => c?.key === 'user-agent').length).toBe(routes.length / 2) - }) - - it('uses a 307 redirect (never a rewrite, which would share the ISR cache entry) and varies on Accept', () => { - for (const route of routes) { - expect(route.status).toBe(307) - expect(route.headers?.Location).toBeTruthy() - expect(route.headers?.vary).toBe('Accept') - } - }) - - it('sends the landing page to llms.txt', () => { - expect(redirect(routes, '/')?.location).toBe('/llms.txt') - }) - - it('sends pages to their raw markdown mirror', () => { - expect(redirect(routes, '/getting-started/installation')?.location).toBe( - '/raw/getting-started/installation.md' - ) - expect(redirect(routes, '/getting-started/installation/')?.location).toBe( - '/raw/getting-started/installation.md' - ) - expect(redirect(routes, '/writing')?.location).toBe('/raw/writing.md') - }) - - it('never redirects versioned previews (no raw mirrors, HTML only)', () => { - for (const path of [ - '/tree/main', - '/tree/release%2Fv1.2/writing/pages', - '/blob/a1b2c3d', - '/blob/a1b2c3d/getting-started/introduction', - '/pr/28', - '/pr/28/getting-started/introduction', - ]) { - expect(redirect(routes, path), path).toBeNull() - } - }) - - it('never redirects the mirrors, APIs, internals or dotted paths', () => { - for (const path of [ - '/raw/getting-started/installation.md', - '/api/content/search-sections', - '/api/assistant', - '/mcp', - '/logos', - '/_nuxt/entry.js', - '/__nuxt_island/foo', - '/llms.txt', - '/llms-full.txt', - '/sitemap.xml', - '/rss.xml', - '/robots.txt', - '/favicon.ico', - '/.well-known/skills/index.json', - ]) { - expect(redirect(routes, path), path).toBeNull() - } - }) -}) diff --git a/utils/first-leaf.ts b/utils/first-leaf.ts index 7a0f426..b24a150 100644 --- a/utils/first-leaf.ts +++ b/utils/first-leaf.ts @@ -3,7 +3,7 @@ import type { NavigationItem } from 'comark-content' /** * First leaf page under the navigation node at `path`, if `path` is a section (directory) node. * Returns `undefined` when the path isn't in the tree or already is a leaf — used to redirect - * directory URLs like `/getting-started` (and their `/raw/**.md` mirrors) to their first page. + * directory URLs like `/getting-started` to their first page. */ export function findFirstLeaf( navigation: NavigationItem[] | undefined | null, diff --git a/utils/markdown-rewrite.ts b/utils/markdown-rewrite.ts deleted file mode 100644 index bff1a2f..0000000 --- a/utils/markdown-rewrite.ts +++ /dev/null @@ -1,46 +0,0 @@ -// Vercel Build Output routes that redirect agents asking for markdown to the raw mirrors. Injected -// ahead of the generated routing table (see `modules/markdown-rewrite.ts`). These must be redirects, -// not rewrites: the ISR (prerender) cache is keyed on the *request* path only and ignores `Vary`, so -// a rewrite would let the HTML and markdown variants poison each other's cache entry for the same -// URL. A 307 resolves client-side before any cache lookup, and each path keeps a single variant. - -export interface VercelRoute { - src: string - status?: number - headers?: Record - has?: Array<{ type: 'header'; key: string; value?: string }> -} - -// A redirect fires when the client either negotiates markdown or is curl (agents shell out to it). -const MATCHERS: NonNullable[] = [ - [{ type: 'header', key: 'accept', value: '(.*)text/markdown(.*)' }], - [{ type: 'header', key: 'user-agent', value: 'curl/.*' }], -] - -// `src`/`location` pairs, expanded per matcher below. Order matters: first match wins. -const REDIRECTS: Array<{ src: string; location: string }> = [ - // Landing page → the full docs index. - { src: '^/$', location: '/llms.txt' }, - // Every other extensionless page → its raw markdown mirror. Excluded: the mirrors themselves, API - // routes, versioned previews (`/tree`, `/blob`, `/pr` serve HTML only), Nuxt/Nitro internals - // (`_nuxt`, `__nuxt_island`, …), the MCP endpoint and the layer-owned `/logos` page (not - // content-derived, so it has no mirror). `[^.]` also skips every dotted path: `llms.txt`, - // `sitemap.xml`, `robots.txt`, `favicon.ico`, `/.well-known/**`, … - { src: '^/(?!raw/|api/|tree/|blob/|pr/|mcp$|logos$|_)([^.]+?)/?$', location: '/raw/$1.md' }, -] - -/** The full route list to prepend to `.vercel/output/config.json`. */ -export function buildMarkdownRewriteRoutes(): VercelRoute[] { - return REDIRECTS.flatMap(({ src, location }) => - MATCHERS.map((has) => ({ - src, - status: 307, - headers: { - Location: location, - // acceptmarkdown.com: negotiated responses must vary on Accept so shared caches key both variants. - vary: 'Accept', - }, - has, - })) - ) -} diff --git a/utils/meta.ts b/utils/meta.ts index 505f57f..fbdd97e 100644 --- a/utils/meta.ts +++ b/utils/meta.ts @@ -18,10 +18,12 @@ export function inferSiteURL(): string | undefined { return url ? withHttps(url) : undefined } -export async function getPackageJsonMetadata(dir: string): Promise<{ name?: string; description?: string }> { +export async function getPackageJsonMetadata( + dir: string +): Promise<{ name?: string; description?: string; version?: string }> { try { const parsed = JSON.parse(await readFile(resolve(dir, 'package.json'), 'utf-8')) - return { name: parsed.name, description: parsed.description } + return { name: parsed.name, description: parsed.description, version: parsed.version } } catch { return {} }