diff --git a/src/server/lib/audit/issues/ai-readiness.test.ts b/src/server/lib/audit/issues/ai-readiness.test.ts new file mode 100644 index 000000000..2e4bc2d07 --- /dev/null +++ b/src/server/lib/audit/issues/ai-readiness.test.ts @@ -0,0 +1,226 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; +import { + checkAiCrawlerAccess, + checkLlmsTxt, + fetchLlmsTxt, + findLlmsTxtStructureProblem, +} from "@/server/lib/audit/issues/ai-readiness"; + +// The SSRF guard does DNS/network work; make it an identity in tests so the +// redirect paths are exercised without touching the network. +vi.mock("@/server/lib/audit/url-policy", () => ({ + normalizeAndValidateStartUrl: async (u: string) => u, +})); + +const ORIGIN = "https://example.com"; + +function mockFetchSequence(responses: Response[]) { + let i = 0; + vi.stubGlobal( + "fetch", + vi.fn(async () => responses[i++]), + ); +} + +function streamOf(totalBytes: number): ReadableStream { + const chunk = new TextEncoder().encode("x".repeat(100_000)); + let sent = 0; + return new ReadableStream({ + pull(controller) { + if (sent >= totalBytes) { + controller.close(); + return; + } + controller.enqueue(chunk); + sent += chunk.byteLength; + }, + }); +} + +afterEach(() => vi.unstubAllGlobals()); + +describe("checkAiCrawlerAccess", () => { + it("reports nothing when robots.txt is missing", () => { + expect(checkAiCrawlerAccess(ORIGIN, null)).toEqual([]); + }); + + it("reports nothing when robots.txt allows everyone", () => { + expect(checkAiCrawlerAccess(ORIGIN, "User-agent: *\nAllow: /\n")).toEqual( + [], + ); + }); + + it("groups specifically blocked agents by purpose as a readable string", () => { + const robots = [ + "User-agent: GPTBot", + "Disallow: /", + "", + "User-agent: ClaudeBot", + "Disallow: /", + "", + "User-agent: PerplexityBot", + "Disallow: /", + "", + "User-agent: ChatGPT-User", + "Disallow: /", + "", + "User-agent: *", + "Allow: /", + ].join("\n"); + + const issues = checkAiCrawlerAccess(ORIGIN, robots); + const byType = new Map(issues.map((issue) => [issue.issueType, issue])); + + expect(byType.size).toBe(3); + expect( + byType.get("ai-training-crawlers-blocked")?.details?.blockedAgents, + ).toBe("GPTBot (OpenAI), ClaudeBot (Anthropic)"); + expect( + byType.get("ai-search-crawlers-blocked")?.details?.blockedAgents, + ).toBe("PerplexityBot (Perplexity)"); + expect( + byType.get("ai-user-fetchers-blocked")?.details?.blockedAgents, + ).toBe("ChatGPT-User (OpenAI)"); + }); + + it("anchors issues to robots.txt with no page id and a stable dedupe key", () => { + const [issue] = checkAiCrawlerAccess(ORIGIN, "User-agent: GPTBot\nDisallow: /\n"); + expect(issue.pageId).toBeNull(); + expect(issue.pageUrl).toBe(`${ORIGIN}/robots.txt`); + expect(issue.dedupeKey).toBe("training"); + }); + + it("stays silent when the whole site is closed to the generic agent", () => { + expect(checkAiCrawlerAccess(ORIGIN, "User-agent: *\nDisallow: /\n")).toEqual( + [], + ); + }); + + it("ignores agents only blocked from subpaths", () => { + expect( + checkAiCrawlerAccess(ORIGIN, "User-agent: GPTBot\nDisallow: /private/\n"), + ).toEqual([]); + }); +}); + +describe("findLlmsTxtStructureProblem", () => { + it("accepts a minimal spec-compliant file", () => { + expect(findLlmsTxtStructureProblem("# Example Corp\n")).toBeNull(); + }); + + it("tolerates leading blank lines before the H1", () => { + expect(findLlmsTxtStructureProblem("\n\n# Example Corp\n")).toBeNull(); + }); + + it("flags an empty file", () => { + expect(findLlmsTxtStructureProblem(" \n\n")).toBe("file-empty"); + }); + + it("flags a file that does not start with an H1", () => { + expect(findLlmsTxtStructureProblem("Example Corp\n## Docs\n")).toBe( + "missing-h1-title", + ); + }); + + it("does not mistake an H2 for the required H1", () => { + expect(findLlmsTxtStructureProblem("## Docs\n")).toBe("missing-h1-title"); + }); +}); + +describe("checkLlmsTxt", () => { + it("reports missing-llms-txt when the file is absent", () => { + expect(checkLlmsTxt(ORIGIN, { status: "missing" })).toEqual([ + { issueType: "missing-llms-txt", pageId: null, pageUrl: `${ORIGIN}/llms.txt` }, + ]); + }); + + it("stays silent when the fetch failed", () => { + expect(checkLlmsTxt(ORIGIN, { status: "unreachable" })).toEqual([]); + }); + + it("reports nothing for a valid file", () => { + expect( + checkLlmsTxt(ORIGIN, { status: "found", text: "# Example Corp\n" }), + ).toEqual([]); + }); + + it("reports llms-txt-invalid with the specific problem", () => { + expect( + checkLlmsTxt(ORIGIN, { status: "found", text: "just some text\n" }), + ).toEqual([ + { + issueType: "llms-txt-invalid", + pageId: null, + pageUrl: `${ORIGIN}/llms.txt`, + details: { problem: "missing-h1-title" }, + }, + ]); + }); +}); + +describe("fetchLlmsTxt", () => { + it("returns the file on a direct 200", async () => { + mockFetchSequence([ + new Response("# Example Corp\n", { + status: 200, + headers: { "content-type": "text/plain" }, + }), + ]); + expect(await fetchLlmsTxt(ORIGIN)).toEqual({ + status: "found", + text: "# Example Corp\n", + }); + }); + + it("follows a single same-origin redirect", async () => { + mockFetchSequence([ + new Response(null, { + status: 302, + headers: { location: `${ORIGIN}/llms.txt?v=2` }, + }), + new Response("# Example Corp\n", { + status: 200, + headers: { "content-type": "text/plain" }, + }), + ]); + expect(await fetchLlmsTxt(ORIGIN)).toEqual({ + status: "found", + text: "# Example Corp\n", + }); + }); + + it("refuses a cross-origin redirect (SSRF guard)", async () => { + mockFetchSequence([ + new Response(null, { + status: 301, + headers: { location: "https://evil.example.net/llms.txt" }, + }), + ]); + expect(await fetchLlmsTxt(ORIGIN)).toEqual({ status: "missing" }); + }); + + it("rejects an oversized body via the bounded reader", async () => { + mockFetchSequence([ + new Response(streamOf(2_200_000), { + status: 200, + headers: { "content-type": "text/plain" }, + }), + ]); + expect(await fetchLlmsTxt(ORIGIN)).toEqual({ status: "missing" }); + }); + + it("treats an HTML body (SPA catch-all) as no llms.txt", async () => { + mockFetchSequence([ + new Response("", { + status: 200, + headers: { "content-type": "text/html" }, + }), + ]); + expect(await fetchLlmsTxt(ORIGIN)).toEqual({ status: "missing" }); + }); + + it("reports missing on 404", async () => { + mockFetchSequence([new Response(null, { status: 404 })]); + expect(await fetchLlmsTxt(ORIGIN)).toEqual({ status: "missing" }); + }); +}); diff --git a/src/server/lib/audit/issues/ai-readiness.ts b/src/server/lib/audit/issues/ai-readiness.ts new file mode 100644 index 000000000..0d5c45bcc --- /dev/null +++ b/src/server/lib/audit/issues/ai-readiness.ts @@ -0,0 +1,219 @@ +/** + * AI / agent-readiness checks: is the site reachable and legible for AI + * assistants? Covers robots.txt treatment of known AI user agents and the + * llms.txt convention (llmstxt.org). + * + * The check functions are pure over fetched bodies so tests need no network; + * `runAiReadinessChecks` does the one extra fetch (llms.txt) and reuses the + * robots.txt text the discovery phase already checkpointed. + */ +import robotsParser from "robots-parser"; +import type { DetectedIssue } from "@/server/lib/audit/issues/page-reporters"; +import { readBoundedText } from "@/server/lib/scrape"; +import { normalizeAndValidateStartUrl } from "@/server/lib/audit/url-policy"; + +type AiCrawlerPurpose = "search" | "user" | "training"; + +interface AiCrawler { + name: string; + vendor: string; + purpose: AiCrawlerPurpose; +} + +/** + * Well-known AI user agents, grouped by why they visit: "search" agents build + * the indexes AI answers cite, "user" agents fetch a page live when an + * assistant's user asks about it, "training" agents collect model training + * data. Google-Extended is listed under training but also controls grounding + * of Gemini answers — the descriptor text calls that out. + */ +const AI_CRAWLERS: AiCrawler[] = [ + { name: "OAI-SearchBot", vendor: "OpenAI", purpose: "search" }, + { name: "Claude-SearchBot", vendor: "Anthropic", purpose: "search" }, + { name: "PerplexityBot", vendor: "Perplexity", purpose: "search" }, + { name: "ChatGPT-User", vendor: "OpenAI", purpose: "user" }, + { name: "Claude-User", vendor: "Anthropic", purpose: "user" }, + { name: "Perplexity-User", vendor: "Perplexity", purpose: "user" }, + { name: "MistralAI-User", vendor: "Mistral", purpose: "user" }, + { name: "GPTBot", vendor: "OpenAI", purpose: "training" }, + { name: "ClaudeBot", vendor: "Anthropic", purpose: "training" }, + { name: "Google-Extended", vendor: "Google", purpose: "training" }, + { name: "Applebot-Extended", vendor: "Apple", purpose: "training" }, + { name: "meta-externalagent", vendor: "Meta", purpose: "training" }, + { name: "CCBot", vendor: "Common Crawl", purpose: "training" }, + { name: "Bytespider", vendor: "ByteDance", purpose: "training" }, +]; + +const PURPOSE_ISSUE_TYPE = { + search: "ai-search-crawlers-blocked", + user: "ai-user-fetchers-blocked", + training: "ai-training-crawlers-blocked", +} as const; + +/** A UA no robots.txt names, so it resolves through the generic `*` group. */ +const GENERIC_PROBE_AGENT = "OpenSEO-Generic-Probe"; + +/** + * One issue per purpose group listing the AI agents robots.txt blocks from + * the site root. A site that blocks the generic `*` agent made a site-wide + * choice, so only agents treated worse than the generic rules count as an + * AI-specific signal. + */ +export function checkAiCrawlerAccess( + origin: string, + robotsText: string | null, +): DetectedIssue[] { + if (robotsText === null) return []; + const robotsUrl = `${origin}/robots.txt`; + const robots = robotsParser(robotsUrl, robotsText); + const rootUrl = `${origin}/`; + + if ((robots.isAllowed(rootUrl, GENERIC_PROBE_AGENT) ?? true) === false) { + return []; + } + + const blockedByPurpose = new Map(); + for (const crawler of AI_CRAWLERS) { + const allowed = robots.isAllowed(rootUrl, crawler.name) ?? true; + if (allowed) continue; + const group = blockedByPurpose.get(crawler.purpose); + if (group) group.push(crawler); + else blockedByPurpose.set(crawler.purpose, [crawler]); + } + + return Array.from(blockedByPurpose, ([purpose, crawlers]) => ({ + issueType: PURPOSE_ISSUE_TYPE[purpose], + pageId: null, + pageUrl: robotsUrl, + dedupeKey: purpose, + // A flat string, not an array/object: the issues UI renders detail values + // as-is, so a structured value would surface as "[object Object]". + details: { + blockedAgents: crawlers + .map(({ name, vendor }) => `${name} (${vendor})`) + .join(", "), + }, + })); +} + +type LlmsTxtFetchResult = + | { status: "found"; text: string } + | { status: "missing" } + | { status: "unreachable" }; + +const LLMS_TXT_TIMEOUT_MS = 10_000; +const LLMS_TXT_HEADERS = { "User-Agent": "OpenSEO-Audit/1.0" }; + +/** + * Fetch /llms.txt safely. `redirect: "manual"` + a same-origin check stop a + * 30x from steering the request at an internal host (SSRF), and the body is + * read through the shared bounded reader so an oversized file can't exhaust + * memory. Only one redirect hop is followed, and only within the same origin + * (llms.txt lives at the origin root — an off-origin redirect means none here). + */ +export async function fetchLlmsTxt( + origin: string, +): Promise { + try { + const url = `${origin}/llms.txt`; + let response = await fetch(url, { + headers: LLMS_TXT_HEADERS, + redirect: "manual", + signal: AbortSignal.timeout(LLMS_TXT_TIMEOUT_MS), + }); + + if (response.status >= 300 && response.status < 400) { + const location = response.headers.get("location"); + if (!location) return { status: "missing" }; + const target = new URL(location, url); + if (target.origin !== origin) return { status: "missing" }; + let validated: string; + try { + validated = await normalizeAndValidateStartUrl(target.toString()); + } catch { + return { status: "unreachable" }; + } + response = await fetch(validated, { + headers: LLMS_TXT_HEADERS, + redirect: "manual", + signal: AbortSignal.timeout(LLMS_TXT_TIMEOUT_MS), + }); + if (response.status >= 300 && response.status < 400) { + return { status: "unreachable" }; + } + } + + if (response.status === 404 || response.status === 410) { + return { status: "missing" }; + } + if (!response.ok) return { status: "unreachable" }; + + const contentType = response.headers.get("content-type") ?? ""; + const text = await readBoundedText(response); + // Oversized body (over the shared cap) — not a real llms.txt file. + if (text === null) return { status: "missing" }; + // SPA catch-alls answer 200 with the app shell for any path; an HTML + // page is not an llms.txt, however the server labels it. + if (contentType.includes("text/html") || looksLikeHtml(text)) { + return { status: "missing" }; + } + return { status: "found", text }; + } catch (error) { + console.warn("Failed to fetch llms.txt:", error); + return { status: "unreachable" }; + } +} + +function looksLikeHtml(text: string): boolean { + return /^\s*( line.trim() !== ""); + if (firstContentLine === undefined) return "file-empty"; + if (!/^#\s+\S/.test(firstContentLine.trim())) return "missing-h1-title"; + return null; +} + +export function checkLlmsTxt( + origin: string, + fetchResult: LlmsTxtFetchResult, +): DetectedIssue[] { + // Unreachable is a transient/network condition, not evidence about the + // site — stay silent rather than mis-report a file that may exist. + if (fetchResult.status === "unreachable") return []; + + const pageUrl = `${origin}/llms.txt`; + if (fetchResult.status === "missing") { + return [{ issueType: "missing-llms-txt", pageId: null, pageUrl }]; + } + + const problem = findLlmsTxtStructureProblem(fetchResult.text); + if (problem === null) return []; + return [ + { + issueType: "llms-txt-invalid", + pageId: null, + pageUrl, + details: { problem }, + }, + ]; +} + +export async function runAiReadinessChecks(input: { + origin: string; + robotsText: string | null; +}): Promise { + const llmsTxt = await fetchLlmsTxt(input.origin); + return [ + ...checkAiCrawlerAccess(input.origin, input.robotsText), + ...checkLlmsTxt(input.origin, llmsTxt), + ]; +} diff --git a/src/server/lib/scrape.ts b/src/server/lib/scrape.ts index 7a8da8f14..ce11d6018 100644 --- a/src/server/lib/scrape.ts +++ b/src/server/lib/scrape.ts @@ -27,7 +27,11 @@ type SiteReadResult = { // Bounded read: accumulate up to MAX_RESPONSE_BYTES regardless of whether // content-length is present (chunked / CDN responses often omit it). -async function readBoundedText(response: Response): Promise { +// Exported so other fetchers (e.g. the llms.txt check) reuse the same cap +// instead of buffering an unbounded third-party body into memory. +export async function readBoundedText( + response: Response, +): Promise { const reader = response.body?.getReader(); if (!reader) return null; const decoder = new TextDecoder(); diff --git a/src/server/workflows/siteAuditWorkflowPhases.ts b/src/server/workflows/siteAuditWorkflowPhases.ts index f0fd0a7c2..cd7baa602 100644 --- a/src/server/workflows/siteAuditWorkflowPhases.ts +++ b/src/server/workflows/siteAuditWorkflowPhases.ts @@ -16,6 +16,7 @@ import { AuditRepository } from "@/server/features/audit/repositories/AuditRepos import { getAuditScratchpad } from "@/server/features/audit/AuditScratchpad"; import { AuditProgressKV } from "@/server/lib/audit/progress-kv"; import { runMultipageChecks } from "@/server/lib/audit/issues/multipage"; +import { runAiReadinessChecks } from "@/server/lib/audit/issues/ai-readiness"; import type { DetectedIssue } from "@/server/lib/audit/issues/page-reporters"; import type { AuditConfig } from "@/server/lib/audit/types"; import { captureServerEvent } from "@/server/lib/posthog"; @@ -100,6 +101,7 @@ export async function runAuditPhases( startUrl, config, crawl, + robotsText: discovery.robotsText, }); } @@ -324,6 +326,7 @@ async function finalizeAudit(args: { startUrl: string; config: AuditConfig; crawl: CrawlPhaseResult; + robotsText: string | null; }) { const { step, @@ -334,6 +337,7 @@ async function finalizeAudit(args: { startUrl, config, crawl, + robotsText, } = args; await pgStep(step, "multipage-checks", MULTIPAGE_CHECKS_STEP, async () => { @@ -355,6 +359,12 @@ async function finalizeAudit(args: { const issues = await runMultipageChecks({ auditId }); issues.push(...(await runScratchpadLinkChecks(auditId, startUrl, crawl))); + issues.push( + ...(await runAiReadinessChecks({ + origin: getOrigin(startUrl), + robotsText, + })), + ); await AuditRepository.insertIssues(auditId, issues); return { issueCount: issues.length }; }); diff --git a/src/shared/audit-issues.ts b/src/shared/audit-issues.ts index baddfa7f0..490c97f14 100644 --- a/src/shared/audit-issues.ts +++ b/src/shared/audit-issues.ts @@ -232,6 +232,46 @@ export const AUDIT_ISSUE_TYPES = { howToFix: "Add links from higher-level pages (hubs, category pages, navigation) to flatten the path to this page.", }, + "ai-search-crawlers-blocked": { + severity: "warning", + title: "AI search crawlers blocked", + explanation: + "robots.txt blocks crawlers that build the indexes AI answers cite (e.g. OAI-SearchBot for ChatGPT search, PerplexityBot). Pages these crawlers cannot see cannot be cited in AI search results — an increasingly important discovery channel that is separate from classic search rankings.", + howToFix: + "Allow the blocked user agents in robots.txt (they are listed in the issue details). If you intentionally opted out of AI search visibility, no action is needed.", + }, + "ai-user-fetchers-blocked": { + severity: "warning", + title: "AI assistant fetchers blocked", + explanation: + "robots.txt blocks user-request fetchers (e.g. ChatGPT-User, Claude-User) — the agents that retrieve a page live when someone asks an AI assistant about it. Blocking them means assistants cannot open your links or quote your content even when a user explicitly requests it.", + howToFix: + "Allow the blocked user agents in robots.txt (listed in the issue details). These fetchers act on direct user requests and are not used for model training.", + }, + "ai-training-crawlers-blocked": { + severity: "info", + title: "AI training crawlers blocked", + explanation: + "robots.txt blocks crawlers that collect model training data (e.g. GPTBot, CCBot). That is often an intentional content-policy choice — this is a heads-up, not an error. Note that Google-Extended also controls whether Gemini can ground answers in your pages, so blocking it can reduce AI answer visibility, not just training use.", + howToFix: + "If the block is intentional, no action is needed. If you want AI answer visibility, review the blocked agents in the issue details — especially Google-Extended — and allow the ones tied to answer grounding.", + }, + "missing-llms-txt": { + severity: "info", + title: "No llms.txt file", + explanation: + "The site has no /llms.txt. It is an emerging convention (llmstxt.org) that gives AI assistants a curated markdown map of your most important pages, improving how accurately they summarize and cite the site. Absence is common and not an error.", + howToFix: + "Consider adding a /llms.txt: a markdown file starting with an H1 site name, an optional blockquote summary, and sections of links to key pages. See llmstxt.org for the format.", + }, + "llms-txt-invalid": { + severity: "warning", + title: "llms.txt does not follow the expected format", + explanation: + "The site serves /llms.txt, but the file does not start with an H1 title as the llms.txt format requires, so AI tools that parse the file may ignore it entirely — the effort of providing it is lost.", + howToFix: + 'Make the first line of /llms.txt an H1 with the site name (e.g. "# Example Corp"), optionally followed by a blockquote summary and markdown sections of links. See llmstxt.org for the full format.', + }, } as const satisfies Record; export type AuditIssueType = keyof typeof AUDIT_ISSUE_TYPES;