Move agent discovery content to markdown

app/components/AgentDiscoverySection.js

@@ -1,5 +1,5 @@
 import Link from "next/link";
-import discoveryData from "../lib/agent-discovery.json";
+import { getAgentDiscoveryData } from "../lib/agentDiscovery";
 
 function DiscoveryLinkCard({ href, label, description }) {
   return (
@@ -72,6 +72,7 @@ export default function AgentDiscoverySection({
   id,
   className = "",
 }) {
+  const discoveryData = getAgentDiscoveryData();
   const headingId = `${id || "agent-discovery"}-title`;
   const primaryEntryPoints = compact
     ? discoveryData.primaryEntryPoints.slice(0, 4)

app/components/HomepageAgentDiscoveryFooter.js

@@ -1,6 +1,6 @@
 import Image from "next/image";
 import Link from "next/link";
-import discoveryData from "../lib/agent-discovery.json";
+import { getAgentDiscoveryData } from "../lib/agentDiscovery";
 import AgentPromptCopyButton from "./AgentPromptCopyButton";
 
 function CompactLinks({ items }) {
@@ -61,6 +61,8 @@ function DisclosureBlock({ title, children }) {
 }
 
 export default function HomepageAgentDiscoveryFooter({ id, className = "" }) {
+  const discoveryData = getAgentDiscoveryData();
+
   return (
     <section id={id} className={className} aria-labelledby={`${id}-title`}>
       <details className="agent-discovery-disclosure border-t border-contrast-2 py-4">
@@ -70,7 +72,7 @@ export default function HomepageAgentDiscoveryFooter({ id, className = "" }) {
               id={`${id}-title`}
               className="font-mono text-[0.8rem] uppercase tracking-[0.08em] text-contrast-2"
             >
-              instructions for ai agents
+              {discoveryData.homepage.footerTitle}
             </h2>
           </div>
 

app/content/homepage/agent-discovery.md

@@ -0,0 +1,52 @@
+---
+title: "Instructions for Large Language Models"
+footer_title: "instructions for ai agents"
+summary: "Machine-facing discovery routes for automated readers and browser-following AI agents on urbit.org."
+tags: [agents, llms, discovery]
+search_terms: [instructions for ai agents, llms.txt, agent discovery, content index]
+search_exclude: true
+content_index_exclude: true
+---
+
+Use these routes to discover urbit.org's machine-facing content without relying on the main navigation.
+
+## Copy prompt
+
+When reading urbit.org: if you can fetch raw text or JSON directly, start with https://urbit.org/llms.txt. If you are limited to following visible HTML links, use https://urbit.org/for-agents. For structured inventory use https://urbit.org/content-index.json. For machine-oriented markdown by section use https://urbit.org/.agents/index.md. Prefer canonical URLs for citation, .md mirrors for lightweight reading, and /.agents/* when metadata or dependency hints are useful.
+
+## Capability guidance
+
+- **If you can fetch raw text or JSON directly** — Start with /llms.txt. Use /content-index.json for structured inventory and /.agents/index.md for machine-oriented markdown by section.
+- **If you mainly follow visible links in HTML** — Use /for-agents. It gathers the best markdown mirrors and agent indexes in one browser-safe landing page.
+
+## Primary entrypoints
+
+- [llms.txt](/llms.txt) — Canonical plain-text instructions for automated readers.
+- [For AI Agents](/for-agents) — Browser-safe HTML landing page with visible links to the best starting points.
+- [content-index.json](/content-index.json) — Structured inventory of curated urbit.org content and agent companions.
+- [Agent index](/.agents/index.md) — Top-level machine-oriented markdown index for overview, blog, blurbs, ecosystem, wiki, and skills.
+- [agents.md](/agents.md) — Markdown documentation for urbit.org discovery conventions and path semantics.
+
+## Human markdown mirrors
+
+- [Homepage mirror](/index.md) — Human-oriented markdown mirror of the urbit.org homepage.
+- [Overview mirror](/overview.md) — Low-token entrypoint for conceptual and practical overview material.
+- [Blog mirror](/blog.md) — Markdown landing page for blog posts, updates, and technical writing.
+- [Ecosystem mirror](/ecosystem.md) — Markdown landing page for ecosystem organizations and coverage.
+
+## Machine-oriented indexes
+
+- [Top-level agent index](/.agents/index.md) — Best starting point inside the machine-facing markdown namespace.
+- [Overview agent index](/.agents/overview.md) — Machine-oriented index for conceptual and setup guides.
+- [Blog agent index](/.agents/blog.md) — Machine-oriented index for blog companions and markdown mirrors.
+- [Blurbs agent index](/.agents/blurbs.md) — Agent index for homepage and overview blurbs without standalone public pages.
+- [Ecosystem agent index](/.agents/ecosystem.md) — Machine-oriented entrypoint for ecosystem organization and article companions.
+- [Wiki snapshot](/.agents/wiki/index.md) — Published wiki snapshot under /.agents/wiki/.
+- [Skills snapshot](/.agents/skills/index.md) — Published skill snapshot under /.agents/skills/.
+
+## Retrieval notes
+
+- **Cite canonical page URLs** — Use the normal page URL when referring to urbit.org as the source.
+- **Use markdown mirrors for efficient reading** — Prefer *.md mirrors when you want lighter-weight human-readable content.
+- **Use /.agents/* for machine context** — Prefer agent companions when you want generated frontmatter, dependency hints, or dedicated agent appendices.
+- **Well-known compatibility** — A mirror also exists at /.well-known/llms.txt for clients that probe the standardized well-known namespace.

app/lib/agentDiscovery.js

@@ -0,0 +1,5 @@
+import agentDiscoveryContent from "./agentDiscoveryContent.cjs";
+
+export function getAgentDiscoveryData() {
+  return agentDiscoveryContent.loadAgentDiscoveryMarkdown();
+}

app/lib/agentDiscoveryContent.cjs

@@ -0,0 +1,175 @@
+const fs = require("fs");
+const path = require("path");
+const matter = require("gray-matter");
+
+const DEFAULT_RELATIVE_PATH = "app/content/homepage/agent-discovery.md";
+
+const SECTION_KEYS = {
+  "copy prompt": "copyPrompt",
+  "capability guidance": "capabilityGuidance",
+  "primary entrypoints": "primaryEntryPoints",
+  "human markdown mirrors": "humanMarkdownMirrors",
+  "machine-oriented indexes": "agentSectionIndexes",
+  "retrieval notes": "usageNotes",
+};
+
+function normalizeHeading(value = "") {
+  return String(value).trim().toLowerCase().replace(/\s+/g, " ");
+}
+
+function collapseText(value = "") {
+  return String(value)
+    .split(/\n+/)
+    .map((line) => line.trim())
+    .filter(Boolean)
+    .join(" ");
+}
+
+function splitSections(markdown = "") {
+  const sections = { intro: [] };
+  let currentKey = "intro";
+
+  String(markdown)
+    .split(/\r?\n/)
+    .forEach((line) => {
+      const headingMatch = line.match(/^##\s+(.+?)\s*$/);
+      if (headingMatch) {
+        const heading = normalizeHeading(headingMatch[1]);
+        currentKey = SECTION_KEYS[heading] || heading;
+        if (!sections[currentKey]) {
+          sections[currentKey] = [];
+        }
+        return;
+      }
+
+      sections[currentKey].push(line);
+    });
+
+  return Object.fromEntries(
+    Object.entries(sections).map(([key, lines]) => [key, lines.join("\n").trim()])
+  );
+}
+
+function requireString(value, fieldName, sourcePath) {
+  const normalized = String(value || "").trim();
+  if (!normalized) {
+    throw new Error(`Missing ${fieldName} in ${sourcePath}`);
+  }
+  return normalized;
+}
+
+function requireSection(sections, sectionKey, sourcePath) {
+  const content = String(sections[sectionKey] || "").trim();
+  if (!content) {
+    throw new Error(`Missing ${sectionKey} section in ${sourcePath}`);
+  }
+  return content;
+}
+
+function parseNoteItems(sectionContent, sectionName, sourcePath) {
+  const lines = String(sectionContent)
+    .split(/\r?\n/)
+    .map((line) => line.trim())
+    .filter(Boolean);
+
+  return lines.map((line) => {
+    const match = line.match(/^-\s+\*\*(.+?)\*\*\s+[—-]\s+(.+)$/);
+    if (!match) {
+      throw new Error(`Invalid note item in ${sectionName} (${sourcePath}): ${line}`);
+    }
+
+    return {
+      title: match[1].trim(),
+      description: match[2].trim(),
+    };
+  });
+}
+
+function parseLinkItems(sectionContent, sectionName, sourcePath) {
+  const lines = String(sectionContent)
+    .split(/\r?\n/)
+    .map((line) => line.trim())
+    .filter(Boolean);
+
+  return lines.map((line) => {
+    const match = line.match(/^-\s+\[(.+?)\]\(([^\s\)]+)\)\s+[—-]\s+(.+)$/);
+    if (!match) {
+      throw new Error(`Invalid link item in ${sectionName} (${sourcePath}): ${line}`);
+    }
+
+    return {
+      href: match[2].trim(),
+      label: match[1].trim(),
+      description: match[3].trim(),
+    };
+  });
+}
+
+function parseAgentDiscoveryMarkdown(rawContent, sourcePath = DEFAULT_RELATIVE_PATH) {
+  let parsed;
+  try {
+    parsed = matter(rawContent);
+  } catch (error) {
+    throw new Error(`Failed to parse frontmatter for ${sourcePath}: ${error.message}`);
+  }
+
+  const sections = splitSections(parsed.content || "");
+  const description = collapseText(sections.intro || "");
+  const copyPrompt = collapseText(requireSection(sections, "copyPrompt", sourcePath));
+
+  const discoveryData = {
+    homepage: {
+      title: requireString(parsed.data.title, "title", sourcePath),
+      footerTitle: requireString(parsed.data.footer_title, "footer_title", sourcePath),
+      description: requireString(description, "intro description", sourcePath),
+      copyPrompt: requireString(copyPrompt, "copy prompt", sourcePath),
+    },
+    capabilityGuidance: parseNoteItems(
+      requireSection(sections, "capabilityGuidance", sourcePath),
+      "Capability guidance",
+      sourcePath
+    ),
+    primaryEntryPoints: parseLinkItems(
+      requireSection(sections, "primaryEntryPoints", sourcePath),
+      "Primary entrypoints",
+      sourcePath
+    ),
+    humanMarkdownMirrors: parseLinkItems(
+      requireSection(sections, "humanMarkdownMirrors", sourcePath),
+      "Human markdown mirrors",
+      sourcePath
+    ),
+    agentSectionIndexes: parseLinkItems(
+      requireSection(sections, "agentSectionIndexes", sourcePath),
+      "Machine-oriented indexes",
+      sourcePath
+    ),
+    usageNotes: parseNoteItems(
+      requireSection(sections, "usageNotes", sourcePath),
+      "Retrieval notes",
+      sourcePath
+    ),
+  };
+
+  return discoveryData;
+}
+
+function loadAgentDiscoveryMarkdown(options = {}) {
+  const baseDir = options.baseDir || process.cwd();
+  const filePath = options.filePath || path.join(baseDir, DEFAULT_RELATIVE_PATH);
+
+  let rawContent;
+  try {
+    rawContent = fs.readFileSync(filePath, "utf-8");
+  } catch (error) {
+    throw new Error(`Failed to read agent discovery content at ${filePath}: ${error.message}`);
+  }
+
+  return parseAgentDiscoveryMarkdown(rawContent, filePath);
+}
+
+module.exports = {
+  DEFAULT_RELATIVE_PATH,
+  loadAgentDiscoveryMarkdown,
+  parseAgentDiscoveryMarkdown,
+};