docs: page descriptions, titles, structured data and crawler files for search and answer engines

.github/workflows/docs-check.yml

@@ -5,6 +5,7 @@ on:
       - 'docs/**'
       - 'overrides/**'
       - 'mkdocs.yml'
+      - 'hooks/**'
       - 'requirements-docs.txt'
       - 'scripts/check_docs_quickstart.py'
       - '.github/workflows/docs-check.yml'

.github/workflows/docs.yml

@@ -7,6 +7,7 @@ on:
       - "docs/**"
       - "schemas/**"
       - "mkdocs.yml"
+      - "hooks/**"
       - "requirements-docs.txt"
       - "README.md"
       - "CHANGELOG.md"

docs/quickstart.md

@@ -1,5 +1,5 @@
 ---
-description: cMCP quickstart. From zero to your first signed TRACE Claim in under 30 minutes using CMCP_DEV_MODE=1, no hardware TEE required. Install, write a Cedar policy and tool catalog, run the gateway, watch a policy block a call, then verify the signed claim.
+description: Run cMCP in dev mode with no hardware TEE, watch a Cedar policy block a tool call, and verify the signed TRACE claim, in under 30 minutes.
 ---
 
 # Quickstart - cMCP Runtime

robots.txt → docs/robots.txt

@@ -1,12 +1,10 @@
-# cMCP documentation - crawling policy
-# Search and answer engines, including AI crawlers, are welcome.
+# cmcp.agentrust-io.com: search and answer-engine crawlers are welcome.
 
 User-agent: *
 Allow: /
-# Cloudflare email-obfuscation endpoint: not real pages, only 404s to crawlers
+# Cloudflare email-obfuscation endpoint: not a page
 Disallow: /cdn-cgi/
 
-# AI and answer-engine crawlers (explicitly welcomed)
 User-agent: GPTBot
 Allow: /
 
@@ -19,15 +17,18 @@ Allow: /
 User-agent: ClaudeBot
 Allow: /
 
-User-agent: anthropic-ai
+User-agent: Claude-SearchBot
 Allow: /
 
-User-agent: Claude-Web
+User-agent: Claude-User
 Allow: /
 
 User-agent: PerplexityBot
 Allow: /
 
+User-agent: Perplexity-User
+Allow: /
+
 User-agent: Google-Extended
 Allow: /
 

docs/spec/failure-modes.md

@@ -1,4 +1,8 @@
-# Failure Mode Specification
+---
+description: "Exact cMCP runtime behavior for each failure: attestation at startup and mid-session, TEE faults, policy hash mismatch, and MCP parse errors."
+---
+
+# Failure Mode Specification
 
 ---
 Status: Draft v0.1

docs/spec/phase2-server.md

@@ -1,4 +1,8 @@
-# Phase 2 cMCP Server Specification
+---
+description: "Phase 2 measures the provider's MCP server binary, tool surface and egress inside a TEE, so a client can verify the server as well as the gateway."
+---
+
+# Phase 2 cMCP Server Specification
 
 ---
 Status: Draft v0.1

docs/spec/tool-identity.md

@@ -1,4 +1,8 @@
-# Tool Identity and Catalog Specification
+---
+description: "How cMCP identifies upstream MCP servers by TLS fingerprint or SPIFFE SVID, prevents tool name collisions, and routes each tool call."
+---
+
+# Tool Identity and Catalog Specification
 
 ---
 Status: Draft v0.1

docs/spec/transport.md

@@ -1,4 +1,8 @@
-# Transport Compatibility Specification
+---
+description: "The MCP transports cMCP supports: stateless Streamable HTTP, stdio servers run as gateway children, and why WebSocket catalog entries are rejected."
+---
+
+# Transport Compatibility Specification
 
 ---
 Status: Draft v0.1

hooks/seo.py

@@ -0,0 +1,102 @@
+"""Give every page its own meta description.
+
+Material falls back to site_description when a page has none in its front
+matter, so every inner page showed search engines and answer engines the same
+snippet. This hook takes the first paragraph of prose on the page instead.
+Front matter still wins; add a `description:` there to write one by hand.
+"""
+import re
+
+LIMIT = 155
+MINIMUM = 50
+
+# Lines that are not prose: headings, admonitions, HTML, tables, quotes, lists,
+# attribute lists, rules, snippet includes, and the chain label that opens
+# landing pages ("[01 \u00b7 Weights: ...](https://agentrust-io.com/#chain)").
+_NOT_PROSE = re.compile(r'^(#|!!!|\?\?\?|<|\||>|[-*+] |\d+\. |\{|:::|---|\*\*\*|--8<--|\[\d\d \u00b7 )')
+
+# Front-of-page metadata such as "**Status**: Accepted" or "Last updated: 2026-08-01".
+_METADATA = re.compile(
+    r'^(\*\*[^*]+\*\*\s*:|\*\*[^*]+:\*\*|'
+    '(Status|Date|Last updated|Updated|Stability|Document status|Applies to|Written|'
+    'Authors?|Contact|Owner|Organisation|Organization|Version|Scope|Target|'
+    r'Spec section|Related issues|Supersedes|Superseded by)\s*:)',
+    re.IGNORECASE,
+)
+
+
+def _plain(text):
+    text = re.sub(r'!\[[^\]]*\]\([^)]*\)', '', text)
+    text = re.sub(r'\[([^\]]+)\]\([^)]*\)', r'\1', text)
+    text = re.sub(r'\[([^\]]+)\]\[[^\]]*\]', r'\1', text)
+    text = re.sub(r'\{\s*[:.#][^}]*\}', '', text)
+    text = re.sub(r'`([^`]*)`', r'\1', text)
+    text = re.sub(r'(\*\*|__)(.+?)\1', r'\2', text)
+    text = re.sub(r'(?<![\w*])\*(?!\s)(.+?)(?<!\s)\*(?![\w*])', r'\1', text)
+    text = re.sub(r'<[^>]+>', '', text)
+    # House style has no em or en dashes; source text sometimes does.
+    text = re.sub(r'\s*\u2014\s*', ', ', text).replace('\u2013', ' to ')
+    # Material writes the description into content="..." without escaping it.
+    text = text.replace('"', "'").replace('\u201c', "'").replace('\u201d', "'")
+    text = re.sub(r'\s+', ' ', text).strip()
+    return re.sub(r'\s+,', ',', text)
+
+
+def first_paragraph(markdown):
+    fence = None
+    lines = []
+    skipping = False
+    for raw in markdown.splitlines():
+        line = raw.strip()
+        if fence:
+            if line.startswith(fence):
+                fence = None
+            continue
+        if line.startswith(('```', '~~~')):
+            fence = line[:3]
+            if lines:
+                break
+            continue
+        if not line:
+            if lines:
+                break
+            skipping = False
+            continue
+        # Indented lines before any prose are admonition bodies; after a skipped
+        # list item or metadata line they are its wrapped continuation.
+        if raw[:1] in (' ', '\t') and (not lines or skipping):
+            continue
+        if _NOT_PROSE.match(line) or _METADATA.match(line):
+            if lines:
+                break
+            skipping = True
+            continue
+        if skipping and not lines:
+            continue
+        lines.append(line)
+    return _plain(' '.join(lines))
+
+
+def cap(text, limit=LIMIT):
+    if len(text) <= limit:
+        return text
+    cut = text[:limit + 1].rsplit(' ', 1)[0].rstrip(',;:')
+    end = cut.rfind('. ')
+    if end >= MINIMUM:
+        return cut[:end + 1]
+    # No sentence end in range: cut short enough that the ellipsis fits the limit.
+    cut = text[:limit - 2].rsplit(' ', 1)[0].rstrip(',;:.')
+    return cut + '...'
+
+
+def on_page_markdown(markdown, page, config, files):
+    if page.meta.get('description'):
+        # Hand-written descriptions reach the same unescaped attribute.
+        page.meta['description'] = _plain(str(page.meta['description']))
+        return markdown
+    text = first_paragraph(markdown)
+    if len(text) >= MINIMUM:
+        page.meta['description'] = cap(text)
+    elif page.title:
+        page.meta['description'] = cap(_plain(f'{page.title}. {config["site_description"]}'))
+    return markdown

mkdocs.yml

@@ -59,6 +59,10 @@ theme:
   icon:
     repo: fontawesome/brands/github
 
+hooks:
+  # Per-page meta descriptions; see the module docstring.
+  - hooks/seo.py
+
 plugins:
   - search
   - llmstxt:
@@ -70,24 +74,38 @@ plugins:
         provenance. Hardware deployments require provider-specific attestation
         verification, trusted inputs, and a deployment that prevents gateway bypass.
         The agent, model, and upstream tool server remain separate components.
+        Part of AgenTrust, open specifications for verifiable AI: https://agentrust-io.com.
       sections:
-        Getting started:
+        Get started:
           - index.md
           - quickstart.md
           - concepts.md
           - configuration.md
+          - spec-index.md
         Specification:
           - SPEC.md
           - spec/cedar-policy.md
+          - spec/session-policy.md
+          - spec/tool-identity.md
+          - spec/transport.md
+          - spec/proxy-security.md
           - spec/attestation.md
-          - spec/threat-model.md
           - spec/verification-library.md
           - spec/embodied-action-evidence.md
-        Tutorials:
+          - spec/error-codes.md
+          - spec/threat-model.md
+        Guides:
+          - tutorials/existing-mcp-clients.md
           - tutorials/connecting-agent-frameworks.md
           - tutorials/cedar-policy-walkthrough.md
           - tutorials/verifying-a-trace-claim.md
           - tutorials/tee-attestation.md
+          - tutorials/deploy-azure.md
+          - tutorials/deploy-gcp.md
+          - testing/hardware-validation.md
+        Project:
+          - limitations.md
+          - sponsors.md
   - minify:
       minify_html: true
   - mkdocstrings:
@@ -188,7 +206,7 @@ nav:
           - TLS pinning: tutorials/tls-pinning.md
           - Verify a TRACE claim: tutorials/verifying-a-trace-claim.md
           - TEE attestation: tutorials/tee-attestation.md
-          - Response inspection: tutorials/response-inspection.md
+          - Response inspection tutorial: tutorials/response-inspection.md
           - AGT SRE kill switch: tutorials/kill-switch.md
       - Deployment:
           - Deploy on Azure: tutorials/deploy-azure.md