Skip to content

Commit 78fe118

Browse files
imran-siddiqueclaude
imran-siddique
and
claude
authored
docs: page descriptions, titles, structured data and crawler files for search and answer engines (#116)
* docs: page descriptions, titles, structured data and crawler files for search and answer engines A crawl of tests.agentrust-io.com on 2026-09-14 found 21 of 22 pages sharing the site description, JSON-LD with no Organization @id and an inline Organization on every page, no robots.txt directives or sitemap line, and llms.txt sections that predate the five tabs. hooks/seo.py gives each page its own description; overrides/main.html adds the title rule, per-page JSON-LD and breadcrumbs under the hub organization; robots.txt is added and copied by the docs workflow; llms.txt follows the tabs and now lists the limitations and self-verification pages. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_013aK3gVWzNdcM3hZ2o2awK2 * docs: update the description hook to the shared kit v2 Indented continuation lines of a skipped list item are no longer read as prose, so CHANGELOG's description is no longer the wrapped second line of its first bullet, and double quotes in a derived description become apostrophes. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_013aK3gVWzNdcM3hZ2o2awK2 --------- Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
1 parent b534879 commit 78fe118

6 files changed

Lines changed: 243 additions & 45 deletions

File tree

‎.github/workflows/docs.yml‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,8 @@ on:
1414
- "measurement/**"
1515
- "PRIVACY.md"
1616
- "overrides/**"
17+
- "hooks/**"
18+
- "robots.txt"
1719
- "index.md"
1820
- "CODE_OF_CONDUCT.md"
1921
- "LIMITATIONS.md"
@@ -61,7 +63,7 @@ jobs:
6163
# measurement/REPORT.md is in the nav as Self-verification.
6264
if [ -d measurement ]; then cp -r measurement $BUILD/measurement; fi
6365
64-
for fname in index.md CHANGELOG.md CONTRIBUTING.md CODE_OF_CONDUCT.md LIMITATIONS.md SPONSORS.md PRIVACY.md CNAME; do
66+
for fname in index.md CHANGELOG.md CONTRIBUTING.md CODE_OF_CONDUCT.md LIMITATIONS.md SPONSORS.md PRIVACY.md CNAME robots.txt; do
6567
if [ -f "$fname" ]; then cp "$fname" "$BUILD/$fname"; fi
6668
done
6769

‎hooks/seo.py‎

Lines changed: 102 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,102 @@
1+
"""Give every page its own meta description.
2+
3+
Material falls back to site_description when a page has none in its front
4+
matter, so every inner page showed search engines and answer engines the same
5+
snippet. This hook takes the first paragraph of prose on the page instead.
6+
Front matter still wins; add a `description:` there to write one by hand.
7+
"""
8+
import re
9+
10+
LIMIT = 155
11+
MINIMUM = 50
12+
13+
# Lines that are not prose: headings, admonitions, HTML, tables, quotes, lists,
14+
# attribute lists, rules, snippet includes, and the chain label that opens
15+
# landing pages ("[01 \u00b7 Weights: ...](https://agentrust-io.com/#chain)").
16+
_NOT_PROSE = re.compile(r'^(#|!!!|\?\?\?|<|\||>|[-*+] |\d+\. |\{|:::|---|\*\*\*|--8<--|\[\d\d \u00b7 )')
17+
18+
# Front-of-page metadata such as "**Status**: Accepted" or "Last updated: 2026-08-01".
19+
_METADATA = re.compile(
20+
r'^(\*\*[^*]+\*\*\s*:|\*\*[^*]+:\*\*|'
21+
'(Status|Date|Last updated|Updated|Stability|Document status|Applies to|Written|'
22+
'Authors?|Contact|Owner|Organisation|Organization|Version|Scope|Target|'
23+
r'Spec section|Related issues|Supersedes|Superseded by)\s*:)',
24+
re.IGNORECASE,
25+
)
26+
27+
28+
def _plain(text):
29+
text = re.sub(r'!\[[^\]]*\]\([^)]*\)', '', text)
30+
text = re.sub(r'\[([^\]]+)\]\([^)]*\)', r'\1', text)
31+
text = re.sub(r'\[([^\]]+)\]\[[^\]]*\]', r'\1', text)
32+
text = re.sub(r'\{\s*[:.#][^}]*\}', '', text)
33+
text = re.sub(r'`([^`]*)`', r'\1', text)
34+
text = re.sub(r'(\*\*|__)(.+?)\1', r'\2', text)
35+
text = re.sub(r'(?<![\w*])\*(?!\s)(.+?)(?<!\s)\*(?![\w*])', r'\1', text)
36+
text = re.sub(r'<[^>]+>', '', text)
37+
# House style has no em or en dashes; source text sometimes does.
38+
text = re.sub(r'\s*\u2014\s*', ', ', text).replace('\u2013', ' to ')
39+
# Material writes the description into content="..." without escaping it.
40+
text = text.replace('"', "'").replace('\u201c', "'").replace('\u201d', "'")
41+
text = re.sub(r'\s+', ' ', text).strip()
42+
return re.sub(r'\s+,', ',', text)
43+
44+
45+
def first_paragraph(markdown):
46+
fence = None
47+
lines = []
48+
skipping = False
49+
for raw in markdown.splitlines():
50+
line = raw.strip()
51+
if fence:
52+
if line.startswith(fence):
53+
fence = None
54+
continue
55+
if line.startswith(('```', '~~~')):
56+
fence = line[:3]
57+
if lines:
58+
break
59+
continue
60+
if not line:
61+
if lines:
62+
break
63+
skipping = False
64+
continue
65+
# Indented lines before any prose are admonition bodies; after a skipped
66+
# list item or metadata line they are its wrapped continuation.
67+
if raw[:1] in (' ', '\t') and (not lines or skipping):
68+
continue
69+
if _NOT_PROSE.match(line) or _METADATA.match(line):
70+
if lines:
71+
break
72+
skipping = True
73+
continue
74+
if skipping and not lines:
75+
continue
76+
lines.append(line)
77+
return _plain(' '.join(lines))
78+
79+
80+
def cap(text, limit=LIMIT):
81+
if len(text) <= limit:
82+
return text
83+
cut = text[:limit + 1].rsplit(' ', 1)[0].rstrip(',;:')
84+
end = cut.rfind('. ')
85+
if end >= MINIMUM:
86+
return cut[:end + 1]
87+
# No sentence end in range: cut short enough that the ellipsis fits the limit.
88+
cut = text[:limit - 2].rsplit(' ', 1)[0].rstrip(',;:.')
89+
return cut + '...'
90+
91+
92+
def on_page_markdown(markdown, page, config, files):
93+
if page.meta.get('description'):
94+
# Hand-written descriptions reach the same unescaped attribute.
95+
page.meta['description'] = _plain(str(page.meta['description']))
96+
return markdown
97+
text = first_paragraph(markdown)
98+
if len(text) >= MINIMUM:
99+
page.meta['description'] = cap(text)
100+
elif page.title:
101+
page.meta['description'] = cap(_plain(f'{page.title}. {config["site_description"]}'))
102+
return markdown

‎index.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@ hide:
33
- navigation
44
- toc
55
title: TRACE conformance suite
6-
description: Run this suite against a TRACE record to see which conformance level it reaches, and produce a report anyone can reproduce from the record and the suite version.
6+
description: Run this suite against a TRACE record to see which conformance level it reaches, with a report anyone can reproduce from the record and suite version.
77
---
88

99
[04 · Evidence: can a third party verify all of it offline, years later?](https://agentrust-io.com/#chain)

‎mkdocs.yml‎

Lines changed: 16 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,8 @@ copyright: "© 2026 AgenTrust contributors. Apache 2.0"
99

1010
exclude_docs: |
1111
.github/
12+
hooks/
13+
overrides/
1214
node_modules/
1315
src/
1416
tests/
@@ -59,6 +61,10 @@ theme:
5961
icon:
6062
repo: fontawesome/brands/github
6163

64+
# Gives each page its own meta description; see hooks/seo.py.
65+
hooks:
66+
- hooks/seo.py
67+
6268
plugins:
6369
- search:
6470
lang: en
@@ -70,13 +76,14 @@ plugins:
7076
portable, signed runtime evidence about AI agent runs. Use it to see
7177
which conformance level a TRACE record reaches and to produce a report
7278
anyone can reproduce. It checks the shape of attestation fields, not
73-
the attestation itself.
79+
the attestation itself. It is part of AgenTrust, open specifications
80+
for verifiable AI: https://agentrust-io.com
7481
sections:
75-
Getting started:
82+
Get started:
7683
- index.md
7784
- docs/quickstart.md
85+
Specification:
7886
- docs/levels.md
79-
Test modules:
8087
- docs/modules.md
8188
- docs/modules/tr-env.md
8289
- docs/modules/tr-sig.md
@@ -86,10 +93,15 @@ plugins:
8693
- docs/modules/tr-txn.md
8794
- docs/modules/tr-anc.md
8895
- docs/modules/tr-sca.md
89-
Reference:
9096
- docs/error-codes.md
97+
Guides:
9198
- docs/tutorials/writing-conformance-tests.md
9299
- docs/tutorials/ci-integration.md
100+
- measurement/REPORT.md
101+
Project:
102+
- LIMITATIONS.md
103+
- CHANGELOG.md
104+
- CONTRIBUTING.md
93105
- minify:
94106
minify_html: true
95107
- mkdocstrings:

‎overrides/main.html‎

Lines changed: 83 additions & 39 deletions
Original file line numberDiff line numberDiff line change
@@ -3,78 +3,122 @@
33
{#
44
SEO and AEO head additions for the TRACE conformance suite.
55

6-
1. The homepage <title> is overridden. Material falls back to site_name when a
7-
page has no front-matter title, which left the homepage titled "TRACE
8-
Tests". Front matter is not an option: docs_dir is the repository root, so
9-
README.md is also the GitHub landing page and YAML would render as noise
10-
there.
6+
Title: Material appends " - TRACE Tests" to every page title and uses the
7+
front-matter title on the home page. The home page now uses that title alone,
8+
and a title that already names the site gets no suffix. Open Graph and
9+
Twitter use the same title string. An earlier comment here described a
10+
homepage title override that did not exist; this block is that override.
1111

12-
2. Open Graph and Twitter meta, so links produce a card instead of a bare URL,
13-
plus JSON-LD so answer engines can model what this suite is.
12+
JSON-LD: one AgenTrust Organization node under the hub's @id, the WebSite,
13+
and a node for the page itself (WebPage on the home page, TechArticle
14+
elsewhere) carrying its own title, description and canonical URL. Inner pages
15+
add a BreadcrumbList; the home page adds the SoftwareApplication node for the
16+
agentrust-trace-tests package. Page descriptions come from hooks/seo.py when
17+
the front matter has none.
1418

1519
Asset paths derive from config.site_url. Do not hardcode them: assets are
1620
served under /docs/assets/ here because docs_dir is the repository root and
17-
CI copies docs/ into the build directory. That differs between sibling
18-
repositories, and a hardcoded path is how cMCP shipped a broken card.
21+
CI copies docs/ into the build directory.
1922
#}
2023

24+
{% block htmltitle %}
25+
{%- set t = page.meta.title if page and page.meta and page.meta.title else (page.title | striptags if page and page.title and not page.is_homepage else "") -%}
26+
{%- if not t %}
27+
<title>{{ config.site_name }}</title>
28+
{%- elif (page and page.is_homepage) or config.site_name in t %}
29+
<title>{{ t }}</title>
30+
{%- else %}
31+
<title>{{ t }} - {{ config.site_name }}</title>
32+
{%- endif %}
33+
{% endblock %}
34+
2135
{% block extrahead %}
2236
{{ super() }}
23-
{# page.title is the nav label ("Home"); page.meta.title is the front matter. #}
24-
{% set page_name = (page.meta.title if page and page.meta and page.meta.title else (page.title if page else None)) %}
25-
{% set page_desc = page.meta.description if page and page.meta and page.meta.description else config.site_description %}
26-
{% set page_url = page.canonical_url if page and page.canonical_url else config.site_url %}
27-
{% set og_image = config.site_url ~ 'docs/assets/og.png' %}
28-
{% set social_title = 'Verify your TRACE implementation' %}
37+
{%- set t = page.meta.title if page and page.meta and page.meta.title else (page.title | striptags if page and page.title and not page.is_homepage else "") -%}
38+
{%- set is_home = page and page.is_homepage -%}
39+
{%- set page_title = config.site_name if not t else (t if is_home or config.site_name in t else t ~ " - " ~ config.site_name) -%}
40+
{%- set page_desc = page.meta.description if page and page.meta and page.meta.description else config.site_description -%}
41+
{%- set page_url = page.canonical_url if page and page.canonical_url else config.site_url %}
2942

3043
<meta property="og:type" content="website">
31-
<meta property="og:site_name" content="{{ config.site_name }}">
32-
<meta property="og:title" content="{% if page_name %}{{ page_name }} - {{ config.site_name }}{% else %}{{ config.site_name }}: {{ social_title }}{% endif %}">
33-
<meta property="og:description" content="{{ page_desc }}">
44+
<meta property="og:site_name" content="AgenTrust">
45+
<meta property="og:title" content="{{ page_title | e }}">
46+
<meta property="og:description" content="{{ page_desc | e }}">
3447
<meta property="og:url" content="{{ page_url }}">
35-
<meta property="og:image" content="{{ og_image }}">
48+
<meta property="og:image" content="{{ config.site_url }}docs/assets/og.png">
3649
<meta property="og:image:width" content="1200">
3750
<meta property="og:image:height" content="630">
3851
<meta property="og:image:alt" content="TRACE Tests: the AgenTrust conformance suite">
3952
<meta property="og:locale" content="en_US">
40-
4153
<meta name="twitter:card" content="summary_large_image">
42-
<meta name="twitter:title" content="{% if page_name %}{{ page_name }} - {{ config.site_name }}{% else %}{{ config.site_name }}: {{ social_title }}{% endif %}">
43-
<meta name="twitter:description" content="{{ page_desc }}">
44-
<meta name="twitter:image" content="{{ og_image }}">
54+
<meta name="twitter:title" content="{{ page_title | e }}">
55+
<meta name="twitter:description" content="{{ page_desc | e }}">
56+
<meta name="twitter:image" content="{{ config.site_url }}docs/assets/og.png">
4557

4658
<script type="application/ld+json">
4759
{
4860
"@context": "https://schema.org",
4961
"@graph": [
50-
{
51-
"@type": "SoftwareApplication",
52-
"name": "TRACE Tests",
53-
"applicationCategory": "DeveloperApplication",
54-
"description": "Conformance tests and an integration harness for TRACE records. It reports which conformance level a record reaches and checks the shape of attestation fields, not the attestation itself.",
55-
"url": "{{ config.site_url }}",
56-
"offers": { "@type": "Offer", "price": "0", "priceCurrency": "USD" },
57-
"author": { "@type": "Organization", "name": "AgenTrust", "url": "https://agentrust-io.com" },
58-
"isPartOf": { "@type": "WebSite", "name": "{{ config.site_name }}", "url": "{{ config.site_url }}" }
59-
},
6062
{
6163
"@type": "Organization",
64+
"@id": "https://agentrust-io.com/#organization",
6265
"name": "AgenTrust",
63-
"url": "https://agentrust-io.com",
66+
"url": "https://agentrust-io.com/",
6467
"sameAs": [
6568
"https://github.com/agentrust-io",
66-
"https://agentrust-io.com/demos/",
6769
"https://trace.agentrust-io.com",
68-
"https://cmcp.agentrust-io.com",
6970
"https://manifest.agentrust-io.com",
70-
"https://ca2a.agentrust-io.com"
71+
"https://cmcp.agentrust-io.com",
72+
"https://ca2a.agentrust-io.com",
73+
"https://wcm.agentrust-io.com",
74+
"https://tests.agentrust-io.com",
75+
"https://governance.agentrust-io.com"
7176
]
7277
},
7378
{
7479
"@type": "WebSite",
75-
"name": "{{ config.site_name }}",
76-
"url": "{{ config.site_url }}"
80+
"@id": {{ (config.site_url ~ "#website") | tojson }},
81+
"name": {{ config.site_name | tojson }},
82+
"url": {{ config.site_url | tojson }},
83+
"description": {{ config.site_description | tojson }},
84+
"publisher": { "@id": "https://agentrust-io.com/#organization" }
85+
},
86+
{
87+
"@type": {{ ("WebPage" if is_home else "TechArticle") | tojson }},
88+
"@id": {{ (page_url ~ "#page") | tojson }},
89+
"url": {{ page_url | tojson }},
90+
"name": {{ page_title | tojson }},
91+
"headline": {{ page_title | tojson }},
92+
"description": {{ page_desc | tojson }},
93+
"inLanguage": "en",
94+
"isPartOf": { "@id": {{ (config.site_url ~ "#website") | tojson }} },
95+
"publisher": { "@id": "https://agentrust-io.com/#organization" }
96+
}
97+
{%- if is_home %},
98+
{
99+
"@type": "SoftwareApplication",
100+
"name": "TRACE Tests",
101+
"alternateName": "agentrust-trace-tests",
102+
"description": "Conformance tests and an integration harness for TRACE records. It reports which conformance level a record reaches and checks the shape of attestation fields, not the attestation itself.",
103+
"applicationCategory": "DeveloperApplication",
104+
"operatingSystem": "Cross-platform",
105+
"url": {{ config.site_url | tojson }},
106+
"downloadUrl": "https://pypi.org/project/agentrust-trace-tests/",
107+
"codeRepository": {{ config.repo_url | tojson }},
108+
"license": "https://www.apache.org/licenses/LICENSE-2.0",
109+
"author": { "@id": "https://agentrust-io.com/#organization" },
110+
"publisher": { "@id": "https://agentrust-io.com/#organization" },
111+
"offers": { "@type": "Offer", "price": "0", "priceCurrency": "USD" }
112+
}
113+
{%- elif page and page.canonical_url %},
114+
{
115+
"@type": "BreadcrumbList",
116+
"itemListElement": [
117+
{ "@type": "ListItem", "position": 1, "name": {{ config.site_name | tojson }}, "item": {{ config.site_url | tojson }} },
118+
{ "@type": "ListItem", "position": 2, "name": {{ (t or page_title) | tojson }}, "item": {{ page_url | tojson }} }
119+
]
77120
}
121+
{%- endif %}
78122
]
79123
}
80124
</script>

‎robots.txt‎

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
# tests.agentrust-io.com: search and answer-engine crawlers are welcome.
2+
3+
User-agent: *
4+
Allow: /
5+
# Cloudflare email-obfuscation endpoint: not a page
6+
Disallow: /cdn-cgi/
7+
8+
User-agent: GPTBot
9+
Allow: /
10+
11+
User-agent: OAI-SearchBot
12+
Allow: /
13+
14+
User-agent: ChatGPT-User
15+
Allow: /
16+
17+
User-agent: ClaudeBot
18+
Allow: /
19+
20+
User-agent: Claude-SearchBot
21+
Allow: /
22+
23+
User-agent: Claude-User
24+
Allow: /
25+
26+
User-agent: PerplexityBot
27+
Allow: /
28+
29+
User-agent: Perplexity-User
30+
Allow: /
31+
32+
User-agent: Google-Extended
33+
Allow: /
34+
35+
User-agent: CCBot
36+
Allow: /
37+
38+
Sitemap: https://tests.agentrust-io.com/sitemap.xml

0 commit comments

Comments
 (0)