SEO: server-render the listing, add sitemap, route SSR through the API layer

- Homepage now server-renders the first page of domains (real <a> links) from
  a shared catalog query layer (src/lib/catalog.ts) — was an empty client-only
  shell. Infinite scroll + search remain as progressive enhancement over the
  same data.
- Expose that query layer as a real endpoint: /api/domains.json. SSR and the
  client both load through it (one data path, a usable public API).
- @astrojs/sitemap -> sitemap-index covering all 5,045 pages; robots.txt points
  to it so crawlers reach every page without depending on JS.
- Narrow the worker's /api routing so static /api/* (domains.json) serves.

Author: RhysSullivan

astro.config.mjs

@@ -1,9 +1,15 @@
 import { defineConfig } from "astro/config";
 import react from "@astrojs/react";
+import sitemap from "@astrojs/sitemap";
 
 export default defineConfig({
   site: "https://integrations.sh",
-  integrations: [react()],
+  integrations: [
+    react(),
+    // Enumerate every static page (homepage, /d/*, /<kind>/<slug>/*) so crawlers
+    // don't depend on the client-rendered listing. Exclude JSON API routes.
+    sitemap({ filter: (page) => !page.includes("/api/") }),
+  ],
   build: {
     format: "directory",
   },

bun.lock

@@ -18,6 +18,7 @@
   },
   "dependencies": {
     "@astrojs/react": "^5.0.4",
+    "@astrojs/sitemap": "^3.7.3",
     "@executor-js/config": "^0.0.2",
     "@executor-js/plugin-graphql": "^0.0.2",
     "@executor-js/plugin-mcp": "^0.0.2",

package.json

@@ -0,0 +1,4 @@
+User-agent: *
+Allow: /
+
+Sitemap: https://integrations.sh/sitemap-index.xml

public/robots.txt

@@ -0,0 +1,59 @@
+/**
+ * Catalog query layer — the single data-loading path for the registry listing.
+ *
+ * Both the public JSON endpoint (`/api/domains.json`) and the server-rendered
+ * pages call these functions, so SSR goes through the same "API" everyone else
+ * uses (no ad-hoc grouping in templates). Built over the enriched index.
+ */
+import { index } from "./data.ts";
+import { faviconUrl } from "./favicon.ts";
+import type { Kind } from "./types.ts";
+
+export interface DomainSummary {
+  domain: string;
+  icon: string | null;
+  total: number;
+  formats: Partial<Record<Kind, number>>;
+  popularity: number;
+  description: string;
+}
+
+const KIND_ORDER: Kind[] = ["mcp", "openapi", "graphql", "cli"];
+
+const DOMAINS: DomainSummary[] = (() => {
+  const map = new Map<string, DomainSummary>();
+  for (const r of index) {
+    const d = r.domain || r.slug;
+    if (!d) continue;
+    let g = map.get(d);
+    if (!g) {
+      g = { domain: d, icon: faviconUrl(d), total: 0, formats: {}, popularity: 0, description: "" };
+      map.set(d, g);
+    }
+    g.total++;
+    g.formats[r.kind] = (g.formats[r.kind] ?? 0) + 1;
+    g.popularity = Math.max(g.popularity, r.popularity ?? 0);
+    if (!g.description && r.description) g.description = r.description.replace(/\s+/g, " ").slice(0, 110);
+  }
+  return [...map.values()].sort(
+    (a, b) => b.popularity - a.popularity || b.total - a.total || a.domain.localeCompare(b.domain),
+  );
+})();
+
+/** Formats present on a domain, in canonical order. */
+export const formatsOf = (d: DomainSummary): Kind[] => KIND_ORDER.filter((k) => d.formats[k]);
+
+/** All domains, popularity-sorted. The full list the JSON endpoint serves. */
+export function allDomains(): DomainSummary[] {
+  return DOMAINS;
+}
+
+/** A page of domains — what each server-rendered listing renders. */
+export function listDomains(opts?: { offset?: number; limit?: number }): DomainSummary[] {
+  const offset = opts?.offset ?? 0;
+  return opts?.limit != null ? DOMAINS.slice(offset, offset + opts.limit) : DOMAINS.slice(offset);
+}
+
+export function domainCount(): number {
+  return DOMAINS.length;
+}

src/lib/catalog.ts

@@ -0,0 +1,13 @@
+import type { APIRoute } from "astro";
+import { allDomains } from "~/lib/catalog.ts";
+
+// GET /api/domains.json — the registry's domains (popularity-sorted), the same
+// data the homepage and /browse pages render server-side. Prerendered at build.
+export const GET: APIRoute = () =>
+  new Response(JSON.stringify(allDomains()), {
+    headers: {
+      "content-type": "application/json; charset=utf-8",
+      "access-control-allow-origin": "*",
+      "cache-control": "public, max-age=3600",
+    },
+  });