fix(privacy): apply noindex+mask policy to machine-readable outputs (#173)

apps/web/app/lib/csv-export.test.ts

@@ -1,6 +1,8 @@
-import { describe, expect, it, vi } from 'vitest';
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
 import { AUTHORITY_FILTER_KEYS, COMPANY_FILTER_KEYS, CONTRACT_FILTER_KEYS } from '@sigma/db';
+import { MASKED_NATURAL_PERSON_LABEL } from '@sigma/shared';
 import { DATA_SOURCE } from './dataSource';
+import * as security from './security';
 import { isUnfilteredCsvExport, servedCsvExport } from './csv-export';
 
 const REFRESHED_AT = '2026-06-13T10:00:00Z';
@@ -253,6 +255,8 @@ function csvBytesResponse(body: Uint8Array): Response {
   });
 }
 
+type CsvRoute = Parameters<typeof servedCsvExport>[0]['route'];
+
 function serve(
   r2: InMemoryR2,
   stream: () => Response,
@@ -261,12 +265,14 @@ function serve(
     params?: object;
     sort?: string;
     refreshedAt?: string | null | undefined;
+    route?: CsvRoute;
   } = {},
 ): Promise<Response> {
+  const route = opts.route ?? 'contracts';
   return servedCsvExport({
     env: envWith(r2, opts.refreshedAt),
-    request: opts.request ?? new Request('http://local/contracts.csv'),
-    route: 'contracts',
+    request: opts.request ?? new Request(`http://local/${route}.csv`),
+    route,
     params: opts.params ?? { sort: opts.sort ?? 'value-desc' },
     stream,
   });
@@ -504,3 +510,167 @@ describe('servedCsvExport', () => {
     expect((await response.arrayBuffer()).byteLength).toBe(largeBody.byteLength);
   });
 });
+
+describe('servedCsvExport privacy', () => {
+  it('stamps the privacy mask marker on a MISS response (contracts) and never writes X-Robots-Tag at the route layer', async () => {
+    const r2 = new InMemoryR2();
+    const stream = vi.fn(() => csvResponse());
+
+    const response = await serve(r2, stream, { route: 'contracts' });
+
+    expect(response.status).toBe(200);
+    expect(response.headers.get('X-Csv-Cache')).toBe('MISS');
+    expect(response.headers.get('X-Privacy-Mask')).toBe('applied');
+    expect(response.headers.get('X-Robots-Tag')).toBeNull();
+  });
+
+  it('stamps the privacy mask marker on a HIT response (companies) and never writes X-Robots-Tag at the route layer', async () => {
+    const r2 = new InMemoryR2();
+    const primeStream = vi.fn(() => csvResponse());
+    await (await serve(r2, primeStream, { route: 'companies' })).text();
+
+    const hitStream = vi.fn(() => csvResponse('from db\n'));
+    const response = await serve(r2, hitStream, { route: 'companies' });
+
+    expect(response.status).toBe(200);
+    expect(response.headers.get('X-Csv-Cache')).toBe('HIT');
+    expect(response.headers.get('X-Privacy-Mask')).toBe('applied');
+    expect(response.headers.get('X-Robots-Tag')).toBeNull();
+    expect(await response.text()).toBe(CSV_BODY);
+    expect(hitStream).not.toHaveBeenCalled();
+  });
+
+  it('stamps the privacy mask marker on a dynamic (filtered) response (authorities) and never writes X-Robots-Tag at the route layer', async () => {
+    const r2 = new InMemoryR2();
+    const stream = vi.fn(() => csvResponse('filtered\n'));
+
+    const response = await serve(r2, stream, {
+      route: 'authorities',
+      params: { sort: 'value-desc', q: 'foo' },
+    });
+
+    expect(response.status).toBe(200);
+    expect(response.headers.get('X-Csv-Cache')).toBe('dynamic');
+    expect(response.headers.get('X-Privacy-Mask')).toBe('applied');
+    expect(response.headers.get('X-Robots-Tag')).toBeNull();
+    expect(await response.text()).toBe('filtered\n');
+    expect(r2.createMultipartUpload).not.toHaveBeenCalled();
+  });
+
+  it('preserves the masking label and excludes the verbatim source name when the streamer emits masked bytes (contracts)', async () => {
+    const VERBATIM_NAME = 'ЕТ ДРИФТ - НИКОЛАЙ КИРОВ';
+    const maskedBody = `id,name,eik\nrow-1,${MASKED_NATURAL_PERSON_LABEL},\n`;
+    const r2 = new InMemoryR2();
+    const stream = vi.fn(() => csvResponse(maskedBody));
+
+    const response = await serve(r2, stream, { route: 'contracts' });
+
+    const text = await response.text();
+    expect(text).toContain(MASKED_NATURAL_PERSON_LABEL);
+    expect(text).not.toContain(VERBATIM_NAME);
+    expect(text).toBe(maskedBody);
+  });
+
+  // Locks in the blanket noindex policy documented in `privacy.tsx` and `docs/privacy-masking.md`:
+  // every CSV export carries `X-Privacy-Mask: applied` regardless of whether the body actually
+  // contains masked natural-person rows. The `authorities.csv` route is explicitly excluded from
+  // body masking (only ЕИК/name redaction is suppressed) but still gets the marker — CSV is a
+  // bulk machine-readable surface and the noindex signal is enforced blanket-wide.
+  it('stamps the privacy mask marker even when the CSV body contains zero masked rows (blanket CSV policy)', async () => {
+    const legalOnlyBody = 'eik,name\n121817309,СОФАРМА ТРЕЙДИНГ АД\n';
+    const r2 = new InMemoryR2();
+    for (const route of ['contracts', 'companies', 'authorities'] as const) {
+      const stream = vi.fn(() => csvResponse(legalOnlyBody));
+      const response = await serve(r2, stream, { route });
+      expect(response.headers.get('X-Privacy-Mask')).toBe('applied');
+      expect(response.headers.get('X-Robots-Tag')).toBeNull();
+    }
+  });
+
+  it('preserves Cache-Control: public, max-age=3600 when the streamer emits a row with masked sole-trader identifiers (contracts) and stamps the privacy mask marker', async () => {
+    const maskedBody = `eik,name\n,${MASKED_NATURAL_PERSON_LABEL}\n`;
+    const r2 = new InMemoryR2();
+    const stream = vi.fn(() => csvResponse(maskedBody));
+
+    const response = await serve(r2, stream, { route: 'contracts' });
+
+    expect(response.headers.get('Cache-Control')).toBe('public, max-age=3600');
+    expect(response.headers.get('X-Privacy-Mask')).toBe('applied');
+    expect(response.headers.get('X-Robots-Tag')).toBeNull();
+  });
+
+  // Guards against the duplicate-call smell flagged in PR #183 review (T-004): `markCsvCache` already
+  // invokes `markPrivacyMaskApplied` on the final headers, so an earlier call inside `responseFromR2Object`
+  // (or on the 304 branch) was redundant. The marker must be applied, but exactly once per response —
+  // a second call is dead code that hides the single source of truth (markCsvCache).
+  describe('marks the privacy mask exactly once per response (no duplicate calls)', () => {
+    let markSpy: ReturnType<typeof vi.spyOn>;
+
+    beforeEach(() => {
+      markSpy = vi.spyOn(security, 'markPrivacyMaskApplied');
+    });
+    afterEach(() => markSpy.mockRestore());
+
+    it('MISS (contracts): marks exactly once', async () => {
+      const r2 = new InMemoryR2();
+      await serve(
+        r2,
+        vi.fn(() => csvResponse()),
+        { route: 'contracts' },
+      );
+      expect(markSpy).toHaveBeenCalledTimes(1);
+    });
+
+    it('HIT (companies): marks exactly once', async () => {
+      const r2 = new InMemoryR2();
+      await (
+        await serve(
+          r2,
+          vi.fn(() => csvResponse()),
+          { route: 'companies' },
+        )
+      ).text();
+      markSpy.mockClear();
+      await serve(
+        r2,
+        vi.fn(() => csvResponse('hit\n')),
+        { route: 'companies' },
+      );
+      expect(markSpy).toHaveBeenCalledTimes(1);
+    });
+
+    it('dynamic (authorities, filtered): marks exactly once', async () => {
+      const r2 = new InMemoryR2();
+      await serve(
+        r2,
+        vi.fn(() => csvResponse('filtered\n')),
+        {
+          route: 'authorities',
+          params: { sort: 'value-desc', q: 'foo' },
+        },
+      );
+      expect(markSpy).toHaveBeenCalledTimes(1);
+    });
+
+    it('304 (conditional GET on a primed object): marks exactly once', async () => {
+      const r2 = new InMemoryR2();
+      const primed = await serve(
+        r2,
+        vi.fn(() => csvResponse()),
+      );
+      const etag = primed.headers.get('ETag');
+      await primed.text();
+      markSpy.mockClear();
+      await serve(
+        r2,
+        vi.fn(() => csvResponse('hit\n')),
+        {
+          request: new Request('http://local/contracts.csv', {
+            headers: { 'If-None-Match': etag ?? '' },
+          }),
+        },
+      );
+      expect(markSpy).toHaveBeenCalledTimes(1);
+    });
+  });
+});

apps/web/app/lib/csv-export.ts

@@ -1,5 +1,6 @@
 import { withDataSource } from './dataSource';
 import { getDb } from '@sigma/db';
+import { markPrivacyMaskApplied } from './security';
 
 const CSV_CONTENT_TYPE = 'text/csv; charset=utf-8';
 const CSV_CACHE_CONTROL = 'public, max-age=3600';
@@ -56,6 +57,7 @@ function hasSearchFilter(params: object): boolean {
 function markCsvCache(response: Response, cache: CsvCacheState): Response {
   const withSource = withDataSource(response);
   withSource.headers.set('X-Csv-Cache', cache);
+  markPrivacyMaskApplied(withSource.headers);
   return withSource;
 }
 
@@ -94,10 +96,13 @@ function responseFromR2Object(
   cache: CsvCacheState,
 ) {
   if (!hasBody(obj)) {
-    return markCsvCache(
-      new Response(null, { status: 304, headers: { ETag: obj.httpEtag } }),
-      cache,
-    );
+    const response304 = new Response(null, {
+      status: 304,
+      headers: { ETag: obj.httpEtag },
+    });
+    // `markCsvCache` (below) stamps the privacy mask on the final headers — the single source of
+    // truth for the marker on every CSV path. Do not call `markPrivacyMaskApplied` here as well.
+    return markCsvCache(response304, cache);
   }
 
   const range = rangeInfo(obj);
@@ -111,6 +116,8 @@ function responseFromR2Object(
   });
   if (range) headers.set('Content-Range', `bytes ${range.start}-${range.end}/${obj.size}`);
 
+  // `markCsvCache` stamps the privacy mask on the final headers — the single source of truth for the
+  // marker on every CSV path. Do not call `markPrivacyMaskApplied` here as well (was a duplicate call).
   return markCsvCache(new Response(obj.body, { status: range ? 206 : 200, headers }), cache);
 }
 

apps/web/app/lib/security.test.ts

@@ -1,5 +1,13 @@
 import { describe, expect, it } from 'vitest';
-import { baseSecurityHeaders, nonceLessSecurityHeaders, securityHeaders } from './security';
+import {
+  PRIVACY_MASK_APPLIED,
+  PRIVACY_MASK_MARKER,
+  applyPrivacyMaskHeaders,
+  baseSecurityHeaders,
+  markPrivacyMaskApplied,
+  nonceLessSecurityHeaders,
+  securityHeaders,
+} from './security';
 
 describe('securityHeaders CSP', () => {
   it('emits a strict nonce script-src and the documented style-src', () => {
@@ -27,3 +35,52 @@ describe('securityHeaders CSP', () => {
     expect(baseSecurityHeaders(true).get('Strict-Transport-Security')).toContain('max-age=');
   });
 });
+
+describe('privacy mask headers', () => {
+  it('sets the marker to PRIVACY_MASK_APPLIED on the provided Headers after markPrivacyMaskApplied', () => {
+    const headers = new Headers();
+    markPrivacyMaskApplied(headers);
+    expect(headers.get(PRIVACY_MASK_MARKER)).toBe(PRIVACY_MASK_APPLIED);
+  });
+
+  it('translates the marker to X-Robots-Tag: noindex and deletes the marker', () => {
+    const headers = new Headers();
+    markPrivacyMaskApplied(headers);
+    applyPrivacyMaskHeaders(headers);
+    expect(headers.get('X-Robots-Tag')).toBe('noindex');
+    expect(headers.has(PRIVACY_MASK_MARKER)).toBe(false);
+  });
+
+  it('adds no X-Robots-Tag and leaves no marker when the marker is absent', () => {
+    const headers = new Headers({ 'Cache-Control': 'public, max-age=3600' });
+    applyPrivacyMaskHeaders(headers);
+    expect(headers.has('X-Robots-Tag')).toBe(false);
+    expect(headers.has(PRIVACY_MASK_MARKER)).toBe(false);
+    // Untouched headers survive the call.
+    expect(headers.get('Cache-Control')).toBe('public, max-age=3600');
+  });
+
+  it('is idempotent on a second call — no re-set, no marker, X-Robots-Tag left intact', () => {
+    const headers = new Headers();
+    markPrivacyMaskApplied(headers);
+    applyPrivacyMaskHeaders(headers);
+    // Second call: marker is already gone, an existing X-Robots-Tag is left as-is.
+    applyPrivacyMaskHeaders(headers);
+    expect(headers.get('X-Robots-Tag')).toBe('noindex');
+    expect(headers.has(PRIVACY_MASK_MARKER)).toBe(false);
+  });
+
+  it('treats PRIVACY_MASK_APPLIED as the literal type — only that exact value triggers the translate', () => {
+    // Type-level guard: PRIVACY_MASK_APPLIED is typed `as const`, so this comparison compiles
+    // only because the constant is the exported literal. A re-typed string-literal in
+    // `markPrivacyMaskApplied` (e.g. 'applied' with whitespace) would no longer satisfy
+    // `=== PRIVACY_MASK_APPLIED` and the test would fail.
+    expect(PRIVACY_MASK_APPLIED).toBe('applied');
+
+    const headers = new Headers();
+    headers.set(PRIVACY_MASK_MARKER, 'something-else');
+    applyPrivacyMaskHeaders(headers);
+    expect(headers.has('X-Robots-Tag')).toBe(false);
+    expect(headers.has(PRIVACY_MASK_MARKER)).toBe(false);
+  });
+});

apps/web/app/lib/security.ts

@@ -61,3 +61,44 @@ export function nonceLessSecurityHeaders(scriptHashes: string[], isProd: boolean
   if (isProd) headers.set('Content-Security-Policy', csp(scriptHashes));
   return headers;
 }
+
+// Internal marker header: a route that decides its response contains natural-person data sets this
+// on the outgoing `Headers` so the worker `hardenResponse` can translate it into a public-facing
+// `X-Robots-Tag: noindex`. The marker is intentionally internal — `applyPrivacyMaskHeaders` deletes
+// it from the final response so it never reaches the edge cache or the client.
+export const PRIVACY_MASK_MARKER = 'X-Privacy-Mask';
+
+// `as const` narrows the type to the literal `'applied'` (not the wider `string`), which forces
+// callers that compare against it to use this exported constant rather than re-typing the string.
+export const PRIVACY_MASK_APPLIED = 'applied' as const;
+
+// Route-layer helper: stamps the privacy-mask marker onto a `Headers` object so the downstream
+// worker `hardenResponse` can pick it up and translate it into `X-Robots-Tag: noindex`.
+//
+// Two legitimate call sites:
+//   1. **Per-row maskers** (e.g. `/companies/:eik.data`, `/contracts/:id.json`) — call only when
+//      the response body actually contains masked natural-person data, so a legal-entity response
+//      stays out of the noindex bucket.
+//   2. **Blanket-policy surfaces** (the three public CSV exports — `/contracts.csv`,
+//      `/companies.csv`, `/authorities.csv`) — the policy documented in `privacy.tsx` and
+//      `docs/privacy-masking.md` says EVERY CSV export carries `noindex` regardless of whether
+//      any specific row is a natural person, because CSV is a bulk machine-readable surface and
+//      the noindex signal is enforced blanket-wide. CSV callers therefore invoke this helper
+//      unconditionally inside `markCsvCache` (csv-export.ts).
+//
+// Workers translate the marker to `X-Robots-Tag: noindex` and delete it before cache/client, so
+// future callers can pick the strategy that fits their surface without leaking the marker.
+export function markPrivacyMaskApplied(headers: Headers): void {
+  headers.set(PRIVACY_MASK_MARKER, PRIVACY_MASK_APPLIED);
+}
+
+// Worker-layer helper: if the privacy-mask marker is set to `applied`, translate it into the
+// public-facing `X-Robots-Tag: noindex` header. The marker is then deleted unconditionally so it
+// never leaks into the edge cache or the response. Idempotent: a second call finds no marker and
+// leaves an existing `X-Robots-Tag` header untouched.
+export function applyPrivacyMaskHeaders(headers: Headers): void {
+  if (headers.get(PRIVACY_MASK_MARKER) === PRIVACY_MASK_APPLIED) {
+    headers.set('X-Robots-Tag', 'noindex');
+  }
+  headers.delete(PRIVACY_MASK_MARKER);
+}