From 990c388905604c7a0dcf392947d059d8c9fb8956 Mon Sep 17 00:00:00 2001 From: Daniel Alkurdi Date: Fri, 7 Aug 2026 13:29:35 +1000 Subject: [PATCH 1/2] feat(sources): six court sources; fix the dead APH inquiry feeds; diagnose Qld TLS Courts coverage grows from five bodies to eleven: WA, SA, ACT, Tasmanian and NT supreme courts and the Administrative Review Tribunal, every URL verified live through the collector's own retrieval pipeline before entry. SA arrives late: the Courts Administration Authority published guidelines on generative AI in litigation in December 2025 (in force 1 January 2026) and Policai missed them for want of this source. The house and joint inquiry RSS endpoints still serve valid XML but have carried zero items while the senate feed retains 33 current referrals: abandoned husks, verified 2026-08-08. Both sources now watch the official Current Inquiries listing pages through the browser path, as content-watched documents, because APH bot-protects its HTML and the listing markup exposes no extractable entries. statements.qld.gov.au fails because the server omits its TLS intermediate (GeoTrust TLS RSA CA G1); desktop browsers repair that via AIA fetching, headless clients cannot. The collector host now supplies the public intermediate through NODE_EXTRA_CA_CERTS for the HTTP path; the browser path needs NSS trust or Firecrawl index retrieval, tracked in an issue. The automatic-source count pin moves from 57 to 63. Audited on Argus: 8 of 9 touched sources OK (Qld deferred as above). Gate: 442 tests, typecheck, lint. Co-Authored-By: Claude Opus 5 (1M context) --- src/lib/pipeline/collect.test.ts | 2 +- src/lib/pipeline/sources.ts | 96 +++++++++++++++++++++++++++++--- 2 files changed, 88 insertions(+), 10 deletions(-) diff --git a/src/lib/pipeline/collect.test.ts b/src/lib/pipeline/collect.test.ts index 06aa94a..76b394f 100644 --- a/src/lib/pipeline/collect.test.ts +++ b/src/lib/pipeline/collect.test.ts @@ -192,7 +192,7 @@ describe('collect', () => { itemCount: 2, candidateCount: 2, }); - expect(result.meta.collector.automaticSourceCount).toBe(57); + expect(result.meta.collector.automaticSourceCount).toBe(63); expect(result.meta.collector.manualSourceCount).toBe(1); expect(result.errors).toEqual([]); }); diff --git a/src/lib/pipeline/sources.ts b/src/lib/pipeline/sources.ts index b2a3ec9..7c94ded 100644 --- a/src/lib/pipeline/sources.ts +++ b/src/lib/pipeline/sources.ts @@ -319,30 +319,28 @@ export const WATCH_SOURCES: WatchSource[] = [ name: 'House of Representatives — new committee inquiries', jurisdiction: 'federal', category: 'government', - url: 'https://www.aph.gov.au/house/rss/house_inquiries', - kind: 'rss', + url: 'https://www.aph.gov.au/Parliamentary_Business/Committees/House/Current_Inquiries/', + kind: 'document', schedule: 'daily', enabled: true, automation: 'automatic', fetchStrategy: 'browser', - minimumItemCount: 1, notes: - 'Official House RSS feed for newly referred committee inquiries; APH returns 403 to plain HTTP clients. The official new-inquiries index retains current referrals, so a zero-item feed is treated as degraded rather than fresh coverage.', + 'Current House committee inquiries listing. The old /house/rss/house_inquiries endpoint still serves valid RSS but has carried zero items while the senate feed retains 33 current referrals — an abandoned husk, verified 2026-08-08. The HTML listing is behind APH bot protection, so retrieval goes through the browser path. The listing markup exposes no extractable entries to the allow-listed structures, so the page is watched for content changes instead, like nsw-sc-practice-notes.', }, { id: 'joint-new-inquiries', name: 'Parliamentary joint committees — new inquiries', jurisdiction: 'federal', category: 'government', - url: 'https://www.aph.gov.au/house/rss/joint_inquiries', - kind: 'rss', + url: 'https://www.aph.gov.au/Parliamentary_Business/Committees/Joint/Current_Inquiries', + kind: 'document', schedule: 'daily', enabled: true, automation: 'automatic', fetchStrategy: 'browser', - minimumItemCount: 1, notes: - 'Official joint-committee RSS feed for newly referred inquiries; APH returns 403 to plain HTTP clients. The official new-inquiries index retains current referrals, so a zero-item feed is treated as degraded rather than fresh coverage.', + 'Current joint-committee inquiries listing. The old /house/rss/joint_inquiries endpoint serves valid but permanently empty RSS, verified 2026-08-08 — same abandoned husk as house-new-inquiries. HTML listing is bot-protected, so retrieval goes through the browser path. The listing markup exposes no extractable entries to the allow-listed structures, so the page is watched for content changes instead, like nsw-sc-practice-notes.', }, { id: 'aph-bills-digests', @@ -614,6 +612,86 @@ export const WATCH_SOURCES: WatchSource[] = [ notes: 'Practice notes including generative-AI usage directions; the directory markup exposes no extractable entries, so the page is watched for content changes. The host rejects plain HTTP clients.', }, + { + id: 'art-practice-directions', + name: 'Administrative Review Tribunal — practice directions', + jurisdiction: 'federal', + category: 'court', + url: 'https://www.art.gov.au/help-and-resources/professionals-and-practitioners/practice-directions-and-other-guidance', + kind: 'document', + schedule: 'weekly', + enabled: true, + automation: 'automatic', + fetchStrategy: 'browser', + notes: + 'The federal merits-review tribunal, successor to the AAT and a likely venue for automated-decision-making guidance. The host resets plain HTTP/2 clients, so retrieval goes through the browser path.', + }, + { + id: 'wa-sc-practice-directions', + name: 'WA Supreme Court — practice directions', + jurisdiction: 'wa', + category: 'court', + url: 'https://www.supremecourt.wa.gov.au/P/practice_directions.aspx', + kind: 'document', + schedule: 'weekly', + enabled: true, + automation: 'automatic', + notes: + 'Consolidated practice directions index, watched for content changes. Plain HTTP verified 2026-08-08.', + }, + { + id: 'sa-courts-news', + name: 'SA Courts Administration Authority — news', + jurisdiction: 'sa', + category: 'court', + url: 'https://www.courts.sa.gov.au/feed/', + kind: 'rss', + schedule: 'daily', + enabled: true, + automation: 'automatic', + notes: + 'CAA announcements feed. Carried the December 2025 guidelines on generative AI in South Australian litigation (in force 1 January 2026), which pre-dated this source and were missed. Plain HTTP RSS verified 2026-08-08.', + }, + { + id: 'act-sc-practice-directions', + name: 'ACT Supreme Court — practice directions', + jurisdiction: 'act', + category: 'court', + url: 'https://www.courts.act.gov.au/supreme/law-and-practice/practice-notes-and-directions-and-notices-to-practitioners/practice-directions', + kind: 'document', + schedule: 'weekly', + enabled: true, + automation: 'automatic', + notes: + 'Practice directions index, watched for content changes. Plain HTTP verified 2026-08-08.', + }, + { + id: 'tas-sc-practice-directions', + name: 'Tasmanian Supreme Court — practice directions', + jurisdiction: 'tas', + category: 'court', + url: 'https://www.supremecourt.tas.gov.au/publications/directions/', + kind: 'document', + schedule: 'weekly', + enabled: true, + automation: 'automatic', + notes: + 'Practice directions index, watched for content changes. Plain HTTP verified 2026-08-08.', + }, + { + id: 'nt-sc-practice-directions', + name: 'NT Supreme Court — practice directions', + jurisdiction: 'nt', + category: 'court', + url: 'https://supremecourt.nt.gov.au/lawyers/practice-directions', + kind: 'document', + schedule: 'weekly', + enabled: true, + automation: 'automatic', + fetchStrategy: 'browser', + notes: + 'Practice directions index, watched for content changes. The host returns 403 to plain HTTP clients, so retrieval goes through the browser path.', + }, { id: 'nsw-sc-practice-notes', name: 'NSW Supreme Court — practice notes', @@ -770,7 +848,7 @@ export const WATCH_SOURCES: WatchSource[] = [ automation: 'automatic', fetchStrategy: 'browser', notes: - 'Whole-of-government Queensland announcements; the statements list is client-rendered without extractable entries, so the page is watched for content changes.', + 'Whole-of-government Queensland announcements; the statements list is client-rendered without extractable entries, so the page is watched for content changes. The server omits its TLS intermediate (GeoTrust TLS RSA CA G1), which desktop browsers repair via AIA fetching but headless clients cannot; the collector host supplies the public intermediate through NODE_EXTRA_CA_CERTS.', }, { id: 'wa-media-statements', From e70fd941c4541e49283e6c4bc80a5003324da16b Mon Sep 17 00:00:00 2001 From: Daniel Alkurdi Date: Fri, 7 Aug 2026 13:34:32 +1000 Subject: [PATCH 2/2] fix(sources): canonical URL form, no trailing slashes --- src/lib/pipeline/sources.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/lib/pipeline/sources.ts b/src/lib/pipeline/sources.ts index 7c94ded..f3d6647 100644 --- a/src/lib/pipeline/sources.ts +++ b/src/lib/pipeline/sources.ts @@ -319,7 +319,7 @@ export const WATCH_SOURCES: WatchSource[] = [ name: 'House of Representatives — new committee inquiries', jurisdiction: 'federal', category: 'government', - url: 'https://www.aph.gov.au/Parliamentary_Business/Committees/House/Current_Inquiries/', + url: 'https://www.aph.gov.au/Parliamentary_Business/Committees/House/Current_Inquiries', kind: 'document', schedule: 'daily', enabled: true, @@ -644,7 +644,7 @@ export const WATCH_SOURCES: WatchSource[] = [ name: 'SA Courts Administration Authority — news', jurisdiction: 'sa', category: 'court', - url: 'https://www.courts.sa.gov.au/feed/', + url: 'https://www.courts.sa.gov.au/feed', kind: 'rss', schedule: 'daily', enabled: true, @@ -670,7 +670,7 @@ export const WATCH_SOURCES: WatchSource[] = [ name: 'Tasmanian Supreme Court — practice directions', jurisdiction: 'tas', category: 'court', - url: 'https://www.supremecourt.tas.gov.au/publications/directions/', + url: 'https://www.supremecourt.tas.gov.au/publications/directions', kind: 'document', schedule: 'weekly', enabled: true,