Decode Codex JSONL before extracting news reports

__tests__/news-regressions.test.mjs

@@ -1,6 +1,7 @@
 import test from 'node:test'
 import assert from 'node:assert/strict'
 import { readFileSync } from 'node:fs'
+import { spawnSync } from 'node:child_process'
 import { dirname, join } from 'node:path'
 import { fileURLToPath } from 'node:url'
 import {
@@ -17,6 +18,61 @@ const HERE = dirname(fileURLToPath(import.meta.url))
 const repo = join(HERE, '..')
 const readRepoFile = (name) => readFileSync(join(repo, name), 'utf8')
 
+const decodeCodexOutput = (input) => {
+  const result = spawnSync('python3', [join(repo, 'codex_output.py')], {
+    input,
+    encoding: 'utf8',
+  })
+  assert.equal(result.status, 0, result.stderr)
+  return result.stdout
+}
+
+test('Codex output decoder unwraps current item.completed messages', () => {
+  const html = '<article class="news-report">\n<p>Today</p>\n</article>'
+  const jsonl = [
+    JSON.stringify({ type: 'thread.started', thread_id: 'thread-1' }),
+    JSON.stringify({
+      type: 'item.completed',
+      item: { type: 'agent_message', text: html },
+    }),
+    JSON.stringify({ type: 'turn.completed', usage: { input_tokens: 10 } }),
+  ].join('\n')
+
+  assert.equal(decodeCodexOutput(jsonl), html)
+  assert.equal(decodeCodexOutput(jsonl).includes('\\n'), false,
+    'JSON escapes must become real newlines before HTML extraction')
+})
+
+test('Codex output decoder preserves legacy agent-message envelopes', () => {
+  const first = JSON.stringify({ type: 'agent_message', message: '<article>' })
+  const second = JSON.stringify({ msg: { type: 'agent_message', message: '</article>' } })
+  assert.equal(decodeCodexOutput(`${first}\n${second}\n`), '<article></article>')
+})
+
+test('Codex output decoder fails closed for unknown JSON transport', () => {
+  // This is the regression guard for the visible "\\n" failure: markup in an
+  // unrecognized transport event must not be handed to the HTML scanner raw.
+  const escapedReport = '<article>\\n<p>transport, not content</p>\\n</article>'
+  const jsonl = JSON.stringify({
+    type: 'item.completed',
+    item: { type: 'reasoning', text: escapedReport },
+  })
+  assert.equal(decodeCodexOutput(jsonl), '')
+})
+
+test('Codex output decoder keeps plain-text compatibility without JSON transport', () => {
+  const html = '<article>\n<p>legacy plain output</p>\n</article>'
+  assert.equal(decodeCodexOutput(html), html)
+})
+
+test('fetch.sh decodes Codex transport before scanning for report HTML', () => {
+  const sh = readRepoFile('fetch.sh')
+  assert.ok(sh.includes('from codex_output import extract_codex_agent_text'))
+  assert.ok(sh.includes('text = extract_codex_agent_text(raw)'))
+  assert.ok(!sh.includes('msg = obj.get("msg", obj)'),
+    'the obsolete envelope-specific inline parser must not return')
+})
+
 // --- Blocker 1: "Generate report now" must terminate on a run-status terminal,
 // even when a preserved good digest leaves reports/<today>.html (and thus its
 // mtime) untouched. These EXECUTE the extracted terminal-detection decision;

codex_output.py

@@ -0,0 +1,107 @@
+#!/usr/bin/env python3
+"""Extract assistant text from ``codex exec --json`` output.
+
+Codex writes JSONL transport events, not the assistant message verbatim.  Keep
+that transport boundary explicit: known agent-message envelopes are decoded,
+plain non-JSON output remains a legacy fallback, and valid-but-unknown JSONL
+fails closed instead of being mistaken for report HTML.
+"""
+
+from __future__ import annotations
+
+import json
+import sys
+from typing import Any
+
+
+_AGENT_MESSAGE_TYPES = {"agent_message", "agentMessage"}
+_COMPLETED_EVENT_TYPES = {"item.completed", "item_completed"}
+
+
+def _content_text(value: Any) -> str:
+    """Normalize string or text-block content without accepting tool output."""
+    if isinstance(value, str):
+        return value
+    if not isinstance(value, list):
+        return ""
+
+    parts: list[str] = []
+    for block in value:
+        if not isinstance(block, dict):
+            continue
+        if block.get("type") not in ("text", "output_text"):
+            continue
+        text = block.get("text")
+        if isinstance(text, str):
+            parts.append(text)
+    return "".join(parts)
+
+
+def _message_text(item: Any) -> str:
+    if not isinstance(item, dict) or item.get("type") not in _AGENT_MESSAGE_TYPES:
+        return ""
+    for key in ("text", "content", "message"):
+        text = _content_text(item.get(key))
+        if text:
+            return text
+    return ""
+
+
+def extract_codex_agent_text(raw: str) -> str:
+    """Return decoded assistant text, or ``""`` for unknown JSONL events.
+
+    Supported envelopes:
+      * current: ``item.completed`` with ``item.type=agent_message`` + ``text``
+      * legacy: a top-level ``agent_message`` event
+      * legacy: an ``agent_message`` nested under ``msg``
+
+    Multiple completed message items are joined in stream order. If there are
+    no parseable JSON objects at all, ``raw`` is returned for compatibility
+    with old/plain-text Codex output. Once JSON transport is detected, however,
+    raw fallback is unsafe: escaped HTML inside an unknown event must never be
+    scanned as if it were the report body.
+    """
+    parts: list[str] = []
+    saw_json_transport = False
+
+    for raw_line in raw.splitlines():
+        line = raw_line.strip()
+        if not line:
+            continue
+        try:
+            event = json.loads(line)
+        except (TypeError, ValueError):
+            continue
+        if not isinstance(event, dict):
+            continue
+
+        saw_json_transport = True
+        event_type = event.get("type")
+        candidates: list[Any] = []
+        if event_type in _COMPLETED_EVENT_TYPES:
+            candidates.append(event.get("item"))
+        elif event_type in _AGENT_MESSAGE_TYPES:
+            candidates.append(event)
+
+        # Older Codex builds wrapped the semantic event in ``msg``.
+        msg = event.get("msg")
+        if isinstance(msg, dict) and msg.get("type") in _AGENT_MESSAGE_TYPES:
+            candidates.append(msg)
+
+        for candidate in candidates:
+            text = _message_text(candidate)
+            if text:
+                parts.append(text)
+
+    if parts:
+        return "".join(parts)
+    return "" if saw_json_transport else raw
+
+
+def main() -> int:
+    sys.stdout.write(extract_codex_agent_text(sys.stdin.read()))
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

fetch.sh

@@ -40,6 +40,8 @@
 
 set -uo pipefail
 
+SCRIPT_DIR=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)
+
 APP_ID="${1:-}"
 if [ -z "$APP_ID" ]; then
   echo "fetch.sh: APP_ID required as first argument" >&2
@@ -890,48 +892,31 @@ fi
 
 # 4. Extract the HTML report article from the agent's output.
 #    - Claude -p: stdout is the final assistant message text verbatim.
-#    - Codex exec --json: stdout is JSONL; the final `agent_message`
-#      event carries the text. python3 grabs the last `agent_message`
-#      payload, or falls back to the raw bytes if parsing fails.
+#    - Codex exec --json: stdout is JSONL. codex_output.py decodes current
+#      and legacy agent-message envelopes. Unknown JSON transport fails closed
+#      instead of letting escaped HTML inside the envelope pass as a report.
 #    The agent is told to reply with bare HTML, but we tolerate
 #    surrounding prose by scanning for the first <article> block. Then
 #    we sanitize server-side: scripts/styles/event handlers are removed,
 #    only a small article-writing tag set is kept, and anchors keep only
 #    http(s) hrefs with safe target/rel attributes.
 EXTRACTED_FILE="$WORK_DIR/extracted.html"
-python3 - "$RAW_OUTPUT" "$EXTRACTED_FILE" "$PROVIDER" "$TODAY" <<'PY' 2>>"$LOG_FILE"
+python3 - "$RAW_OUTPUT" "$EXTRACTED_FILE" "$PROVIDER" "$TODAY" "$SCRIPT_DIR" <<'PY' 2>>"$LOG_FILE"
 from html import escape
 from html.parser import HTMLParser
 import json
 import re
 import sys
 
-raw_path, out_path, provider, today = sys.argv[1:5]
+raw_path, out_path, provider, today, script_dir = sys.argv[1:6]
 with open(raw_path, "r", encoding="utf-8", errors="replace") as f:
     raw = f.read()
 
 text = raw
 if provider == "codex":
-  # Last `agent_message` event holds the final text. Fall back to raw
-  # if no parseable lines (older codex shapes, mid-stream truncation).
-  last = ""
-  for line in raw.splitlines():
-    line = line.strip()
-    if not line:
-      continue
-    try:
-      obj = json.loads(line)
-    except json.JSONDecodeError:
-      continue
-    # Codex shape: {"type": "agent_message", "message": "..."} OR
-    # {"msg": {"type": "agent_message", "message": "..."}}.
-    msg = obj.get("msg", obj)
-    if isinstance(msg, dict) and msg.get("type") == "agent_message":
-      m = msg.get("message", "")
-      if isinstance(m, str):
-        last = m
-  if last:
-    text = last
+  sys.path.insert(0, script_dir)
+  from codex_output import extract_codex_agent_text
+  text = extract_codex_agent_text(raw)
 
 match = re.search(r"<article\b[\s\S]*?</article>", text, re.I)
 if not match: