diff --git a/game.ts b/game.ts index 90e589b..d5f7647 100644 --- a/game.ts +++ b/game.ts @@ -2,6 +2,7 @@ import { generateText } from "ai"; import { createOpenRouter } from "@openrouter/ai-sdk-provider"; import { mkdirSync, appendFileSync } from "node:fs"; import { join } from "node:path"; +import { extractJSON } from "./llm-json-fixer"; // ── Models ────────────────────────────────────────────────────────────────── @@ -89,6 +90,9 @@ const openrouter = createOpenRouter({ }, }); +const EXPERIMENTAL_VERBAL_SAMPLE_COT = + process.env.EXPERIMENTAL_VERBAL_SAMPLE_COT === "1"; + // ── Logger ────────────────────────────────────────────────────────────────── const LOGS_DIR = join(import.meta.dir, "logs"); @@ -199,20 +203,144 @@ ${examples.map((p) => `- ${p}`).join("\n")} Come up with something ORIGINAL — don't copy these examples.`; } +function buildVerbalSampleCotSystem(): string { + const examples = shuffle([...ALL_PROMPTS]).slice(0, 80); + return `You are a comedy writer for the game Quiplash. Generate 5 funny fill-in-the-blank prompts that players will try to answer. + +Think in a verbal, observational stand-up style and explain your thought process. + +Output ONLY a single valid JSON object in this exact shape: +{ + "reasoning": "string", + "jokes": [ + { "joke": "string", "probability": 0.0 } + ] +} + +Rules: +- "reasoning" must be a single string with your step-by-step verbal creative process. +- "jokes" must contain exactly 5 items. +- Each "joke" must be a single Quiplash-style fill-in-the-blank prompt under 15 words. +- Each "probability" must be a number between 0 and 1. +- Be highly varied in prompt formats. Do NOT overuse "The worst thing to..." +- Be original and do not copy examples. + +Style examples: +${examples.map((p) => `- ${p}`).join("\n")}`; +} + export async function callGeneratePrompt(model: Model): Promise { log("INFO", `prompt:${model.name}`, "Calling API", { modelId: model.id }); - const system = buildPromptSystem(); - const { text, usage, reasoning } = await generateText({ + if (!EXPERIMENTAL_VERBAL_SAMPLE_COT) { + const system = buildPromptSystem(); + const { text, usage, reasoning } = await generateText({ + model: openrouter.chat(model.id), + system, + prompt: + "Generate a single original Quiplash prompt. Be creative and don't repeat common patterns.", + }); + + log("INFO", `prompt:${model.name}`, "Raw response", { + rawText: text, + usage, + }); + return cleanResponse(text); + } + + const system = buildVerbalSampleCotSystem(); + const { text, usage } = await generateText({ model: openrouter.chat(model.id), system, - prompt: - "Generate a single original Quiplash prompt. Be creative and don't repeat common patterns.", + prompt: "Generate 5 original Quiplash prompts and return only the JSON object.", }); - log("INFO", `prompt:${model.name}`, "Raw response", { + log("INFO", `prompt:${model.name}`, "Raw verbal sample CoT response", { rawText: text, usage, }); + + const parsed = extractJSON(text) as { + reasoning?: unknown; + jokes?: unknown; + }; + + if (!Array.isArray(parsed.jokes) || parsed.jokes.length !== 5) { + throw new Error("Invalid verbal sample CoT output: jokes must contain 5 items"); + } + + const candidates = parsed.jokes + .map((item) => { + if (!item || typeof item !== "object") { + return null; + } + const jokeValue = (item as { joke?: unknown }).joke; + const probValue = (item as { probability?: unknown }).probability; + if (typeof jokeValue !== "string") { + return null; + } + const joke = cleanResponse(jokeValue); + if (!joke) { + return null; + } + const probability = + typeof probValue === "number" && Number.isFinite(probValue) + ? Math.max(0, Math.min(1, probValue)) + : 0; + return { joke, probability }; + }) + .filter((item): item is { joke: string; probability: number } => item !== null); + + if (!candidates.length) { + throw new Error("Invalid verbal sample CoT output: no valid joke candidates"); + } + + const selected = await callSelectBestPrompt( + model, + candidates.map((c) => c.joke), + ); + + const matched = candidates.find((c) => c.joke === selected); + if (matched) { + return matched.joke; + } + + const fallback = candidates.reduce((best, current) => { + if (!best || current.probability > best.probability) { + return current; + } + return best; + }, null as { joke: string; probability: number } | null); + + if (!fallback) { + throw new Error("Failed to select prompt from verbal sample CoT candidates"); + } + + return fallback.joke; +} + +export async function callSelectBestPrompt( + model: Model, + jokes: string[], +): Promise { + log("INFO", `prompt-select:${model.name}`, "Calling API", { + modelId: model.id, + candidateCount: jokes.length, + }); + + const { text, usage } = await generateText({ + model: openrouter.chat(model.id), + system: + "Step into the mind of a world-class stand-up comic about to headline a sold-out arena. Trust only your battle-tested instinct for what makes real humans explode with laughter. Choose and deliver the one joke you know, from years of reading crowds, will absolutely destroy the room.", + prompt: `Choose exactly one of these Quiplash prompts and reply with ONLY the exact prompt text, nothing else:\n\n${jokes + .map((joke, i) => `${i + 1}. ${joke}`) + .join("\n")}`, + }); + + log("INFO", `prompt-select:${model.name}`, "Raw response", { + rawText: text, + usage, + }); + return cleanResponse(text); } diff --git a/llm-json-fixer.ts b/llm-json-fixer.ts new file mode 100644 index 0000000..c2085ab --- /dev/null +++ b/llm-json-fixer.ts @@ -0,0 +1,284 @@ +export type JSONFix = + | "markdown_stripped" + | "trailing_removed" + | "unescaped_quote" + | "missing_comma"; + +export type ParseJSONResult = { + data: T | null; + fixes: JSONFix[]; + warnings: string[]; +}; + +function extractFromMarkdown(input: string): string { + if (/```(?:json|JSON)?\s*\r?\n\s*\r?\n```/.test(input)) { + return ""; + } + + const fencedJson = /```(?:json|JSON)\s*\r?\n([\s\S]*?)\r?\n\s*```/g; + let match: RegExpExecArray | null; + while ((match = fencedJson.exec(input)) !== null) { + const candidate = (match[1] ?? "").trim(); + if (candidate) { + return candidate; + } + } + + const fenced = /```\s*\r?\n([\s\S]*?)\r?\n\s*```/g; + while ((match = fenced.exec(input)) !== null) { + const candidate = (match[1] ?? "").trim(); + if (candidate.startsWith("{") || candidate.startsWith("[")) { + return candidate; + } + } + + const inline = /`([^`]+)`/g; + while ((match = inline.exec(input)) !== null) { + const candidate = (match[1] ?? "").trim(); + if (candidate.startsWith("{") || candidate.startsWith("[")) { + return candidate; + } + } + + return input; +} + +function findFirstJSONStart(input: string): number { + let inString = false; + let escapeNext = false; + for (let i = 0; i < input.length; i++) { + const char = input[i]!; + if (escapeNext) { + escapeNext = false; + continue; + } + if (inString && char === "\\") { + escapeNext = true; + continue; + } + if (char === '"') { + inString = !inString; + continue; + } + if (!inString && (char === "{" || char === "[")) { + return i; + } + } + return -1; +} + +function removeTrailingContent(input: string): string { + const start = findFirstJSONStart(input); + if (start === -1) { + return input.trim(); + } + + const source = input.slice(start); + let inString = false; + let escapeNext = false; + const stack: string[] = []; + + for (let i = 0; i < source.length; i++) { + const char = source[i]!; + + if (escapeNext) { + escapeNext = false; + continue; + } + if (inString && char === "\\") { + escapeNext = true; + continue; + } + if (char === '"') { + inString = !inString; + continue; + } + if (inString) { + continue; + } + + if (char === "{" || char === "[") { + stack.push(char); + continue; + } + + if (char === "}" || char === "]") { + const opening = stack.pop(); + if (!opening) { + continue; + } + if ((opening === "{" && char !== "}") || (opening === "[" && char !== "]")) { + continue; + } + if (stack.length === 0) { + return source.slice(0, i + 1).trimEnd(); + } + } + } + + return source.trim(); +} + +function fixUnescapedQuotes(input: string): string { + let out = ""; + let inString = false; + let escapeNext = false; + + for (let i = 0; i < input.length; i++) { + const char = input[i]!; + if (escapeNext) { + out += char; + escapeNext = false; + continue; + } + + if (char === "\\" && inString) { + out += char; + escapeNext = true; + continue; + } + + if (char === '"' && !inString) { + inString = true; + out += char; + continue; + } + + if (char === '"' && inString) { + let j = i + 1; + while (j < input.length && /\s/.test(input[j]!)) { + j++; + } + const next = input[j] ?? ""; + const closesString = + next === "" || next === "," || next === ":" || next === "}" || next === "]"; + if (closesString) { + inString = false; + out += char; + } else { + out += "\\\""; + } + continue; + } + + out += char; + } + + return out; +} + +function addMissingCommas(input: string): string { + if (!input.includes("\n")) { + return input + .replace(/([\]}"\d]|true|false|null)\s+("[^"]+"\s*:)/g, "$1, $2") + .replace(/([\]}"\d]|true|false|null)\s+("|\{|\[|\d|true|false|null)/g, "$1, $2"); + } + + const lines = input.split("\n"); + const output: string[] = []; + let depth = 0; + + for (let i = 0; i < lines.length; i++) { + const line = lines[i]!; + const trimmed = line.trim(); + output.push(line); + + if ( + !trimmed || + trimmed.endsWith(",") || + trimmed.endsWith("{") || + trimmed.endsWith("[") || + trimmed.endsWith(":") + ) { + for (const char of line) { + if (char === "{" || char === "[") depth++; + else if (char === "}" || char === "]") depth--; + } + continue; + } + + let nextIndex = i + 1; + while (nextIndex < lines.length && lines[nextIndex]!.trim() === "") { + nextIndex++; + } + + const next = nextIndex < lines.length ? lines[nextIndex]!.trim() : ""; + const canTakeComma = depth > 0 && next && !next.startsWith("}") && !next.startsWith("]"); + if (canTakeComma) { + output[output.length - 1] = `${line},`; + } + + for (const char of line) { + if (char === "{" || char === "[") depth++; + else if (char === "}" || char === "]") depth--; + } + } + + return output.join("\n"); +} + +export function tryParseJSON(input: string): ParseJSONResult { + const fixes: JSONFix[] = []; + const warnings: string[] = []; + let processed = input.trim(); + + if (!processed) { + return { data: null, fixes, warnings: ["Input is empty"] }; + } + + const attempts: Array<() => string> = [ + () => { + const next = extractFromMarkdown(processed); + if (next !== processed) fixes.push("markdown_stripped"); + return next; + }, + () => { + const next = removeTrailingContent(processed); + if (next !== processed) fixes.push("trailing_removed"); + return next; + }, + () => { + const next = fixUnescapedQuotes(processed); + if (next !== processed) fixes.push("unescaped_quote"); + return next; + }, + () => { + const next = addMissingCommas(processed); + if (next !== processed) fixes.push("missing_comma"); + return next; + }, + ]; + + for (let i = 0; i <= attempts.length; i++) { + try { + return { + data: JSON.parse(processed) as T, + fixes, + warnings, + }; + } catch (error) { + warnings.push((error as Error).message); + if (i === attempts.length) { + break; + } + const next = attempts[i]!(); + if (next !== processed) { + processed = next; + } + } + } + + return { data: null, fixes, warnings }; +} + +export function parseJSON(input: string): T | null { + return tryParseJSON(input).data; +} + +export function extractJSON(text: string): unknown { + const parsed = tryParseJSON(text); + if (parsed.data === null) { + throw new Error(`Failed to parse JSON from AI response: ${parsed.warnings.join(" | ")}`); + } + return parsed.data; +} \ No newline at end of file