From 2d8419544876da8b75018348e0e935e5cd9d8fe7 Mon Sep 17 00:00:00 2001 From: Abhishek Kumbhar Date: Sun, 26 Jul 2026 15:25:40 +0530 Subject: [PATCH 01/13] feat: add support for multilingual output in OpenWiki agent --- src/agent/index.ts | 9 +++++++-- src/agent/prompt.ts | 17 ++++++++++++++++- src/agent/types.ts | 2 ++ src/agent/utils.ts | 8 ++++++++ src/cli.tsx | 2 ++ src/commands.ts | 25 +++++++++++++++++++++++++ test/commands.test.ts | 34 ++++++++++++++++++++++++++++++++++ test/prompt.test.ts | 23 +++++++++++++++++++++++ test/run-context.test.ts | 24 ++++++++++++++++++++++++ 9 files changed, 141 insertions(+), 3 deletions(-) create mode 100644 test/run-context.test.ts diff --git a/src/agent/index.ts b/src/agent/index.ts index 2792ce64..275d1d31 100644 --- a/src/agent/index.ts +++ b/src/agent/index.ts @@ -232,7 +232,12 @@ async function runOpenWikiAgentCore( providerRetryAttempts: number, ): Promise { const outputMode = options.outputMode ?? "local-wiki"; - const context = await createRunContext(command, cwd, outputMode); + const context = await createRunContext( + command, + cwd, + outputMode, + options.language, + ); emitDebug(options, "context=created"); const openWikiSnapshotBefore = command === "chat" @@ -279,7 +284,7 @@ async function runOpenWikiAgentCore( permissions: [ { operations: ["write"], paths: ["/skills/**"], mode: "deny" }, ], - systemPrompt: createSystemPrompt(command, outputMode), + systemPrompt: createSystemPrompt(command, outputMode, context.language), }); emitDebug(options, "agent=created"); diff --git a/src/agent/prompt.ts b/src/agent/prompt.ts index a2c98315..422ddcd6 100644 --- a/src/agent/prompt.ts +++ b/src/agent/prompt.ts @@ -16,13 +16,15 @@ function formatLastUpdate(lastUpdate: UpdateMetadata | null): string { export function createSystemPrompt( command: OpenWikiCommand, outputMode: OpenWikiOutputMode = "local-wiki", + language?: string, ): string { const output = getOutputPromptConfig(outputMode); + const languageInstructions = createLanguageInstructions(language); return ` You are OpenWiki, an expert technical writer, software architect, and product analyst. -Your job is to inspect the relevant source evidence and local OpenWiki knowledge sources, then produce documentation in ${output.docsLocation} that is excellent for both humans and future agents. OpenWiki can maintain a local general-purpose knowledge wiki from connector raw dumps under ~/.openwiki. +Your job is to inspect the relevant source evidence and local OpenWiki knowledge sources, then produce documentation in ${output.docsLocation} that is excellent for both humans and future agents. OpenWiki can maintain a local general-purpose knowledge wiki from connector raw dumps under ~/.openwiki.${languageInstructions} ${output.canonicalLocationInstruction} @@ -200,6 +202,19 @@ ${createModeInstructions(command, outputMode)} `.trim(); } +function createLanguageInstructions(language: string | undefined): string { + if (!language) { + return ""; + } + + return ` + +Output language: +- Write generated wiki prose, headings, table content, and documentation in ${language}. +- Apply this language only to generated wiki files. Do not translate OpenWiki CLI text or runtime messages. +- Keep code identifiers, file paths, commands, API names, URLs, and code blocks unchanged where translation would reduce technical accuracy or usability.`; +} + export function createDiagramInstructions(): string { return ` Diagram discipline: diff --git a/src/agent/types.ts b/src/agent/types.ts index a77cf8f4..a00ef2f2 100644 --- a/src/agent/types.ts +++ b/src/agent/types.ts @@ -34,6 +34,7 @@ export type OpenWikiRunEvent = export type OpenWikiRunOptions = { debug?: boolean; isFollowup?: boolean; + language?: string | null; modelId?: string | null; onEvent?: (event: OpenWikiRunEvent) => void; outputMode?: OpenWikiOutputMode; @@ -55,5 +56,6 @@ export type UpdateMetadata = { export type RunContext = { lastUpdate: UpdateMetadata | null; gitSummary: string; + language?: string; wikiGoal?: string; }; diff --git a/src/agent/utils.ts b/src/agent/utils.ts index 6129540c..a54e5e8c 100644 --- a/src/agent/utils.ts +++ b/src/agent/utils.ts @@ -46,14 +46,20 @@ export async function createRunContext( command: OpenWikiCommand, cwd: string, outputMode: OpenWikiOutputMode = "repository", + language?: string | null, ): Promise { const lastUpdate = await readLastUpdate(cwd, outputMode); + const normalizedLanguage = language?.trim() || undefined; + const languageContext = normalizedLanguage + ? { language: normalizedLanguage } + : {}; const wikiGoal = await readRunWikiGoal(cwd, outputMode); if (command === "chat") { return { lastUpdate, gitSummary: "Not applicable for chat.", + ...languageContext, wikiGoal, }; } @@ -63,6 +69,7 @@ export async function createRunContext( lastUpdate, gitSummary: "Local wiki mode: connector source evidence is provided through raw data paths and OpenWiki connector tools. Git repository diff context is not used for this run.", + ...languageContext, wikiGoal, }; } @@ -70,6 +77,7 @@ export async function createRunContext( return { lastUpdate, gitSummary: await createGitSummary(command, cwd, lastUpdate), + ...languageContext, wikiGoal, }; } diff --git a/src/cli.tsx b/src/cli.tsx index 1bb4abcb..4056b37a 100644 --- a/src/cli.tsx +++ b/src/cli.tsx @@ -623,6 +623,7 @@ function App({ command }: AppProps) { return runOpenWikiAgent(resolvedCommand, runtimeCwd, { debug: isDebugMode(), isFollowup: activeMessageIsFollowup, + language: command.language, modelId: sessionModelId, outputMode: runtimeOutputMode, threadId: sessionThreadId.current, @@ -4152,6 +4153,7 @@ async function runPrintCommand( await runOpenWikiAgent(command.command, runtimeCwd, { debug: isDebugMode(), isFollowup: command.command === "chat", + language: command.language, modelId: command.modelId, outputMode: runtimeOutputMode, threadId: createOpenWikiThreadId(runtimeCwd), diff --git a/src/commands.ts b/src/commands.ts index 2e7ab490..5d2c7370 100644 --- a/src/commands.ts +++ b/src/commands.ts @@ -58,6 +58,7 @@ export type CliCommand = exitCode: 0; command: OpenWikiCommand; dryRun: boolean; + language: string | null; mode: OpenWikiRunMode; modeSource: OpenWikiRunModeSource; modelId: string | null; @@ -338,6 +339,7 @@ function parseRunCommand( initialModeSource: OpenWikiRunModeSource, ): CliCommand { let dryRun = false; + let language: string | null = null; let mode = initialMode; let modeSource = initialModeSource; let modelId: string | null = null; @@ -394,6 +396,22 @@ function parseRunCommand( continue; } + if (arg === "--language" || arg === "-l") { + const nextArg = argv[index + 1]; + + if (!nextArg || nextArg.startsWith("-")) { + return { + kind: "error", + exitCode: 1, + message: `${arg} requires a locale.`, + }; + } + + language = nextArg; + index += 1; + continue; + } + if (arg === "--mode") { const nextArg = argv[index + 1]; @@ -564,6 +582,7 @@ function parseRunCommand( exitCode: 0, command, dryRun, + language, mode, modeSource, modelId, @@ -644,6 +663,7 @@ export const helpContent: HelpContent = { "openwiki code [--init|--update] [message]", "openwiki personal [--init|--update] [message]", "openwiki --mode [--init|--update] [message]", + "openwiki [--language ] [--init|--update] [message]", "openwiki [--modelId ]", "openwiki [--modelId ] [message]", "openwiki --update [message]", @@ -733,6 +753,11 @@ export const helpContent: HelpContent = { description: "Choose the personal brain (local, over configured sources) or the code brain (repository docs).", }, + { + label: "-l, --language ", + description: + "Generate wiki documentation in the requested language or locale.", + }, { label: "-p, --print", description: "Run once and print the final assistant output.", diff --git a/test/commands.test.ts b/test/commands.test.ts index ed1c89f4..1c5ecdbf 100644 --- a/test/commands.test.ts +++ b/test/commands.test.ts @@ -65,6 +65,10 @@ describe("parseCommand — help", () => { expect(helpText).toContain("scheduled-only ingestion"); expect(helpText).toContain("openwiki ingest all --scheduled --print"); }); + + test("help documents the output language option", () => { + expect(getHelpText()).toContain("-l, --language "); + }); }); describe("parseCommand — chat default", () => { @@ -180,6 +184,36 @@ describe("parseCommand — mode after flags", () => { }); describe("parseCommand — init/update", () => { + test("--language passes the selected locale to an init run", () => { + expect(parseCommand(["--init", "--language", "zh-CN"])).toMatchObject({ + kind: "run", + command: "init", + language: "zh-CN", + }); + }); + + test("-l passes the selected locale to an update run", () => { + expect(parseCommand(["--update", "-l", "zh-CN"])).toMatchObject({ + kind: "run", + command: "update", + language: "zh-CN", + }); + }); + + test("language is unset when no language option is supplied", () => { + expect(parseCommand(["--init"])).toMatchObject({ + kind: "run", + language: null, + }); + }); + + test("--language requires a locale", () => { + expect(parseCommand(["--language", "--init"])).toMatchObject({ + kind: "error", + message: "--language requires a locale.", + }); + }); + test("personal --init selects the init command and starts", () => { expect(parseCommand(["personal", "--init"])).toMatchObject({ kind: "run", diff --git a/test/prompt.test.ts b/test/prompt.test.ts index f28289fd..c55a3a5e 100644 --- a/test/prompt.test.ts +++ b/test/prompt.test.ts @@ -4,6 +4,29 @@ import { createSystemPrompt, } from "../src/agent/prompt.ts"; +describe("createSystemPrompt output language", () => { + test("instructs the agent to write wiki documentation in the selected language", () => { + const prompt = createSystemPrompt("init", "repository", "zh-CN"); + + expect(prompt).toContain("Output language:"); + expect(prompt).toContain( + "Write generated wiki prose, headings, table content, and documentation in zh-CN.", + ); + expect(prompt).toContain( + "Apply this language only to generated wiki files.", + ); + expect(prompt).toContain( + "Keep code identifiers, file paths, commands, API names, URLs, and code blocks unchanged", + ); + }); + + test("preserves the existing prompt behavior when no language is supplied", () => { + expect(createSystemPrompt("init", "repository")).not.toContain( + "Output language:", + ); + }); +}); + /** * Guards against the 0.2 regression where the shared "Canonical wiki location" * and "Wiki-first question answering" blocks hardcoded ~/.openwiki/wiki and diff --git a/test/run-context.test.ts b/test/run-context.test.ts new file mode 100644 index 00000000..c9013cdf --- /dev/null +++ b/test/run-context.test.ts @@ -0,0 +1,24 @@ +import { mkdtemp, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import { describe, expect, test } from "vitest"; +import { createRunContext } from "../src/agent/utils.ts"; + +describe("createRunContext output language", () => { + test("propagates a selected language and leaves it unset by default", async () => { + const cwd = await mkdtemp(path.join(tmpdir(), "openwiki-run-context-")); + + try { + await expect( + createRunContext("chat", cwd, "repository", "zh-CN"), + ).resolves.toMatchObject({ + language: "zh-CN", + }); + expect( + await createRunContext("chat", cwd, "repository"), + ).not.toHaveProperty("language"); + } finally { + await rm(cwd, { recursive: true, force: true }); + } + }); +}); From e741c58fdd5a92e705617d76e529d5d0288c7918 Mon Sep 17 00:00:00 2001 From: Colin Francis Date: Sun, 26 Jul 2026 19:51:05 -0700 Subject: [PATCH 02/13] validate the --language locale and fall back to english --- src/cli.tsx | 5 ++++ src/commands.ts | 9 ++++++- src/language.ts | 60 +++++++++++++++++++++++++++++++++++++++++++ test/commands.test.ts | 17 ++++++++++++ test/language.test.ts | 33 ++++++++++++++++++++++++ 5 files changed, 123 insertions(+), 1 deletion(-) create mode 100644 src/language.ts create mode 100644 test/language.test.ts diff --git a/src/cli.tsx b/src/cli.tsx index 4056b37a..0ff17717 100644 --- a/src/cli.tsx +++ b/src/cli.tsx @@ -3732,6 +3732,11 @@ if (commandEmitsTelemetry(command)) { showFirstRunNotice = await firstRunNoticePending(); } +if (command.kind === "run" && command.languageWarning) { + // stderr keeps piped stdout clean while still warning about an ignored locale. + process.stderr.write(`${command.languageWarning}\n`); +} + if (command.kind === "auth") { await runAuthCommand(command); } else if (command.kind === "ngrok") { diff --git a/src/commands.ts b/src/commands.ts index 5d2c7370..43a65413 100644 --- a/src/commands.ts +++ b/src/commands.ts @@ -1,5 +1,6 @@ import { isValidModelId, normalizeModelId } from "./constants.js"; import type { OpenWikiCommand } from "./agent/types.js"; +import { resolveLanguage } from "./language.js"; import { isAuthProviderId } from "./auth/providers.js"; import type { AuthProviderId } from "./auth/types.js"; import { parseIngestionTarget, type IngestionTarget } from "./ingestion.js"; @@ -59,6 +60,7 @@ export type CliCommand = command: OpenWikiCommand; dryRun: boolean; language: string | null; + languageWarning: string | null; mode: OpenWikiRunMode; modeSource: OpenWikiRunModeSource; modelId: string | null; @@ -565,6 +567,10 @@ function parseRunCommand( userMessageParts.length > 0 ? userMessageParts.join(" ") : null; const shouldStart = command !== "chat" || userMessage !== null; + // Canonicalize the requested locale here so an unrecognized value is dropped + // (and surfaced as a warning) before it reaches the run or persisted state. + const resolvedLanguage = resolveLanguage(language); + if (command !== "chat" && modeSource === "default") { mode = "code"; } @@ -582,7 +588,8 @@ function parseRunCommand( exitCode: 0, command, dryRun, - language, + language: resolvedLanguage.language ?? null, + languageWarning: resolvedLanguage.warning ?? null, mode, modeSource, modelId, diff --git a/src/language.ts b/src/language.ts new file mode 100644 index 00000000..897df165 --- /dev/null +++ b/src/language.ts @@ -0,0 +1,60 @@ +/** + * Result of resolving a user-supplied output-language string. + */ +export interface ResolvedLanguage { + /** + * The canonical BCP-47 tag when the input is a recognized language, and + * undefined when the input was empty or not recognized (fall back to English). + * + * @default undefined + */ + language?: string; + + /** + * A user-facing warning, set only when a non-empty input was provided but + * could not be recognized as a real language. + * + * @default undefined + */ + warning?: string; +} + +/** + * Validates and canonicalizes an output-language flag using only the built-in + * Intl APIs, so no dependency is added. + * + * getCanonicalLocales rejects malformed tags (wrong length, digits, underscores) + * by throwing, and DisplayNames distinguishes recognized codes from + * structurally valid but unknown ones (for example "xx" or "english") by + * echoing the input back instead of returning a real language name. + * + * An unrecognized value resolves to no language plus a warning, so callers fall + * back to English rather than persisting or prompting with garbage. + */ +export function resolveLanguage( + input: string | null | undefined, +): ResolvedLanguage { + const trimmed = input?.trim(); + + if (!trimmed) { + return {}; + } + + try { + const [canonical] = Intl.getCanonicalLocales(trimmed); + const primary = new Intl.Locale(canonical).language; + const displayName = new Intl.DisplayNames(["en"], { + type: "language", + }).of(primary); + + if (displayName && displayName.toLowerCase() !== primary.toLowerCase()) { + return { language: canonical }; + } + } catch { + // Malformed tag: fall through to the unrecognized-language warning. + } + + return { + warning: `Unrecognized language "${trimmed}"; generating in English. Use a BCP-47 code such as zh-CN, hi, or pt-BR.`, + }; +} diff --git a/test/commands.test.ts b/test/commands.test.ts index 1c5ecdbf..c03e8d9f 100644 --- a/test/commands.test.ts +++ b/test/commands.test.ts @@ -214,6 +214,23 @@ describe("parseCommand — init/update", () => { }); }); + test("--language canonicalizes the locale and sets no warning", () => { + expect(parseCommand(["--init", "--language", "PT-br"])).toMatchObject({ + kind: "run", + language: "pt-BR", + languageWarning: null, + }); + }); + + test("an unrecognized --language is dropped and warned", () => { + const result = parseCommand(["--init", "--language", "fake-language"]); + + expect(result).toMatchObject({ kind: "run", language: null }); + if (result.kind === "run") { + expect(result.languageWarning).toContain("fake-language"); + } + }); + test("personal --init selects the init command and starts", () => { expect(parseCommand(["personal", "--init"])).toMatchObject({ kind: "run", diff --git a/test/language.test.ts b/test/language.test.ts new file mode 100644 index 00000000..3964bdf0 --- /dev/null +++ b/test/language.test.ts @@ -0,0 +1,33 @@ +import { describe, expect, test } from "vitest"; +import { resolveLanguage } from "../src/language.ts"; + +describe("resolveLanguage", () => { + test("canonicalizes recognized BCP-47 codes", () => { + expect(resolveLanguage("zh-CN")).toEqual({ language: "zh-CN" }); + expect(resolveLanguage("hi")).toEqual({ language: "hi" }); + expect(resolveLanguage("PT-br")).toEqual({ language: "pt-BR" }); + expect(resolveLanguage(" en-US ")).toEqual({ language: "en-US" }); + }); + + test("returns nothing for empty or missing input", () => { + expect(resolveLanguage(undefined)).toEqual({}); + expect(resolveLanguage(null)).toEqual({}); + expect(resolveLanguage(" ")).toEqual({}); + }); + + test("warns and drops malformed tags", () => { + const result = resolveLanguage("fake-language"); + + expect(result.language).toBeUndefined(); + expect(result.warning).toContain("fake-language"); + }); + + test("warns and drops structurally valid but unknown codes", () => { + for (const unknown of ["xx", "english"]) { + const result = resolveLanguage(unknown); + + expect(result.language, unknown).toBeUndefined(); + expect(result.warning, unknown).toBeTruthy(); + } + }); +}); From 54168f94e14ea323b54651ab4708191ae4e0f2ec Mon Sep 17 00:00:00 2001 From: Colin Francis Date: Sun, 26 Jul 2026 19:56:28 -0700 Subject: [PATCH 03/13] persist and inherit the wiki output language --- src/agent/index.ts | 3 ++ src/agent/types.ts | 1 + src/agent/utils.ts | 27 ++++++++++-- test/run-context.test.ts | 94 +++++++++++++++++++++++++++++++++++++++- 4 files changed, 120 insertions(+), 5 deletions(-) diff --git a/src/agent/index.ts b/src/agent/index.ts index b3be11a2..f86761ce 100644 --- a/src/agent/index.ts +++ b/src/agent/index.ts @@ -359,6 +359,7 @@ async function runOpenWikiAgentCore( outputMode, openWikiSnapshotBefore, "interrupted", + context.language, ); emitDebug( options, @@ -383,6 +384,8 @@ async function runOpenWikiAgentCore( modelId, outputMode, openWikiSnapshotBefore, + "complete", + context.language, ); if (metadataWritten) { diff --git a/src/agent/types.ts b/src/agent/types.ts index a00ef2f2..74ab8cf6 100644 --- a/src/agent/types.ts +++ b/src/agent/types.ts @@ -51,6 +51,7 @@ export type UpdateMetadata = { gitHead?: string; model: string; status?: UpdateRunStatus; + language?: string; }; export type RunContext = { diff --git a/src/agent/utils.ts b/src/agent/utils.ts index a54e5e8c..97e26087 100644 --- a/src/agent/utils.ts +++ b/src/agent/utils.ts @@ -8,6 +8,7 @@ import { isExpectedSnapshotRaceError, isFileNotFoundError, } from "../fs-errors.js"; +import { resolveLanguage } from "../language.js"; import { readOpenWikiOnboardingConfig, readRepositoryWikiInstructions, @@ -49,9 +50,13 @@ export async function createRunContext( language?: string | null, ): Promise { const lastUpdate = await readLastUpdate(cwd, outputMode); - const normalizedLanguage = language?.trim() || undefined; - const languageContext = normalizedLanguage - ? { language: normalizedLanguage } + // A validated flag wins; otherwise inherit the wiki's persisted language so an + // update without --language keeps the existing wiki consistent instead of + // producing a mix of the old and new language. + const requestedLanguage = resolveLanguage(language).language; + const effectiveLanguage = requestedLanguage ?? lastUpdate?.language; + const languageContext = effectiveLanguage + ? { language: effectiveLanguage } : {}; const wikiGoal = await readRunWikiGoal(cwd, outputMode); @@ -163,6 +168,7 @@ export async function writeLastUpdateMetadata( modelId: string, outputMode: OpenWikiOutputMode = "repository", status: UpdateRunStatus = "complete", + language?: string, ): Promise { const metadataFile = getMetadataFilePath(cwd, outputMode); const metadata: UpdateMetadata = { @@ -171,6 +177,7 @@ export async function writeLastUpdateMetadata( gitHead: outputMode === "repository" ? await getGitHead(cwd) : undefined, model: modelId, status, + ...(language ? { language } : {}), }; await mkdir(path.dirname(metadataFile), { recursive: true }); @@ -193,6 +200,7 @@ export async function persistRunMetadataIfChanged( outputMode: OpenWikiOutputMode, snapshotBefore: OpenWikiContentSnapshot | null, status: UpdateRunStatus = "complete", + language?: string, ): Promise { if (command === "chat" || snapshotBefore === null) { return false; @@ -209,7 +217,14 @@ export async function persistRunMetadataIfChanged( } } - await writeLastUpdateMetadata(command, cwd, modelId, outputMode, status); + await writeLastUpdateMetadata( + command, + cwd, + modelId, + outputMode, + status, + language, + ); return true; } @@ -280,6 +295,10 @@ async function readLastUpdate( // complete so upgrades do not force a spurious re-run. status: parsedMetadata.status === "interrupted" ? "interrupted" : "complete", + language: + typeof parsedMetadata.language === "string" + ? parsedMetadata.language + : undefined, }; } diff --git a/test/run-context.test.ts b/test/run-context.test.ts index c9013cdf..95f1ad6a 100644 --- a/test/run-context.test.ts +++ b/test/run-context.test.ts @@ -2,7 +2,10 @@ import { mkdtemp, rm } from "node:fs/promises"; import { tmpdir } from "node:os"; import path from "node:path"; import { describe, expect, test } from "vitest"; -import { createRunContext } from "../src/agent/utils.ts"; +import { + createRunContext, + writeLastUpdateMetadata, +} from "../src/agent/utils.ts"; describe("createRunContext output language", () => { test("propagates a selected language and leaves it unset by default", async () => { @@ -21,4 +24,93 @@ describe("createRunContext output language", () => { await rm(cwd, { recursive: true, force: true }); } }); + + test("canonicalizes the selected language", async () => { + const cwd = await mkdtemp(path.join(tmpdir(), "openwiki-run-context-")); + + try { + await expect( + createRunContext("chat", cwd, "local-wiki", "PT-br"), + ).resolves.toMatchObject({ language: "pt-BR" }); + } finally { + await rm(cwd, { recursive: true, force: true }); + } + }); + + test("ignores an unrecognized language and falls back to English", async () => { + const cwd = await mkdtemp(path.join(tmpdir(), "openwiki-run-context-")); + + try { + expect( + await createRunContext("chat", cwd, "local-wiki", "fake-language"), + ).not.toHaveProperty("language"); + } finally { + await rm(cwd, { recursive: true, force: true }); + } + }); +}); + +describe("createRunContext language inheritance", () => { + test("inherits the persisted wiki language when no flag is given", async () => { + const cwd = await mkdtemp(path.join(tmpdir(), "openwiki-run-context-")); + + try { + await writeLastUpdateMetadata( + "update", + cwd, + "model-x", + "local-wiki", + "complete", + "zh-CN", + ); + + expect(await createRunContext("chat", cwd, "local-wiki")).toMatchObject({ + language: "zh-CN", + }); + } finally { + await rm(cwd, { recursive: true, force: true }); + } + }); + + test("an explicit flag overrides the persisted language", async () => { + const cwd = await mkdtemp(path.join(tmpdir(), "openwiki-run-context-")); + + try { + await writeLastUpdateMetadata( + "update", + cwd, + "model-x", + "local-wiki", + "complete", + "zh-CN", + ); + + expect( + await createRunContext("chat", cwd, "local-wiki", "hi"), + ).toMatchObject({ language: "hi" }); + } finally { + await rm(cwd, { recursive: true, force: true }); + } + }); + + test("an unrecognized flag falls back to the persisted language", async () => { + const cwd = await mkdtemp(path.join(tmpdir(), "openwiki-run-context-")); + + try { + await writeLastUpdateMetadata( + "update", + cwd, + "model-x", + "local-wiki", + "complete", + "zh-CN", + ); + + expect( + await createRunContext("chat", cwd, "local-wiki", "not-a-language"), + ).toMatchObject({ language: "zh-CN" }); + } finally { + await rm(cwd, { recursive: true, force: true }); + } + }); }); From ec900cb99653a5a5933eb73cd4bd90a33ae81f7d Mon Sep 17 00:00:00 2001 From: Colin Francis Date: Sun, 26 Jul 2026 20:02:14 -0700 Subject: [PATCH 04/13] retranslate the wiki when update switches --language --- src/agent/index.ts | 28 ++- src/agent/translation-middleware.ts | 259 ++++++++++++++++++++++++++++ 2 files changed, 286 insertions(+), 1 deletion(-) create mode 100644 src/agent/translation-middleware.ts diff --git a/src/agent/index.ts b/src/agent/index.ts index f86761ce..fe58fb9e 100644 --- a/src/agent/index.ts +++ b/src/agent/index.ts @@ -24,8 +24,13 @@ import { import { isFileNotFoundError } from "../fs-errors.js"; import { SECRET_KEY_PATTERN_SOURCE } from "../diagnostics.js"; import { openWikiLocalWikiDir, openWikiSkillsDir } from "../openwiki-home.js"; +import { resolveLanguage } from "../language.js"; import { OpenWikiLocalShellBackend } from "./docs-only-backend.js"; import { createOpenWikiIndexMiddleware } from "./okf-middleware.js"; +import { + createWikiTranslationMiddleware, + resolveTranslationSwitch, +} from "./translation-middleware.js"; import { CODEX_ORIGINATOR, CODEX_RESPONSES_BASE_URL, @@ -283,6 +288,15 @@ async function runOpenWikiAgentCore( virtualMode: true, }), }); + // An update inherits the wiki's persisted language unless --language requests a + // different one. When it does, retranslate every existing page before the agent + // runs so the incremental update does not leave a mix of the old and new + // language. + const translation = resolveTranslationSwitch( + command, + resolveLanguage(options.language).language, + context.lastUpdate?.language, + ); const agent = createDeepAgent({ model, tools: createOpenWikiConnectorTools(), @@ -291,7 +305,19 @@ async function runOpenWikiAgentCore( middleware: command === "chat" ? [] - : [createOpenWikiIndexMiddleware(wikiBackend, outputMode)], + : [ + ...(translation + ? [ + createWikiTranslationMiddleware( + wikiBackend, + outputMode, + model, + translation, + ), + ] + : []), + createOpenWikiIndexMiddleware(wikiBackend, outputMode), + ], skills: ["/skills/"], permissions: [ { operations: ["write"], paths: ["/skills/**"], mode: "deny" }, diff --git a/src/agent/translation-middleware.ts b/src/agent/translation-middleware.ts new file mode 100644 index 00000000..cfaeeb3b --- /dev/null +++ b/src/agent/translation-middleware.ts @@ -0,0 +1,259 @@ +import type { BaseChatModel } from "@langchain/core/language_models/chat_models"; +import type { MessageContent } from "@langchain/core/messages"; +import { HumanMessage, SystemMessage } from "@langchain/core/messages"; +import type { BackendProtocolV2, FileInfo } from "deepagents"; +import { createMiddleware } from "langchain"; +import path from "node:path"; +import type { OpenWikiCommand, OpenWikiOutputMode } from "./types.js"; + +/** + * Wiki files that are regenerated deterministically (indexes) or are not + * human-facing prose (logs, scratch plans), so they are never translated. + */ +const EXCLUDED_FILES = new Set([ + "index.md", + "log.md", + "_plan.md", + "INSTRUCTIONS.md", +]); + +/** + * The source and target languages for a translate-all pass, as canonical + * BCP-47 tags (for example `en` and `zh-CN`). + */ +export interface WikiTranslationOptions { + /** + * The language the wiki is currently written in. + */ + from: string; + + /** + * The language every page should be rewritten into. + */ + to: string; +} + +/** + * Decides whether a run must retranslate the existing wiki, and between which + * languages. + * + * A translate-all pass is warranted only for an `update` that explicitly + * requests a language whose primary subtag differs from the wiki's persisted + * one (an absent persisted language means the wiki is English). Comparing + * primary subtags avoids a needless full retranslation for a region-only change + * such as `en` to `en-GB`. Returns the source and target languages when a switch + * is needed, or `undefined` otherwise. + */ +export function resolveTranslationSwitch( + command: OpenWikiCommand, + requestedLanguage: string | undefined, + currentWikiLanguage: string | undefined, +): WikiTranslationOptions | undefined { + if (command !== "update" || requestedLanguage === undefined) { + return undefined; + } + if (primarySubtag(requestedLanguage) === primarySubtag(currentWikiLanguage)) { + return undefined; + } + return { from: currentWikiLanguage ?? "en", to: requestedLanguage }; +} + +/** + * Returns a language tag's primary subtag (for example `zh` for `zh-CN`), + * treating an absent wiki language as English. + */ +function primarySubtag(tag: string | undefined): string { + if (!tag) return "en"; + try { + return new Intl.Locale(tag).language; + } catch { + return tag; + } +} + +/** + * Creates middleware that retranslates every existing wiki page before the + * agent runs. + * + * OpenWiki treats the wiki's language as persisted state: an update inherits it + * unless `--language` requests a different one. When it differs, the agent's + * incremental update alone would leave a mix of the old and new language, since + * it only rewrites pages whose source changed. This `beforeAgent` hook closes + * that gap by walking the wiki up front and rewriting each page into the target + * language, so the agent then edits an already-uniform wiki. + * + * The model's output is treated purely as file text and written back through + * the sandboxed docs-only backend; it is never executed, and every path comes + * from backend enumeration rather than model output. + */ +export function createWikiTranslationMiddleware( + backend: BackendProtocolV2, + outputMode: OpenWikiOutputMode, + model: BaseChatModel, + languages: WikiTranslationOptions, +) { + return createMiddleware({ + name: "OpenWikiTranslationMiddleware", + beforeAgent: async () => { + await translateWiki(backend, outputMode, model, languages); + }, + }); +} + +/** + * Rewrites every concept page in the wiki from one language into another. + */ +async function translateWiki( + backend: BackendProtocolV2, + outputMode: OpenWikiOutputMode, + model: BaseChatModel, + languages: WikiTranslationOptions, +): Promise { + const root = outputMode === "local-wiki" ? "/" : "/openwiki"; + for (const filePath of await collectMarkdownFiles(backend, root)) { + await translateFile(backend, model, filePath, languages); + } +} + +/** + * Translates one concept file in place, skipping writes that would not change + * the content. + */ +async function translateFile( + backend: BackendProtocolV2, + model: BaseChatModel, + filePath: string, + languages: WikiTranslationOptions, +): Promise { + const original = await readText(backend, filePath); + if (!original.trim()) return; + + const translated = await translateMarkdown(model, original, languages); + if (!translated.trim() || translated.trim() === original.trim()) return; + + const result = await backend.edit(filePath, original, translated); + if (result.error) { + throw new Error(`Unable to translate ${filePath}: ${result.error}`); + } +} + +/** + * Asks the model to translate a Markdown document, returning its raw text. + */ +async function translateMarkdown( + model: BaseChatModel, + content: string, + { from, to }: WikiTranslationOptions, +): Promise { + const response = await model.invoke([ + new SystemMessage(buildTranslationPrompt(from, to)), + new HumanMessage(content), + ]); + return extractText(response.content); +} + +/** + * Builds the system instruction that constrains the translation to prose while + * preserving Markdown structure and code. + */ +function buildTranslationPrompt(from: string, to: string): string { + return `You are a professional technical translator for a software documentation wiki. +Translate the Markdown document provided by the user from ${describeLanguage( + from, + )} into ${describeLanguage(to)}. + +Rules: +- Translate prose, headings, list items, blockquotes, and table cell text. +- In the YAML front matter, translate only the human-readable "title" and "description" values. Keep every front matter key and all other values byte-for-byte unchanged. +- Do NOT translate code identifiers, file paths, commands, API names, URLs, or anything inside inline code spans or fenced code blocks. +- Preserve all Markdown syntax, link targets, mermaid fences, and the document's whitespace and structure. +- Return ONLY the translated document text, with no explanation, commentary, or surrounding code fences.`; +} + +/** + * Renders a language tag for the prompt as its English display name (which + * already includes any region, for example "Chinese (China)"), falling back to + * the bare tag when no name is known. + */ +function describeLanguage(tag: string): string { + try { + const name = new Intl.DisplayNames(["en"], { type: "language" }).of(tag); + if (name && name.toLowerCase() !== tag.toLowerCase()) { + return name; + } + } catch { + // Malformed tag: fall through to the bare tag. + } + return tag; +} + +/** + * Flattens model message content into plain text, joining any content blocks. + */ +function extractText(content: MessageContent): string { + if (typeof content === "string") return content; + return content + .map((block) => + typeof block === "string" + ? block + : block.type === "text" + ? block.text + : "", + ) + .join(""); +} + +/** + * Recursively collects visible, translatable Markdown files under a directory. + */ +async function collectMarkdownFiles( + backend: BackendProtocolV2, + directoryPath: string, +): Promise { + const result = await backend.ls(directoryPath); + if (result.error) return []; + + const files: string[] = []; + for (const entry of result.files ?? []) { + const name = entryName(entry); + if (!name || name.startsWith(".")) continue; + + const entryPath = path.posix.join(directoryPath, name); + if (entry.is_dir) { + files.push(...(await collectMarkdownFiles(backend, entryPath))); + continue; + } + if ( + path.posix.extname(name).toLowerCase() === ".md" && + !EXCLUDED_FILES.has(name) + ) { + files.push(entryPath); + } + } + return files; +} + +/** + * Reads a text file from the backend or throws an actionable error. + */ +async function readText( + backend: BackendProtocolV2, + filePath: string, +): Promise { + const result = await backend.readRaw(filePath); + if (result.error) { + throw new Error(`Unable to read ${filePath}: ${result.error}`); + } + + const content = result.data?.content; + if (Array.isArray(content)) return content.join("\n"); + if (typeof content === "string") return content; + throw new Error(`${filePath} is not a text file.`); +} + +/** + * Extracts an entry's basename from its virtual path. + */ +function entryName(entry: FileInfo): string { + return path.posix.basename(entry.path.replace(/\/$/u, "")); +} From 81a0d5b2aad3825b0b10003a20d8f21816407887 Mon Sep 17 00:00:00 2001 From: Colin Francis Date: Sun, 26 Jul 2026 20:13:12 -0700 Subject: [PATCH 05/13] localize the deterministic index section headings --- src/agent/index.ts | 6 +- src/agent/okf-middleware.ts | 4 +- src/okf/index-labels.ts | 106 +++++++++++++++ src/okf/index-sync.ts | 19 ++- test/index-labels.test.ts | 54 ++++++++ test/index-middleware.test.ts | 31 +++++ test/translation-middleware.test.ts | 200 ++++++++++++++++++++++++++++ 7 files changed, 413 insertions(+), 7 deletions(-) create mode 100644 src/okf/index-labels.ts create mode 100644 test/index-labels.test.ts create mode 100644 test/translation-middleware.test.ts diff --git a/src/agent/index.ts b/src/agent/index.ts index fe58fb9e..a546fd93 100644 --- a/src/agent/index.ts +++ b/src/agent/index.ts @@ -25,6 +25,7 @@ import { isFileNotFoundError } from "../fs-errors.js"; import { SECRET_KEY_PATTERN_SOURCE } from "../diagnostics.js"; import { openWikiLocalWikiDir, openWikiSkillsDir } from "../openwiki-home.js"; import { resolveLanguage } from "../language.js"; +import { resolveIndexLabels } from "../okf/index-labels.js"; import { OpenWikiLocalShellBackend } from "./docs-only-backend.js"; import { createOpenWikiIndexMiddleware } from "./okf-middleware.js"; import { @@ -297,6 +298,9 @@ async function runOpenWikiAgentCore( resolveLanguage(options.language).language, context.lastUpdate?.language, ); + // Localized headings for the deterministic directory indexes. Falls back to + // English for any language not in the static map. + const indexLabels = resolveIndexLabels(context.language); const agent = createDeepAgent({ model, tools: createOpenWikiConnectorTools(), @@ -316,7 +320,7 @@ async function runOpenWikiAgentCore( ), ] : []), - createOpenWikiIndexMiddleware(wikiBackend, outputMode), + createOpenWikiIndexMiddleware(wikiBackend, outputMode, indexLabels), ], skills: ["/skills/"], permissions: [ diff --git a/src/agent/okf-middleware.ts b/src/agent/okf-middleware.ts index 6351668a..7c25a55f 100644 --- a/src/agent/okf-middleware.ts +++ b/src/agent/okf-middleware.ts @@ -8,6 +8,7 @@ import { type FrontmatterIssue, } from "../okf/frontmatter.js"; import { migrateWikiToOkf, synchronizeWikiIndexes } from "../okf/index-sync.js"; +import { ENGLISH_INDEX_LABELS, type IndexLabels } from "../okf/index-labels.js"; import { MUTATION_PATH_METADATA_KEY } from "./docs-only-backend.js"; import type { OpenWikiOutputMode } from "./types.js"; @@ -22,6 +23,7 @@ const WRITE_TOOLS = new Set(["write_file", "edit_file"]); export function createOpenWikiIndexMiddleware( backend: BackendProtocolV2, outputMode: OpenWikiOutputMode, + labels: IndexLabels = ENGLISH_INDEX_LABELS, ) { return createMiddleware({ name: "OpenWikiIndexMiddleware", @@ -37,7 +39,7 @@ export function createOpenWikiIndexMiddleware( ), afterAgent: async () => { await validateWikiMermaid(backend, outputMode); - await synchronizeWikiIndexes(backend, outputMode); + await synchronizeWikiIndexes(backend, outputMode, labels); }, }); } diff --git a/src/okf/index-labels.ts b/src/okf/index-labels.ts new file mode 100644 index 00000000..3da9f0f4 --- /dev/null +++ b/src/okf/index-labels.ts @@ -0,0 +1,106 @@ +/** + * Localized labels for the two deterministic index sections that the OKF index + * sync renders in every directory's `index.md`. + */ +export interface IndexLabels { + /** + * Heading for the list of concept pages in a directory (English "Files"). + */ + files: string; + + /** + * Heading for the list of subdirectories in a directory (English + * "Directories"). + */ + directories: string; +} + +/** + * Built-in English labels, used as the structural default whenever a wiki's + * language is English or has no localized entry in the map below. + */ +export const ENGLISH_INDEX_LABELS: IndexLabels = { + files: "Files", + directories: "Directories", +}; + +/** + * Localized index section headings keyed by BCP-47 language tag. + * + * These two words are structural navigation chrome rather than wiki prose, so + * they are curated here instead of translated per run: the lookup stays + * deterministic and model-free, and an unlisted language falls back to English. + * + * To localize a language, add one entry. Region-specific forms (for example + * `zh-CN` versus `zh-TW`) may key on the full tag; a bare primary-subtag entry + * (for example `zh`) supplies the default for every region of that language. + */ +const INDEX_LABELS: Record = { + en: ENGLISH_INDEX_LABELS, + ar: { files: "ملفات", directories: "مجلدات" }, + bg: { files: "Файлове", directories: "Директории" }, + ca: { files: "Fitxers", directories: "Directoris" }, + cs: { files: "Soubory", directories: "Adresáře" }, + da: { files: "Filer", directories: "Mapper" }, + de: { files: "Dateien", directories: "Verzeichnisse" }, + el: { files: "Αρχεία", directories: "Κατάλογοι" }, + es: { files: "Archivos", directories: "Directorios" }, + fi: { files: "Tiedostot", directories: "Hakemistot" }, + fr: { files: "Fichiers", directories: "Répertoires" }, + he: { files: "קבצים", directories: "תיקיות" }, + hi: { files: "फ़ाइलें", directories: "निर्देशिकाएँ" }, + hr: { files: "Datoteke", directories: "Direktoriji" }, + hu: { files: "Fájlok", directories: "Könyvtárak" }, + id: { files: "Berkas", directories: "Direktori" }, + it: { files: "File", directories: "Cartelle" }, + ja: { files: "ファイル", directories: "ディレクトリ" }, + ko: { files: "파일", directories: "디렉터리" }, + ms: { files: "Fail", directories: "Direktori" }, + nb: { files: "Filer", directories: "Mapper" }, + nl: { files: "Bestanden", directories: "Mappen" }, + no: { files: "Filer", directories: "Mapper" }, + pl: { files: "Pliki", directories: "Katalogi" }, + pt: { files: "Arquivos", directories: "Diretórios" }, + "pt-PT": { files: "Ficheiros", directories: "Diretórios" }, + ro: { files: "Fișiere", directories: "Directoare" }, + ru: { files: "Файлы", directories: "Каталоги" }, + sk: { files: "Súbory", directories: "Adresáre" }, + sl: { files: "Datoteke", directories: "Mape" }, + sr: { files: "Датотеке", directories: "Директоријуми" }, + sv: { files: "Filer", directories: "Kataloger" }, + th: { files: "ไฟล์", directories: "ไดเรกทอรี" }, + tr: { files: "Dosyalar", directories: "Dizinler" }, + uk: { files: "Файли", directories: "Каталоги" }, + vi: { files: "Tập tin", directories: "Thư mục" }, + zh: { files: "文件", directories: "目录" }, + "zh-TW": { files: "檔案", directories: "目錄" }, +}; + +/** + * Resolves the index section headings for a wiki language. + * + * Tries the full canonical tag, then its primary subtag, then falls back to the + * built-in English labels, so an unlisted or malformed language degrades to + * English structural headings rather than failing. For example `pt-BR` resolves + * to the `pt` default while `pt-PT` uses its own override. + */ +export function resolveIndexLabels(language: string | undefined): IndexLabels { + if (!language) return ENGLISH_INDEX_LABELS; + return ( + INDEX_LABELS[language] ?? + INDEX_LABELS[primarySubtag(language)] ?? + ENGLISH_INDEX_LABELS + ); +} + +/** + * Returns a language tag's primary subtag (for example `zh` for `zh-CN`), + * echoing the input back when it cannot be parsed. + */ +function primarySubtag(tag: string): string { + try { + return new Intl.Locale(tag).language; + } catch { + return tag; + } +} diff --git a/src/okf/index-sync.ts b/src/okf/index-sync.ts index c39d3a5c..9b0aca5e 100644 --- a/src/okf/index-sync.ts +++ b/src/okf/index-sync.ts @@ -1,6 +1,7 @@ import type { BackendProtocolV2, FileInfo } from "deepagents"; import path from "node:path"; import type { OpenWikiOutputMode } from "../agent/types.js"; +import { ENGLISH_INDEX_LABELS, type IndexLabels } from "./index-labels.js"; import { normalizeConceptContent, parseFrontmatterFields, @@ -56,10 +57,11 @@ interface Link { export async function synchronizeWikiIndexes( backend: BackendProtocolV2, outputMode: OpenWikiOutputMode, + labels: IndexLabels = ENGLISH_INDEX_LABELS, ): Promise { const root = outputMode === "local-wiki" ? "/" : "/openwiki"; for (const directory of await collectDirectories(backend, root, true)) { - await synchronizeDirectory(backend, directory, root); + await synchronizeDirectory(backend, directory, root, labels); } } @@ -155,6 +157,7 @@ async function synchronizeDirectory( backend: BackendProtocolV2, directory: Directory, root: string, + labels: IndexLabels, ): Promise { const files: Link[] = []; const directories: Link[] = []; @@ -185,7 +188,12 @@ async function synchronizeDirectory( } const indexPath = path.posix.join(directory.path, INDEX_FILE); - const content = renderIndex(files, directories, directory.path === root); + const content = renderIndex( + files, + directories, + directory.path === root, + labels, + ); const existing = directory.entries.some( (entry) => !entry.is_dir && entryName(entry) === INDEX_FILE, ) @@ -208,15 +216,16 @@ function renderIndex( files: Link[], directories: Link[], isRoot: boolean, + labels: IndexLabels, ): string { const sections = [ - renderLinks("Files", files, true), - renderLinks("Directories", directories, false), + renderLinks(labels.files, files, true), + renderLinks(labels.directories, directories, false), ] .filter(Boolean) .join("\n\n"); const version = isRoot ? '---\nokf_version: "0.1"\n---\n\n' : ""; - return `${version}${sections || "# Files"}\n`; + return `${version}${sections || `# ${labels.files}`}\n`; } /** diff --git a/test/index-labels.test.ts b/test/index-labels.test.ts new file mode 100644 index 00000000..fa139dc0 --- /dev/null +++ b/test/index-labels.test.ts @@ -0,0 +1,54 @@ +import { describe, expect, test } from "vitest"; +import { resolveIndexLabels } from "../src/okf/index-labels.ts"; + +const ENGLISH = { files: "Files", directories: "Directories" }; + +describe("resolveIndexLabels", () => { + test("returns English for English, empty, or an unlisted language", () => { + expect(resolveIndexLabels(undefined)).toEqual(ENGLISH); + expect(resolveIndexLabels("en")).toEqual(ENGLISH); + expect(resolveIndexLabels("en-US")).toEqual(ENGLISH); + expect(resolveIndexLabels("tlh")).toEqual(ENGLISH); // Klingon: not seeded + }); + + test("localizes a listed language", () => { + expect(resolveIndexLabels("hr")).toEqual({ + files: "Datoteke", + directories: "Direktoriji", + }); + expect(resolveIndexLabels("ja")).toEqual({ + files: "ファイル", + directories: "ディレクトリ", + }); + }); + + test("falls back from a region tag to its primary subtag", () => { + expect(resolveIndexLabels("hr-HR")).toEqual({ + files: "Datoteke", + directories: "Direktoriji", + }); + }); + + test("honors region overrides and primary-subtag defaults", () => { + expect(resolveIndexLabels("zh-CN")).toEqual({ + files: "文件", + directories: "目录", + }); + expect(resolveIndexLabels("zh-TW")).toEqual({ + files: "檔案", + directories: "目錄", + }); + expect(resolveIndexLabels("pt-BR")).toEqual({ + files: "Arquivos", + directories: "Diretórios", + }); + expect(resolveIndexLabels("pt-PT")).toEqual({ + files: "Ficheiros", + directories: "Diretórios", + }); + }); + + test("degrades to English on a malformed tag", () => { + expect(resolveIndexLabels("!!")).toEqual(ENGLISH); + }); +}); diff --git a/test/index-middleware.test.ts b/test/index-middleware.test.ts index 4c4750b2..db2029b6 100644 --- a/test/index-middleware.test.ts +++ b/test/index-middleware.test.ts @@ -68,6 +68,37 @@ describe("synchronizeWikiIndexes", () => { ); }); + test("renders localized section headings when labels are supplied", async () => { + const { backend, rootDir } = await setup(); + await backend.write( + "/openwiki/quickstart.md", + document("Brzi početak", "Počnite ovdje."), + ); + await backend.write( + "/openwiki/architecture/overview.md", + document("Pregled", "Struktura sustava."), + ); + + await synchronizeWikiIndexes(backend, "repository", { + files: "Datoteke", + directories: "Direktoriji", + }); + + const rootIndex = await readFile( + path.join(rootDir, "openwiki/index.md"), + "utf8", + ); + + expect(rootIndex).toContain( + "# Datoteke\n\n- [Brzi početak](quickstart.md)", + ); + expect(rootIndex).toContain( + "# Direktoriji\n\n- [architecture](architecture/)", + ); + expect(rootIndex).not.toContain("# Files"); + expect(rootIndex).not.toContain("# Directories"); + }); + test("uses OKF version frontmatter only at the bundle root", async () => { const { backend, rootDir } = await setup(); await backend.write( diff --git a/test/translation-middleware.test.ts b/test/translation-middleware.test.ts new file mode 100644 index 00000000..2052a71a --- /dev/null +++ b/test/translation-middleware.test.ts @@ -0,0 +1,200 @@ +import { AIMessage, type BaseMessage } from "@langchain/core/messages"; +import type { BaseChatModel } from "@langchain/core/language_models/chat_models"; +import { mkdir, mkdtemp, readFile, writeFile } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { describe, expect, test, vi } from "vitest"; +import { OpenWikiLocalShellBackend } from "../src/agent/docs-only-backend.ts"; +import { + createWikiTranslationMiddleware, + resolveTranslationSwitch, +} from "../src/agent/translation-middleware.ts"; + +/** + * Records every prompt the middleware sends and returns a scripted translation + * for each page body, without any network access. + */ +function fakeModel(translate: (content: string) => string) { + const calls: { system: string; human: string }[] = []; + const asText = (message: BaseMessage): string => + typeof message.content === "string" ? message.content : ""; + const model = { + invoke: (messages: BaseMessage[]) => { + const [system, human] = messages; + calls.push({ system: asText(system), human: asText(human) }); + return Promise.resolve(new AIMessage(translate(asText(human)))); + }, + }; + return { model: model as unknown as BaseChatModel, calls }; +} + +async function setup(outputMode: "local-wiki" | "repository" = "repository") { + const rootDir = await mkdtemp(path.join(os.tmpdir(), "openwiki-translate-")); + const backend = new OpenWikiLocalShellBackend({ + docsOnly: true, + outputMode, + rootDir, + virtualMode: true, + }); + return { backend, rootDir }; +} + +/** + * Invokes a middleware's beforeAgent hook regardless of its exact shape. + */ +async function runBeforeAgent( + middleware: ReturnType, +): Promise { + const beforeAgent = + typeof middleware.beforeAgent === "function" + ? middleware.beforeAgent + : middleware.beforeAgent?.hook; + expect(beforeAgent).toBeTypeOf("function"); + await (beforeAgent as () => Promise)(); +} + +describe("resolveTranslationSwitch", () => { + test("triggers when an update requests a different language", () => { + expect(resolveTranslationSwitch("update", "zh-CN", "en")).toEqual({ + from: "en", + to: "zh-CN", + }); + }); + + test("treats an absent persisted language as English", () => { + expect(resolveTranslationSwitch("update", "zh-CN", undefined)).toEqual({ + from: "en", + to: "zh-CN", + }); + expect( + resolveTranslationSwitch("update", "en-US", undefined), + ).toBeUndefined(); + }); + + test("ignores a region-only change with the same primary subtag", () => { + expect(resolveTranslationSwitch("update", "en-GB", "en")).toBeUndefined(); + expect( + resolveTranslationSwitch("update", "zh-TW", "zh-CN"), + ).toBeUndefined(); + }); + + test("does nothing without a requested language or for non-update commands", () => { + expect(resolveTranslationSwitch("update", undefined, "en")).toBeUndefined(); + expect(resolveTranslationSwitch("init", "zh-CN", "en")).toBeUndefined(); + expect(resolveTranslationSwitch("chat", "zh-CN", "en")).toBeUndefined(); + }); +}); + +describe("createWikiTranslationMiddleware beforeAgent", () => { + test("rewrites every eligible page and passes the original to the model", async () => { + const { backend, rootDir } = await setup(); + await backend.write("/openwiki/quickstart.md", "# Quickstart\n\nHello.\n"); + await backend.write( + "/openwiki/architecture/overview.md", + "# Overview\n\nStructure.\n", + ); + + const { model, calls } = fakeModel((content) => `TRANSLATED\n${content}`); + const middleware = createWikiTranslationMiddleware( + backend, + "repository", + model, + { from: "en", to: "zh-CN" }, + ); + await runBeforeAgent(middleware); + + await expect( + readFile(path.join(rootDir, "openwiki/quickstart.md"), "utf8"), + ).resolves.toBe("TRANSLATED\n# Quickstart\n\nHello.\n"); + await expect( + readFile(path.join(rootDir, "openwiki/architecture/overview.md"), "utf8"), + ).resolves.toBe("TRANSLATED\n# Overview\n\nStructure.\n"); + + // The model saw each page's original content and a prompt naming both langs. + expect(calls).toHaveLength(2); + expect(calls.map((call) => call.human).sort()).toEqual([ + "# Overview\n\nStructure.\n", + "# Quickstart\n\nHello.\n", + ]); + expect(calls[0].system).toContain("Chinese (China)"); + expect(calls[0].system).toContain("English"); + }); + + test("skips indexes, logs, control files, and dotfiles", async () => { + const { backend, rootDir } = await setup(); + const dir = path.join(rootDir, "openwiki"); + await mkdir(path.join(dir, ".hidden"), { recursive: true }); + await backend.write("/openwiki/page.md", "# Page\n\nBody.\n"); + for (const name of ["index.md", "log.md", "_plan.md", "INSTRUCTIONS.md"]) { + await writeFile(path.join(dir, name), "# Control\n\nBody.\n"); + } + await writeFile(path.join(dir, ".secret.md"), "# Secret\n\nBody.\n"); + await writeFile(path.join(dir, ".hidden", "buried.md"), "# Buried\n"); + + const { model, calls } = fakeModel((content) => `X\n${content}`); + await runBeforeAgent( + createWikiTranslationMiddleware(backend, "repository", model, { + from: "en", + to: "hi", + }), + ); + + // Only the one real concept page is translated. + expect(calls).toHaveLength(1); + expect(calls[0].human).toBe("# Page\n\nBody.\n"); + await expect(readFile(path.join(dir, "index.md"), "utf8")).resolves.toBe( + "# Control\n\nBody.\n", + ); + await expect(readFile(path.join(dir, ".secret.md"), "utf8")).resolves.toBe( + "# Secret\n\nBody.\n", + ); + }); + + test("does not write a page whose translation is unchanged", async () => { + const { backend } = await setup(); + await backend.write("/openwiki/page.md", "# Page\n\nBody.\n"); + + const edit = vi.spyOn(backend, "edit"); + const { model } = fakeModel((content) => content); + await runBeforeAgent( + createWikiTranslationMiddleware(backend, "repository", model, { + from: "en", + to: "zh-CN", + }), + ); + + expect(edit).not.toHaveBeenCalled(); + }); + + test("translates from the local-wiki root", async () => { + const { backend, rootDir } = await setup("local-wiki"); + await backend.write("/note.md", "# Note\n\nBody.\n"); + + const { model } = fakeModel((content) => `[hi] ${content}`); + await runBeforeAgent( + createWikiTranslationMiddleware(backend, "local-wiki", model, { + from: "en", + to: "hi", + }), + ); + + await expect(readFile(path.join(rootDir, "note.md"), "utf8")).resolves.toBe( + "[hi] # Note\n\nBody.\n", + ); + }); + + test("is a no-op when the wiki root is missing", async () => { + const { backend } = await setup("repository"); + const { model, calls } = fakeModel((content) => `X\n${content}`); + + await expect( + runBeforeAgent( + createWikiTranslationMiddleware(backend, "repository", model, { + from: "en", + to: "zh-CN", + }), + ), + ).resolves.toBeUndefined(); + expect(calls).toHaveLength(0); + }); +}); From 87e88c1f076a27fa35863687e3272a2da50e53de Mon Sep 17 00:00:00 2001 From: Colin Francis Date: Sun, 26 Jul 2026 20:34:10 -0700 Subject: [PATCH 06/13] persist English as an explicit en in wiki metadata --- src/agent/utils.ts | 9 +++++---- test/run-context.test.ts | 10 +++++----- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/src/agent/utils.ts b/src/agent/utils.ts index 97e26087..016a6ac1 100644 --- a/src/agent/utils.ts +++ b/src/agent/utils.ts @@ -54,10 +54,11 @@ export async function createRunContext( // update without --language keeps the existing wiki consistent instead of // producing a mix of the old and new language. const requestedLanguage = resolveLanguage(language).language; - const effectiveLanguage = requestedLanguage ?? lastUpdate?.language; - const languageContext = effectiveLanguage - ? { language: effectiveLanguage } - : {}; + // English is materialized as "en" rather than encoded by an absent key, so the + // wiki's language is always explicit in metadata and every run inherits a + // concrete value. + const effectiveLanguage = requestedLanguage ?? lastUpdate?.language ?? "en"; + const languageContext = { language: effectiveLanguage }; const wikiGoal = await readRunWikiGoal(cwd, outputMode); if (command === "chat") { diff --git a/test/run-context.test.ts b/test/run-context.test.ts index 95f1ad6a..fd4141c0 100644 --- a/test/run-context.test.ts +++ b/test/run-context.test.ts @@ -8,7 +8,7 @@ import { } from "../src/agent/utils.ts"; describe("createRunContext output language", () => { - test("propagates a selected language and leaves it unset by default", async () => { + test("propagates a selected language and defaults to English", async () => { const cwd = await mkdtemp(path.join(tmpdir(), "openwiki-run-context-")); try { @@ -17,9 +17,9 @@ describe("createRunContext output language", () => { ).resolves.toMatchObject({ language: "zh-CN", }); - expect( - await createRunContext("chat", cwd, "repository"), - ).not.toHaveProperty("language"); + expect(await createRunContext("chat", cwd, "repository")).toMatchObject({ + language: "en", + }); } finally { await rm(cwd, { recursive: true, force: true }); } @@ -43,7 +43,7 @@ describe("createRunContext output language", () => { try { expect( await createRunContext("chat", cwd, "local-wiki", "fake-language"), - ).not.toHaveProperty("language"); + ).toMatchObject({ language: "en" }); } finally { await rm(cwd, { recursive: true, force: true }); } From 38233ef210bc534ecd15fcfe93f90fe495901b45 Mon Sep 17 00:00:00 2001 From: Colin Francis Date: Mon, 27 Jul 2026 07:11:58 -0700 Subject: [PATCH 07/13] localize OKF frontmatter into the target wiki language --- src/agent/index.ts | 18 ++++++-- src/agent/okf-middleware.ts | 11 +++-- src/agent/prompt.ts | 1 + src/agent/translation-middleware.ts | 2 +- src/okf/frontmatter.ts | 12 ++++- src/okf/index-labels.ts | 68 +++++++++++++++++++++++++++++ src/okf/index-sync.ts | 17 ++++++-- test/index-labels.test.ts | 32 +++++++++++++- test/index-middleware.test.ts | 58 ++++++++++++++++++++++++ test/okf/frontmatter.test.ts | 18 +++++++- test/prompt.test.ts | 3 ++ test/translation-middleware.test.ts | 5 +++ 12 files changed, 229 insertions(+), 16 deletions(-) diff --git a/src/agent/index.ts b/src/agent/index.ts index a546fd93..799622d4 100644 --- a/src/agent/index.ts +++ b/src/agent/index.ts @@ -25,7 +25,10 @@ import { isFileNotFoundError } from "../fs-errors.js"; import { SECRET_KEY_PATTERN_SOURCE } from "../diagnostics.js"; import { openWikiLocalWikiDir, openWikiSkillsDir } from "../openwiki-home.js"; import { resolveLanguage } from "../language.js"; -import { resolveIndexLabels } from "../okf/index-labels.js"; +import { + resolveConceptTypeLabel, + resolveIndexLabels, +} from "../okf/index-labels.js"; import { OpenWikiLocalShellBackend } from "./docs-only-backend.js"; import { createOpenWikiIndexMiddleware } from "./okf-middleware.js"; import { @@ -298,9 +301,11 @@ async function runOpenWikiAgentCore( resolveLanguage(options.language).language, context.lastUpdate?.language, ); - // Localized headings for the deterministic directory indexes. Falls back to - // English for any language not in the static map. + // Localized headings for the deterministic directory indexes, plus the + // localized fallback `type` stamped on pages the code has to repair. Both fall + // back to English for any language not in the static maps. const indexLabels = resolveIndexLabels(context.language); + const conceptType = resolveConceptTypeLabel(context.language); const agent = createDeepAgent({ model, tools: createOpenWikiConnectorTools(), @@ -320,7 +325,12 @@ async function runOpenWikiAgentCore( ), ] : []), - createOpenWikiIndexMiddleware(wikiBackend, outputMode, indexLabels), + createOpenWikiIndexMiddleware( + wikiBackend, + outputMode, + indexLabels, + conceptType, + ), ], skills: ["/skills/"], permissions: [ diff --git a/src/agent/okf-middleware.ts b/src/agent/okf-middleware.ts index 7c25a55f..fb8001a1 100644 --- a/src/agent/okf-middleware.ts +++ b/src/agent/okf-middleware.ts @@ -8,7 +8,11 @@ import { type FrontmatterIssue, } from "../okf/frontmatter.js"; import { migrateWikiToOkf, synchronizeWikiIndexes } from "../okf/index-sync.js"; -import { ENGLISH_INDEX_LABELS, type IndexLabels } from "../okf/index-labels.js"; +import { + ENGLISH_CONCEPT_TYPE, + ENGLISH_INDEX_LABELS, + type IndexLabels, +} from "../okf/index-labels.js"; import { MUTATION_PATH_METADATA_KEY } from "./docs-only-backend.js"; import type { OpenWikiOutputMode } from "./types.js"; @@ -24,11 +28,12 @@ export function createOpenWikiIndexMiddleware( backend: BackendProtocolV2, outputMode: OpenWikiOutputMode, labels: IndexLabels = ENGLISH_INDEX_LABELS, + conceptType: string = ENGLISH_CONCEPT_TYPE, ) { return createMiddleware({ name: "OpenWikiIndexMiddleware", beforeAgent: async () => { - await migrateWikiToOkf(backend, outputMode); + await migrateWikiToOkf(backend, outputMode, conceptType); }, wrapToolCall: async (request, handler) => addFrontmatterWarning( @@ -39,7 +44,7 @@ export function createOpenWikiIndexMiddleware( ), afterAgent: async () => { await validateWikiMermaid(backend, outputMode); - await synchronizeWikiIndexes(backend, outputMode, labels); + await synchronizeWikiIndexes(backend, outputMode, labels, conceptType); }, }); } diff --git a/src/agent/prompt.ts b/src/agent/prompt.ts index 422ddcd6..ee2fa212 100644 --- a/src/agent/prompt.ts +++ b/src/agent/prompt.ts @@ -211,6 +211,7 @@ function createLanguageInstructions(language: string | undefined): string { Output language: - Write generated wiki prose, headings, table content, and documentation in ${language}. +- In each page's YAML front matter, write the human-readable "title", "description", "type", and "tags" values in ${language}. Keep the YAML keys and any URL, file-path, timestamp, or identifier-like values unchanged. - Apply this language only to generated wiki files. Do not translate OpenWiki CLI text or runtime messages. - Keep code identifiers, file paths, commands, API names, URLs, and code blocks unchanged where translation would reduce technical accuracy or usability.`; } diff --git a/src/agent/translation-middleware.ts b/src/agent/translation-middleware.ts index cfaeeb3b..4b840778 100644 --- a/src/agent/translation-middleware.ts +++ b/src/agent/translation-middleware.ts @@ -164,7 +164,7 @@ Translate the Markdown document provided by the user from ${describeLanguage( Rules: - Translate prose, headings, list items, blockquotes, and table cell text. -- In the YAML front matter, translate only the human-readable "title" and "description" values. Keep every front matter key and all other values byte-for-byte unchanged. +- In the YAML front matter, translate the human-readable "title", "description", "type", and "tags" values. Keep every front matter key and all other values (URLs, file paths, identifiers, timestamps) byte-for-byte unchanged. - Do NOT translate code identifiers, file paths, commands, API names, URLs, or anything inside inline code spans or fenced code blocks. - Preserve all Markdown syntax, link targets, mermaid fences, and the document's whitespace and structure. - Return ONLY the translated document text, with no explanation, commentary, or surrounding code fences.`; diff --git a/src/okf/frontmatter.ts b/src/okf/frontmatter.ts index d704ef5d..4aa9a35b 100644 --- a/src/okf/frontmatter.ts +++ b/src/okf/frontmatter.ts @@ -241,13 +241,17 @@ function titleFromFilename(filePath: string): string { /** * Derives minimal OKF fields from a page body and its path. + * + * `conceptType` is the localized fallback stamped as the `type`; it defaults to + * English "Reference" for callers that do not localize. */ export function deriveMinimalFrontmatter( body: string, filePath: string, + conceptType: string = "Reference", ): DerivedFrontmatter { return { - type: "Reference", + type: conceptType, title: firstHeading(body) ?? titleFromFilename(filePath), }; } @@ -279,16 +283,20 @@ export function renderFrontmatter( * `title` are junk, so an author's `type` and custom fields are never * overwritten; the index generator already ignores unusable optional fields. * Never throws. Returns the new content and whether it changed. + * + * `conceptType` is the localized fallback used for a repaired page's `type`; it + * defaults to English "Reference" for callers that do not localize. */ export function normalizeConceptContent( content: string, filePath: string, + conceptType: string = "Reference", ): { changed: boolean; content: string } { if (hasUsableConceptType(content)) { return { changed: false, content }; } const { body } = splitFrontmatter(content); - const derived = deriveMinimalFrontmatter(body, filePath); + const derived = deriveMinimalFrontmatter(body, filePath, conceptType); const front = renderFrontmatter(derived, { generated: true }); return { changed: true, content: `${front}${body.replace(/^\s+/u, "")}` }; } diff --git a/src/okf/index-labels.ts b/src/okf/index-labels.ts index 3da9f0f4..c3f9b8f5 100644 --- a/src/okf/index-labels.ts +++ b/src/okf/index-labels.ts @@ -93,6 +93,74 @@ export function resolveIndexLabels(language: string | undefined): IndexLabels { ); } +/** + * Built-in English fallback for the concept `type` the code derives when a page + * has missing or malformed OKF front matter. + */ +export const ENGLISH_CONCEPT_TYPE = "Reference"; + +/** + * Localized forms of the derived concept `type` keyed by BCP-47 language tag. + * + * `deriveMinimalFrontmatter` stamps this single word onto pages whose front + * matter it has to repair, so like the index headings it is curated structural + * chrome rather than translated prose: the lookup stays deterministic and + * model-free, and an unlisted language falls back to English. + * + * To localize a language, add one entry. As with the index labels, a region tag + * may key on its full form while a bare primary-subtag entry supplies the + * default for every region of that language. + */ +const CONCEPT_TYPE_LABELS: Record = { + en: ENGLISH_CONCEPT_TYPE, + ar: "مرجع", + ca: "Referència", + cs: "Reference", + da: "Reference", + de: "Referenz", + el: "Αναφορά", + es: "Referencia", + fr: "Référence", + hi: "संदर्भ", + hr: "Referenca", + id: "Referensi", + it: "Riferimento", + ja: "リファレンス", + ko: "참조", + ms: "Rujukan", + nb: "Referanse", + nl: "Referentie", + no: "Referanse", + pt: "Referência", + ro: "Referință", + ru: "Справочник", + sk: "Referencia", + sr: "Референца", + sv: "Referens", + th: "อ้างอิง", + tr: "Referans", + uk: "Довідник", + vi: "Tham khảo", + zh: "参考", + "zh-TW": "參考", +}; + +/** + * Resolves the derived concept `type` label for a wiki language. + * + * Uses the same full-tag then primary-subtag then English resolution as + * `resolveIndexLabels`, so an unlisted or malformed language degrades to the + * English "Reference" rather than failing. + */ +export function resolveConceptTypeLabel(language: string | undefined): string { + if (!language) return ENGLISH_CONCEPT_TYPE; + return ( + CONCEPT_TYPE_LABELS[language] ?? + CONCEPT_TYPE_LABELS[primarySubtag(language)] ?? + ENGLISH_CONCEPT_TYPE + ); +} + /** * Returns a language tag's primary subtag (for example `zh` for `zh-CN`), * echoing the input back when it cannot be parsed. diff --git a/src/okf/index-sync.ts b/src/okf/index-sync.ts index 9b0aca5e..5b24a3fd 100644 --- a/src/okf/index-sync.ts +++ b/src/okf/index-sync.ts @@ -1,7 +1,11 @@ import type { BackendProtocolV2, FileInfo } from "deepagents"; import path from "node:path"; import type { OpenWikiOutputMode } from "../agent/types.js"; -import { ENGLISH_INDEX_LABELS, type IndexLabels } from "./index-labels.js"; +import { + ENGLISH_CONCEPT_TYPE, + ENGLISH_INDEX_LABELS, + type IndexLabels, +} from "./index-labels.js"; import { normalizeConceptContent, parseFrontmatterFields, @@ -58,10 +62,11 @@ export async function synchronizeWikiIndexes( backend: BackendProtocolV2, outputMode: OpenWikiOutputMode, labels: IndexLabels = ENGLISH_INDEX_LABELS, + conceptType: string = ENGLISH_CONCEPT_TYPE, ): Promise { const root = outputMode === "local-wiki" ? "/" : "/openwiki"; for (const directory of await collectDirectories(backend, root, true)) { - await synchronizeDirectory(backend, directory, root, labels); + await synchronizeDirectory(backend, directory, root, labels, conceptType); } } @@ -77,6 +82,7 @@ export async function synchronizeWikiIndexes( export async function migrateWikiToOkf( backend: BackendProtocolV2, outputMode: OpenWikiOutputMode, + conceptType: string = ENGLISH_CONCEPT_TYPE, ): Promise { const root = outputMode === "local-wiki" ? "/" : "/openwiki"; for (const directory of await collectDirectories(backend, root, true)) { @@ -94,6 +100,7 @@ export async function migrateWikiToOkf( await normalizeConceptFile( backend, path.posix.join(directory.path, name), + conceptType, ); } } @@ -109,9 +116,10 @@ export async function migrateWikiToOkf( async function normalizeConceptFile( backend: BackendProtocolV2, filePath: string, + conceptType: string = ENGLISH_CONCEPT_TYPE, ): Promise { const original = await readText(backend, filePath); - const normalized = normalizeConceptContent(original, filePath); + const normalized = normalizeConceptContent(original, filePath, conceptType); if (normalized.changed) { const result = await backend.edit(filePath, original, normalized.content); if (result.error) { @@ -158,6 +166,7 @@ async function synchronizeDirectory( directory: Directory, root: string, labels: IndexLabels, + conceptType: string, ): Promise { const files: Link[] = []; const directories: Link[] = []; @@ -178,7 +187,7 @@ async function synchronizeDirectory( } const filePath = path.posix.join(directory.path, name); - const content = await normalizeConceptFile(backend, filePath); + const content = await normalizeConceptFile(backend, filePath, conceptType); const metadata = readIndexMetadata(content); files.push({ description: metadata.description, diff --git a/test/index-labels.test.ts b/test/index-labels.test.ts index fa139dc0..04b972d8 100644 --- a/test/index-labels.test.ts +++ b/test/index-labels.test.ts @@ -1,5 +1,8 @@ import { describe, expect, test } from "vitest"; -import { resolveIndexLabels } from "../src/okf/index-labels.ts"; +import { + resolveConceptTypeLabel, + resolveIndexLabels, +} from "../src/okf/index-labels.ts"; const ENGLISH = { files: "Files", directories: "Directories" }; @@ -52,3 +55,30 @@ describe("resolveIndexLabels", () => { expect(resolveIndexLabels("!!")).toEqual(ENGLISH); }); }); + +describe("resolveConceptTypeLabel", () => { + test("returns English for English, empty, or an unlisted language", () => { + expect(resolveConceptTypeLabel(undefined)).toBe("Reference"); + expect(resolveConceptTypeLabel("en")).toBe("Reference"); + expect(resolveConceptTypeLabel("en-US")).toBe("Reference"); + expect(resolveConceptTypeLabel("tlh")).toBe("Reference"); // Klingon: not seeded + }); + + test("localizes a listed language", () => { + expect(resolveConceptTypeLabel("de")).toBe("Referenz"); + expect(resolveConceptTypeLabel("ja")).toBe("リファレンス"); + }); + + test("falls back from a region tag to its primary subtag", () => { + expect(resolveConceptTypeLabel("de-DE")).toBe("Referenz"); + expect(resolveConceptTypeLabel("zh-CN")).toBe("参考"); + }); + + test("honors a region override", () => { + expect(resolveConceptTypeLabel("zh-TW")).toBe("參考"); + }); + + test("degrades to English on a malformed tag", () => { + expect(resolveConceptTypeLabel("!!")).toBe("Reference"); + }); +}); diff --git a/test/index-middleware.test.ts b/test/index-middleware.test.ts index db2029b6..7034a7ca 100644 --- a/test/index-middleware.test.ts +++ b/test/index-middleware.test.ts @@ -4,6 +4,7 @@ import path from "node:path"; import { describe, expect, test, vi } from "vitest"; import { OpenWikiLocalShellBackend } from "../src/agent/docs-only-backend.ts"; import { createOpenWikiIndexMiddleware } from "../src/agent/okf-middleware.ts"; +import { ENGLISH_INDEX_LABELS } from "../src/okf/index-labels.ts"; import { migrateWikiToOkf, synchronizeWikiIndexes, @@ -99,6 +100,25 @@ describe("synchronizeWikiIndexes", () => { expect(rootIndex).not.toContain("# Directories"); }); + test("stamps the localized concept type on a repaired page", async () => { + const { backend, rootDir } = await setup(); + await backend.write("/openwiki/legacy.md", "# Pregled\n\nTijelo.\n"); + + await synchronizeWikiIndexes( + backend, + "repository", + { files: "Datoteke", directories: "Direktoriji" }, + "Referenca", + ); + + const legacy = await readFile( + path.join(rootDir, "openwiki/legacy.md"), + "utf8", + ); + expect(legacy).toContain('type: "Referenca"'); + expect(legacy).not.toContain('type: "Reference"'); + }); + test("uses OKF version frontmatter only at the bundle root", async () => { const { backend, rootDir } = await setup(); await backend.write( @@ -327,6 +347,20 @@ describe("migrateWikiToOkf", () => { ).resolves.toBe(goodBefore); }); + test("stamps a localized concept type when supplied", async () => { + const { backend, rootDir } = await setup(); + await backend.write("/openwiki/legacy.md", "# Pregled\n\nTijelo.\n"); + + await migrateWikiToOkf(backend, "repository", "Referenca"); + + const legacy = await readFile( + path.join(rootDir, "openwiki/legacy.md"), + "utf8", + ); + expect(legacy).toContain('type: "Referenca"'); + expect(legacy).not.toContain('type: "Reference"'); + }); + test("skips reserved files, dotfiles, and dot-directories", async () => { const { backend, rootDir } = await setup(); const dir = path.join(rootDir, "openwiki"); @@ -395,6 +429,30 @@ describe("createOpenWikiIndexMiddleware beforeAgent", () => { expect(legacy).toContain('type: "Reference"'); expect(legacy).toContain("openwiki_generated: true"); }); + + test("migrates using the localized concept type it was created with", async () => { + const { backend, rootDir } = await setup(); + await backend.write("/openwiki/legacy.md", "# Pregled\n\nTijelo.\n"); + + const middleware = createOpenWikiIndexMiddleware( + backend, + "repository", + ENGLISH_INDEX_LABELS, + "Referenca", + ); + const beforeAgent = + typeof middleware.beforeAgent === "function" + ? middleware.beforeAgent + : middleware.beforeAgent?.hook; + expect(beforeAgent).toBeTypeOf("function"); + await (beforeAgent as () => Promise)(); + + const legacy = await readFile( + path.join(rootDir, "openwiki/legacy.md"), + "utf8", + ); + expect(legacy).toContain('type: "Referenca"'); + }); }); describe("createOpenWikiIndexMiddleware afterAgent", () => { diff --git a/test/okf/frontmatter.test.ts b/test/okf/frontmatter.test.ts index d0653310..aa3a7a3e 100644 --- a/test/okf/frontmatter.test.ts +++ b/test/okf/frontmatter.test.ts @@ -58,6 +58,18 @@ describe("normalizeConceptContent", () => { expect(result.content).toContain("openwiki_generated: true"); }); + test("stamps a supplied localized concept type on a repaired page", () => { + const result = normalizeConceptContent( + "# 架构概览\n描述平台。\n", + PATH, + "参考", + ); + + expect(result.changed).toBe(true); + expect(result.content).toContain('type: "参考"'); + expect(result.content).not.toContain('type: "Reference"'); + }); + test("regenerates unparseable YAML instead of throwing", () => { const result = normalizeConceptContent( "---\ntype: [unterminated\n---\n\n# Broken\nProse.\n", @@ -106,9 +118,13 @@ describe("deriveMinimalFrontmatter", () => { ).toEqual({ type: "Reference", title: "Architecture Overview" }); }); - test("always uses type Reference", () => { + test("defaults the type to Reference", () => { expect(deriveMinimalFrontmatter("body", PATH).type).toBe("Reference"); }); + + test("uses a supplied localized concept type", () => { + expect(deriveMinimalFrontmatter("body", PATH, "参考").type).toBe("参考"); + }); }); describe("splitFrontmatter", () => { diff --git a/test/prompt.test.ts b/test/prompt.test.ts index c55a3a5e..101b032a 100644 --- a/test/prompt.test.ts +++ b/test/prompt.test.ts @@ -12,6 +12,9 @@ describe("createSystemPrompt output language", () => { expect(prompt).toContain( "Write generated wiki prose, headings, table content, and documentation in zh-CN.", ); + expect(prompt).toContain( + 'write the human-readable "title", "description", "type", and "tags" values in zh-CN', + ); expect(prompt).toContain( "Apply this language only to generated wiki files.", ); diff --git a/test/translation-middleware.test.ts b/test/translation-middleware.test.ts index 2052a71a..ecc5b878 100644 --- a/test/translation-middleware.test.ts +++ b/test/translation-middleware.test.ts @@ -118,6 +118,11 @@ describe("createWikiTranslationMiddleware beforeAgent", () => { ]); expect(calls[0].system).toContain("Chinese (China)"); expect(calls[0].system).toContain("English"); + // The front matter instruction now covers type and tags, not just title and + // description, so no structural value is left in the source language. + expect(calls[0].system).toContain( + '"title", "description", "type", and "tags"', + ); }); test("skips indexes, logs, control files, and dotfiles", async () => { From 834f3601465358ec879a8ee752dde548d2a078e6 Mon Sep 17 00:00:00 2001 From: Colin Francis Date: Mon, 27 Jul 2026 08:27:19 -0700 Subject: [PATCH 08/13] auto-retry failed wiki-page translations via a pending marker --- src/agent/index.ts | 17 +- src/agent/prompt.ts | 1 + src/agent/translation-middleware.ts | 247 ++++++++++++++++++++------ src/okf/frontmatter.ts | 99 ++++++++++- test/okf/frontmatter.test.ts | 126 ++++++++++++++ test/prompt.test.ts | 16 ++ test/translation-middleware.test.ts | 258 ++++++++++++++++++++++++---- 7 files changed, 669 insertions(+), 95 deletions(-) diff --git a/src/agent/index.ts b/src/agent/index.ts index 799622d4..8e7ff499 100644 --- a/src/agent/index.ts +++ b/src/agent/index.ts @@ -33,7 +33,7 @@ import { OpenWikiLocalShellBackend } from "./docs-only-backend.js"; import { createOpenWikiIndexMiddleware } from "./okf-middleware.js"; import { createWikiTranslationMiddleware, - resolveTranslationSwitch, + resolveTranslationPlan, } from "./translation-middleware.js"; import { CODEX_ORIGINATOR, @@ -293,10 +293,11 @@ async function runOpenWikiAgentCore( }), }); // An update inherits the wiki's persisted language unless --language requests a - // different one. When it does, retranslate every existing page before the agent - // runs so the incremental update does not leave a mix of the old and new - // language. - const translation = resolveTranslationSwitch( + // different one. The plan drives a beforeAgent pass that, on a switch, + // retranslates every page so the incremental update does not leave a mix of the + // old and new language, and on any update retries pages a prior run left + // pending. It is undefined for init and chat, which never translate. + const translation = resolveTranslationPlan( command, resolveLanguage(options.language).language, context.lastUpdate?.language, @@ -322,6 +323,12 @@ async function runOpenWikiAgentCore( outputMode, model, translation, + (message) => { + options.onEvent?.({ type: "text", text: message }); + // Also emit to stderr so the warning survives the TUI + // re-render and --print's discard of streamed text. + process.stderr.write(`${message}\n`); + }, ), ] : []), diff --git a/src/agent/prompt.ts b/src/agent/prompt.ts index ee2fa212..6b84cf70 100644 --- a/src/agent/prompt.ts +++ b/src/agent/prompt.ts @@ -169,6 +169,7 @@ timestamp: - The description field is especially useful for retrieval tools. When present, make it clear, detailed, and optimized for search. - When updating an existing Markdown concept, preserve accurate body content and correct its opening front matter only when needed for compliance or accuracy. - OpenWiki repairs front matter deterministically after every run, so a page is never rejected for missing or invalid front matter. If a page's front matter contains \`openwiki_generated: true\`, that metadata was code-derived as a fallback: replace it with an accurate \`type\`, \`title\`, and \`description\` grounded in the page body, then remove the \`openwiki_generated\` field. +- If a page's front matter contains an \`openwiki_translation_pending\` field, ignore it: it is a translation-system marker that OpenWiki manages automatically. Do not add, edit, remove, or act on it. Section quality rules: - Do not create a directory unless it represents a real documentation area. diff --git a/src/agent/translation-middleware.ts b/src/agent/translation-middleware.ts index 4b840778..412b745f 100644 --- a/src/agent/translation-middleware.ts +++ b/src/agent/translation-middleware.ts @@ -4,11 +4,19 @@ import { HumanMessage, SystemMessage } from "@langchain/core/messages"; import type { BackendProtocolV2, FileInfo } from "deepagents"; import { createMiddleware } from "langchain"; import path from "node:path"; +import { getErrorMessage } from "../diagnostics.js"; +import { + OPENWIKI_TRANSLATION_PENDING_FIELD, + readFrontmatterField, + removeFrontmatterField, + setFrontmatterField, +} from "../okf/frontmatter.js"; import type { OpenWikiCommand, OpenWikiOutputMode } from "./types.js"; /** * Wiki files that are regenerated deterministically (indexes) or are not - * human-facing prose (logs, scratch plans), so they are never translated. + * human-facing prose (logs, scratch plans, the user brief), so they are never + * translated. */ const EXCLUDED_FILES = new Set([ "index.md", @@ -18,44 +26,67 @@ const EXCLUDED_FILES = new Set([ ]); /** - * The source and target languages for a translate-all pass, as canonical - * BCP-47 tags (for example `en` and `zh-CN`). + * What an `update` run should do about the wiki's language before the agent + * runs. + * + * The plan is resolved once per run and drives the translation middleware: it + * says which language every page must end up in, which language to translate + * from, and whether the whole wiki needs converting or only pages a prior run + * left pending. */ -export interface WikiTranslationOptions { +export interface TranslationPlan { + /** + * The language every eligible page must end up written in, as a canonical + * BCP-47 tag (for example `en` or `zh-CN`). + */ + target: string; + /** - * The language the wiki is currently written in. + * The language the wiki is currently written in, used as the translation + * source hint. Detection is best-effort: a page left pending by an earlier + * failed switch may not actually be in this language, so the model is asked to + * detect the real source rather than trust this blindly. */ - from: string; + source: string; /** - * The language every page should be rewritten into. + * When true the run must translate every page (a real language switch); when + * false only pages carrying a pending-translation marker are retranslated, and + * a wiki with none is left untouched. */ - to: string; + translateAll: boolean; } /** - * Decides whether a run must retranslate the existing wiki, and between which - * languages. + * Resolves the translation plan for a run, or undefined when translation never + * applies (any command other than `update`). * - * A translate-all pass is warranted only for an `update` that explicitly - * requests a language whose primary subtag differs from the wiki's persisted - * one (an absent persisted language means the wiki is English). Comparing - * primary subtags avoids a needless full retranslation for a region-only change - * such as `en` to `en-GB`. Returns the source and target languages when a switch - * is needed, or `undefined` otherwise. + * OpenWiki treats the wiki's language as persisted state: an `update` inherits + * it unless `--language` requests a different one. The target is the requested + * language, else the persisted one, else English. A full translate-all pass is + * warranted only when a language is explicitly requested whose primary subtag + * differs from the persisted one; comparing primary subtags avoids a needless + * retranslation for a region-only change such as `en` to `en-GB`. Even without a + * switch the plan is returned for every update so the middleware can still sweep + * pages a prior run left pending. */ -export function resolveTranslationSwitch( +export function resolveTranslationPlan( command: OpenWikiCommand, requestedLanguage: string | undefined, currentWikiLanguage: string | undefined, -): WikiTranslationOptions | undefined { - if (command !== "update" || requestedLanguage === undefined) { +): TranslationPlan | undefined { + if (command !== "update") { return undefined; } - if (primarySubtag(requestedLanguage) === primarySubtag(currentWikiLanguage)) { - return undefined; - } - return { from: currentWikiLanguage ?? "en", to: requestedLanguage }; + + const source = currentWikiLanguage ?? "en"; + return { + target: requestedLanguage ?? source, + source, + translateAll: + requestedLanguage !== undefined && + primarySubtag(requestedLanguage) !== primarySubtag(currentWikiLanguage), + }; } /** @@ -72,78 +103,181 @@ function primarySubtag(tag: string | undefined): string { } /** - * Creates middleware that retranslates every existing wiki page before the - * agent runs. + * Creates middleware that brings every existing wiki page into the run's target + * language before the agent runs. + * + * OpenWiki treats the wiki's language as persisted state, so an incremental + * update alone would leave a mix of the old and new language on a switch, since + * the agent only rewrites pages whose source changed. This `beforeAgent` hook + * closes that gap. It is mounted on every `update` and does one of three things, + * cheapest first: with nothing to do it only walks the tree (zero model calls); + * on a plain update it retranslates just the pages a prior run left marked + * `openwiki_translation_pending`; on a real language switch it retranslates every + * page. * - * OpenWiki treats the wiki's language as persisted state: an update inherits it - * unless `--language` requests a different one. When it differs, the agent's - * incremental update alone would leave a mix of the old and new language, since - * it only rewrites pages whose source changed. This `beforeAgent` hook closes - * that gap by walking the wiki up front and rewriting each page into the target - * language, so the agent then edits an already-uniform wiki. + * The model's output is treated purely as file text and written back through the + * sandboxed docs-only backend; it is never executed, and every path comes from + * backend enumeration rather than model output. * - * The model's output is treated purely as file text and written back through - * the sandboxed docs-only backend; it is never executed, and every path comes - * from backend enumeration rather than model output. + * A single page's translation failure never aborts the run: the page is left in + * its previous language, stamped with a pending marker so the next update retries + * it, and reported through `onWarning`. `onWarning` defaults to writing the + * (already secret-redacted) summary to stderr. */ export function createWikiTranslationMiddleware( backend: BackendProtocolV2, outputMode: OpenWikiOutputMode, model: BaseChatModel, - languages: WikiTranslationOptions, + plan: TranslationPlan, + onWarning: (message: string) => void = (message) => { + process.stderr.write(`${message}\n`); + }, ) { return createMiddleware({ name: "OpenWikiTranslationMiddleware", beforeAgent: async () => { - await translateWiki(backend, outputMode, model, languages); + await translateWiki(backend, outputMode, model, plan, onWarning); }, }); } /** - * Rewrites every concept page in the wiki from one language into another. + * Brings each concept page into the plan's target language. + * + * Each page is handled independently: a plain update skips pages that are not + * marked pending, and any failure (a model or backend error) is caught, the page + * is stamped for retry, and the run continues rather than aborting. All failures + * are reported once through `onWarning`, so a single bad page degrades the update + * gracefully instead of crashing it. */ async function translateWiki( backend: BackendProtocolV2, outputMode: OpenWikiOutputMode, model: BaseChatModel, - languages: WikiTranslationOptions, + plan: TranslationPlan, + onWarning: (message: string) => void, ): Promise { const root = outputMode === "local-wiki" ? "/" : "/openwiki"; + const failures: string[] = []; + for (const filePath of await collectMarkdownFiles(backend, root)) { - await translateFile(backend, model, filePath, languages); + let content: string; + try { + content = await readText(backend, filePath); + } catch (error) { + // getErrorMessage redacts any provider credential an error may carry + // before the path and reason reach stderr or the TUI. + failures.push(`- ${filePath}: ${getErrorMessage(error)}`); + continue; + } + + // On a plain update only retry pages a prior run left pending; a real switch + // retranslates every page. + const pending = + readFrontmatterField(content, OPENWIKI_TRANSLATION_PENDING_FIELD) !== + undefined; + if (!plan.translateAll && !pending) continue; + + try { + await translatePage(backend, model, filePath, content, plan); + } catch (error) { + const reasons = [getErrorMessage(error)]; + const stampError = await markPending( + backend, + filePath, + content, + plan.target, + ); + if (stampError) { + reasons.push(`could not mark it for retry: ${stampError}`); + } + failures.push(`- ${filePath}: ${reasons.join("; ")}`); + } + } + + if (failures.length > 0) { + onWarning( + `OpenWiki could not translate ${failures.length} page(s) into ` + + `${describeLanguage(plan.target)}; they keep their previous language ` + + `and will be retried on the next update:\n${failures.join("\n")}`, + ); } } /** - * Translates one concept file in place, skipping writes that would not change - * the content. + * Translates one concept page into the plan's target language and clears any + * pending marker on success, writing only when the content changed. + * + * Throws on a model or backend failure, and on empty model output, so the caller + * can stamp the page for retry. A successful translation always drops the pending + * marker deterministically, whatever the model returned, so a page that has just + * been converted is never left flagged. */ -async function translateFile( +async function translatePage( backend: BackendProtocolV2, model: BaseChatModel, filePath: string, - languages: WikiTranslationOptions, + original: string, + plan: TranslationPlan, ): Promise { - const original = await readText(backend, filePath); if (!original.trim()) return; - const translated = await translateMarkdown(model, original, languages); - if (!translated.trim() || translated.trim() === original.trim()) return; + const translated = await translateMarkdown( + model, + original, + plan.source, + plan.target, + ); + if (!translated.trim()) { + throw new Error("the model returned an empty translation"); + } + + const finalized = removeFrontmatterField( + translated, + OPENWIKI_TRANSLATION_PENDING_FIELD, + ); + if (finalized === original) return; - const result = await backend.edit(filePath, original, translated); + const result = await backend.edit(filePath, original, finalized); if (result.error) { - throw new Error(`Unable to translate ${filePath}: ${result.error}`); + throw new Error(`could not write the translation: ${result.error}`); } } +/** + * Stamps a page with the pending-translation marker so a later update retries + * it, preserving all other front matter. + * + * Returns undefined on success (including when the marker already matched and no + * write was needed), or a sanitized reason string when the stamp could not be + * written, so the caller can fold it into the surrounding failure report rather + * than swallow it. + */ +async function markPending( + backend: BackendProtocolV2, + filePath: string, + content: string, + target: string, +): Promise { + const stamped = setFrontmatterField( + content, + OPENWIKI_TRANSLATION_PENDING_FIELD, + target, + ); + if (stamped === content) return undefined; + + const result = await backend.edit(filePath, content, stamped); + return result.error ? getErrorMessage(new Error(result.error)) : undefined; +} + /** * Asks the model to translate a Markdown document, returning its raw text. */ async function translateMarkdown( model: BaseChatModel, content: string, - { from, to }: WikiTranslationOptions, + from: string, + to: string, ): Promise { const response = await model.invoke([ new SystemMessage(buildTranslationPrompt(from, to)), @@ -155,12 +289,23 @@ async function translateMarkdown( /** * Builds the system instruction that constrains the translation to prose while * preserving Markdown structure and code. + * + * The source language is a hint, not a guarantee: a page left pending by an + * earlier failed switch may still be in a different language, so the model is + * told to detect the real source and to leave content already in the target + * language unchanged. */ function buildTranslationPrompt(from: string, to: string): string { return `You are a professional technical translator for a software documentation wiki. -Translate the Markdown document provided by the user from ${describeLanguage( +Translate the Markdown document provided by the user into ${describeLanguage( + to, + )}. It is expected to be in ${describeLanguage( from, - )} into ${describeLanguage(to)}. + )}, but detect its actual language and translate any content that is not already in ${describeLanguage( + to, + )}. If the document is already entirely in ${describeLanguage( + to, + )}, return it unchanged. Rules: - Translate prose, headings, list items, blockquotes, and table cell text. diff --git a/src/okf/frontmatter.ts b/src/okf/frontmatter.ts index 4aa9a35b..b6fb9d59 100644 --- a/src/okf/frontmatter.ts +++ b/src/okf/frontmatter.ts @@ -17,6 +17,24 @@ const OKF_STRING_FIELDS = [ */ export const OPENWIKI_GENERATED_FIELD = "openwiki_generated"; +/** + * Extension field marking a page whose translation is still owed, carrying the + * BCP-47 target language (for example `"zh-CN"`). Written and cleared only by the + * translation middleware; the deterministic OKF pass merely preserves it. + */ +export const OPENWIKI_TRANSLATION_PENDING_FIELD = + "openwiki_translation_pending"; + +/** + * Extension fields that must survive a deterministic front-matter regeneration. + * + * When a page fails OKF validation its front matter is rebuilt from a minimal + * derived block, which would otherwise drop every extension field. Fields listed + * here are carried across that rebuild so control markers are not lost when a + * page happens to be both non-conformant and, say, pending translation. + */ +const PRESERVED_EXTENSION_FIELDS = [OPENWIKI_TRANSLATION_PENDING_FIELD]; + /** * Matches a leading YAML front-matter block and captures its inner text. */ @@ -222,6 +240,78 @@ export function parseFrontmatterFields( : undefined; } +/** + * Reads a single front-matter field's string value, or undefined when the field + * is absent, the block is unparseable, or the value is not a string. + */ +export function readFrontmatterField( + content: string, + key: string, +): string | undefined { + const value = parseFrontmatterFields(content)?.[key]; + return typeof value === "string" ? value : undefined; +} + +/** + * Sets or replaces a single scalar field in a page's front matter, preserving + * every other line byte-for-byte. + * + * This deliberately edits the raw block rather than parsing and re-rendering, + * because {@link renderFrontmatter} only knows a fixed set of fields and would + * drop producer extensions on a round trip. When the page has no front-matter + * block, a minimal one holding just this field is prepended. The value is + * JSON-quoted so colons and other YAML-significant characters stay safe. + */ +export function setFrontmatterField( + content: string, + key: string, + value: string, +): string { + const line = `${key}: ${JSON.stringify(value)}`; + const { block, body } = splitFrontmatter(content); + if (block === undefined) { + return `---\n${line}\n---\n\n${content}`; + } + + const lines = block.split("\n"); + const index = lines.findIndex((current) => isFieldLine(current, key)); + if (index === -1) { + lines.push(line); + } else { + lines[index] = line; + } + return `---\n${lines.join("\n")}\n---\n${body}`; +} + +/** + * Removes a single field from a page's front matter, preserving every other line + * byte-for-byte, and returns the content unchanged when the field is absent. If + * the field was the block's only line, the now-empty block is dropped entirely. + */ +export function removeFrontmatterField(content: string, key: string): string { + const { block, body } = splitFrontmatter(content); + if (block === undefined) return content; + + const kept = block.split("\n").filter((line) => !isFieldLine(line, key)); + if (kept.length === block.split("\n").length) return content; + if (kept.length === 0) return body.replace(/^\r?\n/u, ""); + return `---\n${kept.join("\n")}\n---\n${body}`; +} + +/** + * Reports whether a raw front-matter line declares the given top-level key. + */ +function isFieldLine(line: string, key: string): boolean { + return new RegExp(`^${escapeRegExp(key)}\\s*:`, "u").test(line); +} + +/** + * Escapes a string for safe interpolation into a regular expression. + */ +function escapeRegExp(value: string): string { + return value.replace(/[.*+?^${}()|[\]\\]/gu, "\\$&"); +} + /** * Returns the text of the first ATX H1 in a body, if any. */ @@ -298,7 +388,14 @@ export function normalizeConceptContent( const { body } = splitFrontmatter(content); const derived = deriveMinimalFrontmatter(body, filePath, conceptType); const front = renderFrontmatter(derived, { generated: true }); - return { changed: true, content: `${front}${body.replace(/^\s+/u, "")}` }; + let rebuilt = `${front}${body.replace(/^\s+/u, "")}`; + for (const field of PRESERVED_EXTENSION_FIELDS) { + const value = readFrontmatterField(content, field); + if (value !== undefined) { + rebuilt = setFrontmatterField(rebuilt, field, value); + } + } + return { changed: true, content: rebuilt }; } /** diff --git a/test/okf/frontmatter.test.ts b/test/okf/frontmatter.test.ts index aa3a7a3e..7359e146 100644 --- a/test/okf/frontmatter.test.ts +++ b/test/okf/frontmatter.test.ts @@ -3,7 +3,10 @@ import { deriveMinimalFrontmatter, normalizeConceptContent, parseFrontmatterFields, + readFrontmatterField, + removeFrontmatterField, renderFrontmatter, + setFrontmatterField, splitFrontmatter, } from "../../src/okf/frontmatter.ts"; @@ -90,6 +93,129 @@ describe("normalizeConceptContent", () => { expect(result.changed).toBe(true); expect(result.content).toContain("openwiki_generated: true"); }); + + test("carries a pending-translation marker across a regeneration", () => { + // A non-conformant page (no type) is rebuilt, which drops extension fields; + // the translation marker must survive so the page is still retried. + const result = normalizeConceptContent( + '---\ntitle: Orphan\nopenwiki_translation_pending: "zh-CN"\n---\n\n# Orphan\n', + PATH, + ); + + expect(result.changed).toBe(true); + expect(result.content).toContain('type: "Reference"'); + expect(result.content).toContain("openwiki_generated: true"); + expect(result.content).toContain('openwiki_translation_pending: "zh-CN"'); + }); + + test("preserves a pending-translation marker on a valid page for free", () => { + const content = + '---\ntype: "参考"\nopenwiki_translation_pending: "zh-CN"\n---\n\n# X\n'; + const result = normalizeConceptContent(content, PATH); + + expect(result.changed).toBe(false); + expect(result.content).toBe(content); + }); +}); + +describe("setFrontmatterField", () => { + test("inserts a new field, preserving other lines and extensions", () => { + const result = setFrontmatterField( + "---\ntype: Reference\ncustom_ext: keep-me\n---\n\n# Page\n", + "openwiki_translation_pending", + "zh-CN", + ); + + expect(result).toBe( + '---\ntype: Reference\ncustom_ext: keep-me\nopenwiki_translation_pending: "zh-CN"\n---\n\n# Page\n', + ); + }); + + test("replaces an existing field's value in place", () => { + const result = setFrontmatterField( + '---\ntype: Reference\nopenwiki_translation_pending: "en"\n---\n\n# Page\n', + "openwiki_translation_pending", + "hi", + ); + + expect(result).toBe( + '---\ntype: Reference\nopenwiki_translation_pending: "hi"\n---\n\n# Page\n', + ); + }); + + test("prepends a minimal block when the page has no front matter", () => { + expect( + setFrontmatterField( + "# Page\nBody.\n", + "openwiki_translation_pending", + "hi", + ), + ).toBe('---\nopenwiki_translation_pending: "hi"\n---\n\n# Page\nBody.\n'); + }); + + test("quotes the value so special characters stay safe", () => { + expect( + setFrontmatterField("---\ntype: Reference\n---\n\n# P\n", "note", "a: b"), + ).toContain('note: "a: b"'); + }); +}); + +describe("removeFrontmatterField", () => { + test("removes a field while preserving the other lines", () => { + expect( + removeFrontmatterField( + '---\ntype: Reference\nopenwiki_translation_pending: "zh-CN"\ntitle: Page\n---\n\n# Page\n', + "openwiki_translation_pending", + ), + ).toBe("---\ntype: Reference\ntitle: Page\n---\n\n# Page\n"); + }); + + test("returns the content unchanged when the field is absent", () => { + const content = "---\ntype: Reference\n---\n\n# Page\n"; + expect( + removeFrontmatterField(content, "openwiki_translation_pending"), + ).toBe(content); + }); + + test("returns the content unchanged when there is no block", () => { + const content = "# Page\nNo front matter.\n"; + expect( + removeFrontmatterField(content, "openwiki_translation_pending"), + ).toBe(content); + }); + + test("drops a block that becomes empty", () => { + expect( + removeFrontmatterField( + '---\nopenwiki_translation_pending: "hi"\n---\n\n# Page\n', + "openwiki_translation_pending", + ), + ).toBe("# Page\n"); + }); +}); + +describe("readFrontmatterField", () => { + test("reads a string field's value", () => { + expect( + readFrontmatterField( + '---\nopenwiki_translation_pending: "zh-CN"\n---\n\n# Page\n', + "openwiki_translation_pending", + ), + ).toBe("zh-CN"); + }); + + test("returns undefined when the field is absent or there is no block", () => { + expect( + readFrontmatterField("---\ntype: Reference\n---\n", "missing"), + ).toBeUndefined(); + expect(readFrontmatterField("# Page\n", "type")).toBeUndefined(); + }); + + test("returns undefined for a non-string value", () => { + expect( + readFrontmatterField("---\ncount: 3\n---\n", "count"), + ).toBeUndefined(); + }); }); describe("deriveMinimalFrontmatter", () => { diff --git a/test/prompt.test.ts b/test/prompt.test.ts index 101b032a..c3e758a4 100644 --- a/test/prompt.test.ts +++ b/test/prompt.test.ts @@ -120,6 +120,22 @@ describe("createSystemPrompt openwiki_generated enrichment guidance", () => { } }); +/** + * The translation middleware is the sole owner of the + * `openwiki_translation_pending` marker. The prompt must tell the agent to leave + * it alone so the model never adds, edits, or clears a marker code manages. + */ +describe("createSystemPrompt translation-marker guidance", () => { + for (const outputMode of ["repository", "local-wiki"] as const) { + test(`${outputMode} mode: tells the agent to ignore the pending marker`, () => { + const prompt = createSystemPrompt("update", outputMode); + + expect(prompt).toContain("openwiki_translation_pending"); + expect(prompt).toMatch(/Do not add, edit, remove, or act on it/); + }); + } +}); + describe("createDiagramInstructions", () => { test("nudges toward diagrams and defers label-safety to the skill", () => { const text = createDiagramInstructions(); diff --git a/test/translation-middleware.test.ts b/test/translation-middleware.test.ts index ecc5b878..0547cdb1 100644 --- a/test/translation-middleware.test.ts +++ b/test/translation-middleware.test.ts @@ -7,9 +7,24 @@ import { describe, expect, test, vi } from "vitest"; import { OpenWikiLocalShellBackend } from "../src/agent/docs-only-backend.ts"; import { createWikiTranslationMiddleware, - resolveTranslationSwitch, + resolveTranslationPlan, + type TranslationPlan, } from "../src/agent/translation-middleware.ts"; +/** + * A translate-all plan (a real language switch) into the given target. + */ +function switchTo(target: string, source = "en"): TranslationPlan { + return { target, source, translateAll: true }; +} + +/** + * A plain-update plan that only sweeps pages left pending, never a full pass. + */ +function sweep(language: string): TranslationPlan { + return { target: language, source: language, translateAll: false }; +} + /** * Records every prompt the middleware sends and returns a scripted translation * for each page body, without any network access. @@ -53,35 +68,54 @@ async function runBeforeAgent( await (beforeAgent as () => Promise)(); } -describe("resolveTranslationSwitch", () => { - test("triggers when an update requests a different language", () => { - expect(resolveTranslationSwitch("update", "zh-CN", "en")).toEqual({ - from: "en", - to: "zh-CN", +describe("resolveTranslationPlan", () => { + test("requests a full pass when an update switches to a different language", () => { + expect(resolveTranslationPlan("update", "zh-CN", "en")).toEqual({ + target: "zh-CN", + source: "en", + translateAll: true, }); }); test("treats an absent persisted language as English", () => { - expect(resolveTranslationSwitch("update", "zh-CN", undefined)).toEqual({ - from: "en", - to: "zh-CN", + expect(resolveTranslationPlan("update", "zh-CN", undefined)).toEqual({ + target: "zh-CN", + source: "en", + translateAll: true, }); - expect( - resolveTranslationSwitch("update", "en-US", undefined), - ).toBeUndefined(); }); - test("ignores a region-only change with the same primary subtag", () => { - expect(resolveTranslationSwitch("update", "en-GB", "en")).toBeUndefined(); - expect( - resolveTranslationSwitch("update", "zh-TW", "zh-CN"), - ).toBeUndefined(); + test("does not switch for a region-only change with the same primary subtag", () => { + expect(resolveTranslationPlan("update", "en-GB", "en")).toEqual({ + target: "en-GB", + source: "en", + translateAll: false, + }); + expect(resolveTranslationPlan("update", "zh-TW", "zh-CN")).toEqual({ + target: "zh-TW", + source: "zh-CN", + translateAll: false, + }); + }); + + test("returns a marker-sweep plan for a plain update with no requested language", () => { + // No switch, but still a plan so pending pages get retried, targeting the + // persisted language (source and target match). + expect(resolveTranslationPlan("update", undefined, "zh-CN")).toEqual({ + target: "zh-CN", + source: "zh-CN", + translateAll: false, + }); + expect(resolveTranslationPlan("update", undefined, undefined)).toEqual({ + target: "en", + source: "en", + translateAll: false, + }); }); - test("does nothing without a requested language or for non-update commands", () => { - expect(resolveTranslationSwitch("update", undefined, "en")).toBeUndefined(); - expect(resolveTranslationSwitch("init", "zh-CN", "en")).toBeUndefined(); - expect(resolveTranslationSwitch("chat", "zh-CN", "en")).toBeUndefined(); + test("never applies to non-update commands", () => { + expect(resolveTranslationPlan("init", "zh-CN", "en")).toBeUndefined(); + expect(resolveTranslationPlan("chat", "zh-CN", "en")).toBeUndefined(); }); }); @@ -99,7 +133,7 @@ describe("createWikiTranslationMiddleware beforeAgent", () => { backend, "repository", model, - { from: "en", to: "zh-CN" }, + switchTo("zh-CN"), ); await runBeforeAgent(middleware); @@ -125,6 +159,146 @@ describe("createWikiTranslationMiddleware beforeAgent", () => { ); }); + test("keeps translating other pages when one page's write fails and reports it", async () => { + const { backend, rootDir } = await setup(); + await backend.write("/openwiki/good.md", "# Good\n\nBody.\n"); + await backend.write("/openwiki/bad.md", "# Bad\n\nBody.\n"); + + // Fail every write for one page only; the others must still convert. Both the + // translation write and the retry-stamp write are refused here. + const realEdit = backend.edit.bind(backend); + vi.spyOn(backend, "edit").mockImplementation((filePath, before, after) => + filePath.endsWith("bad.md") + ? Promise.resolve({ error: "disk on fire" }) + : realEdit(filePath, before, after), + ); + + const warnings: string[] = []; + const { model } = fakeModel((content) => `T\n${content}`); + await runBeforeAgent( + createWikiTranslationMiddleware( + backend, + "repository", + model, + switchTo("zh-CN"), + (message) => warnings.push(message), + ), + ); + + // The healthy page was still translated despite the sibling failure. + await expect( + readFile(path.join(rootDir, "openwiki/good.md"), "utf8"), + ).resolves.toBe("T\n# Good\n\nBody.\n"); + // The failing page is left untouched and named in a single warning. + await expect( + readFile(path.join(rootDir, "openwiki/bad.md"), "utf8"), + ).resolves.toBe("# Bad\n\nBody.\n"); + expect(warnings).toHaveLength(1); + expect(warnings[0]).toContain("bad.md"); + expect(warnings[0]).toContain("disk on fire"); + }); + + test("stamps a page for retry when the model fails to translate it", async () => { + const { backend, rootDir } = await setup(); + await backend.write("/openwiki/good.md", "# Good\n\nBody.\n"); + await backend.write("/openwiki/bad.md", "# Bad\n\nBody.\n"); + + // The model translates every page except the one whose body mentions "Bad". + const model = { + invoke: (messages: BaseMessage[]) => { + const human = messages[1]; + const text = typeof human.content === "string" ? human.content : ""; + return text.includes("# Bad") + ? Promise.reject(new Error("model exploded")) + : Promise.resolve(new AIMessage(`T\n${text}`)); + }, + } as unknown as BaseChatModel; + + const warnings: string[] = []; + await runBeforeAgent( + createWikiTranslationMiddleware( + backend, + "repository", + model, + switchTo("zh-CN"), + (message) => warnings.push(message), + ), + ); + + // The healthy page converts; the failed one keeps its body but gains a + // pending marker carrying the target language, so a later update retries it. + await expect( + readFile(path.join(rootDir, "openwiki/good.md"), "utf8"), + ).resolves.toBe("T\n# Good\n\nBody.\n"); + const bad = await readFile(path.join(rootDir, "openwiki/bad.md"), "utf8"); + expect(bad).toContain('openwiki_translation_pending: "zh-CN"'); + expect(bad).toContain("# Bad"); + expect(warnings).toHaveLength(1); + expect(warnings[0]).toContain("bad.md"); + expect(warnings[0]).toContain("model exploded"); + }); + + test("on a plain update, retries only pending pages and clears the marker", async () => { + const { backend, rootDir } = await setup(); + await backend.write( + "/openwiki/settled.md", + '---\ntype: "参考"\ntitle: "已完成"\n---\n\n# 已完成\n', + ); + await backend.write( + "/openwiki/pending.md", + '---\ntype: "参考"\ntitle: "待翻译"\nopenwiki_translation_pending: "zh-CN"\n---\n\n# 待翻译\n', + ); + + // Identity model: the pending page is already in the target language, so a + // successful pass only needs to strip its marker. + const { model, calls } = fakeModel((content) => content); + await runBeforeAgent( + createWikiTranslationMiddleware( + backend, + "repository", + model, + sweep("zh-CN"), + ), + ); + + // Only the pending page was sent to the model; the settled page was skipped. + expect(calls).toHaveLength(1); + expect(calls[0].human).toContain("待翻译"); + + // The settled page is byte-for-byte unchanged. + await expect( + readFile(path.join(rootDir, "openwiki/settled.md"), "utf8"), + ).resolves.toBe('---\ntype: "参考"\ntitle: "已完成"\n---\n\n# 已完成\n'); + // The marker is cleared while the rest of the front matter survives. + const pending = await readFile( + path.join(rootDir, "openwiki/pending.md"), + "utf8", + ); + expect(pending).not.toContain("openwiki_translation_pending"); + expect(pending).toContain('type: "参考"'); + expect(pending).toContain('title: "待翻译"'); + }); + + test("on a plain update with no pending pages, calls the model zero times", async () => { + const { backend } = await setup(); + await backend.write("/openwiki/one.md", "# One\n\nBody.\n"); + await backend.write("/openwiki/two.md", "# Two\n\nBody.\n"); + + const edit = vi.spyOn(backend, "edit"); + const { model, calls } = fakeModel((content) => `X\n${content}`); + await runBeforeAgent( + createWikiTranslationMiddleware( + backend, + "repository", + model, + sweep("en"), + ), + ); + + expect(calls).toHaveLength(0); + expect(edit).not.toHaveBeenCalled(); + }); + test("skips indexes, logs, control files, and dotfiles", async () => { const { backend, rootDir } = await setup(); const dir = path.join(rootDir, "openwiki"); @@ -138,10 +312,12 @@ describe("createWikiTranslationMiddleware beforeAgent", () => { const { model, calls } = fakeModel((content) => `X\n${content}`); await runBeforeAgent( - createWikiTranslationMiddleware(backend, "repository", model, { - from: "en", - to: "hi", - }), + createWikiTranslationMiddleware( + backend, + "repository", + model, + switchTo("hi"), + ), ); // Only the one real concept page is translated. @@ -162,10 +338,12 @@ describe("createWikiTranslationMiddleware beforeAgent", () => { const edit = vi.spyOn(backend, "edit"); const { model } = fakeModel((content) => content); await runBeforeAgent( - createWikiTranslationMiddleware(backend, "repository", model, { - from: "en", - to: "zh-CN", - }), + createWikiTranslationMiddleware( + backend, + "repository", + model, + switchTo("zh-CN"), + ), ); expect(edit).not.toHaveBeenCalled(); @@ -177,10 +355,12 @@ describe("createWikiTranslationMiddleware beforeAgent", () => { const { model } = fakeModel((content) => `[hi] ${content}`); await runBeforeAgent( - createWikiTranslationMiddleware(backend, "local-wiki", model, { - from: "en", - to: "hi", - }), + createWikiTranslationMiddleware( + backend, + "local-wiki", + model, + switchTo("hi"), + ), ); await expect(readFile(path.join(rootDir, "note.md"), "utf8")).resolves.toBe( @@ -194,10 +374,12 @@ describe("createWikiTranslationMiddleware beforeAgent", () => { await expect( runBeforeAgent( - createWikiTranslationMiddleware(backend, "repository", model, { - from: "en", - to: "zh-CN", - }), + createWikiTranslationMiddleware( + backend, + "repository", + model, + switchTo("zh-CN"), + ), ), ).resolves.toBeUndefined(); expect(calls).toHaveLength(0); From 3fdb7d02e64c9eb6169be7e577b77b7b1f48baf3 Mon Sep 17 00:00:00 2001 From: Colin Francis Date: Mon, 27 Jul 2026 09:56:36 -0700 Subject: [PATCH 09/13] don't stream docs translation --- src/agent/index.ts | 6 +++ src/agent/translation-middleware.ts | 63 ++++++++++++++++++++++++++--- test/translation-middleware.test.ts | 59 +++++++++++++++++++++++++-- 3 files changed, 119 insertions(+), 9 deletions(-) diff --git a/src/agent/index.ts b/src/agent/index.ts index 8e7ff499..094d04ee 100644 --- a/src/agent/index.ts +++ b/src/agent/index.ts @@ -329,6 +329,12 @@ async function runOpenWikiAgentCore( // re-render and --print's discard of streamed text. process.stderr.write(`${message}\n`); }, + // The pass announces itself with one line in place of the + // suppressed per-token translation output. It is routine + // progress, so unlike a warning it is not mirrored to stderr. + (message) => { + options.onEvent?.({ type: "text", text: message }); + }, ), ] : []), diff --git a/src/agent/translation-middleware.ts b/src/agent/translation-middleware.ts index 412b745f..c5b35a58 100644 --- a/src/agent/translation-middleware.ts +++ b/src/agent/translation-middleware.ts @@ -25,6 +25,23 @@ const EXCLUDED_FILES = new Set([ "INSTRUCTIONS.md", ]); +/** + * LangGraph run tag that excludes a model call from the `messages` stream mode. + * + * The agent is driven with `messages`-mode streaming, which captures every LLM + * token in the graph, including the translation calls this middleware makes in + * `beforeAgent`. Tagging those calls keeps their raw translated Markdown out of + * the token stream so it never scrolls past in the TUI; LangGraph honors this tag + * (and the bare `nostream`) when deciding which runs to surface. + */ +const NOSTREAM_TAG = "langsmith:nostream"; + +/** + * Single friendly line shown once while the wiki is being translated, in place + * of the suppressed per-token translation output. + */ +const TRANSLATION_STATUS_MESSAGE = "Translating wiki docs..."; + /** * What an `update` run should do about the wiki's language before the agent * runs. @@ -123,6 +140,12 @@ function primarySubtag(tag: string | undefined): string { * its previous language, stamped with a pending marker so the next update retries * it, and reported through `onWarning`. `onWarning` defaults to writing the * (already secret-redacted) summary to stderr. + * + * The raw translated Markdown is kept out of the token stream (see + * {@link NOSTREAM_TAG}); `onStatus` is called once instead, with a short line + * announcing the pass, so the user sees progress without the flood of tokens. It + * fires only when at least one page is actually translated, so a no-op marker + * sweep stays silent, and defaults to writing the line to stderr. */ export function createWikiTranslationMiddleware( backend: BackendProtocolV2, @@ -132,11 +155,21 @@ export function createWikiTranslationMiddleware( onWarning: (message: string) => void = (message) => { process.stderr.write(`${message}\n`); }, + onStatus: (message: string) => void = (message) => { + process.stderr.write(`${message}\n`); + }, ) { return createMiddleware({ name: "OpenWikiTranslationMiddleware", beforeAgent: async () => { - await translateWiki(backend, outputMode, model, plan, onWarning); + await translateWiki( + backend, + outputMode, + model, + plan, + onWarning, + onStatus, + ); }, }); } @@ -148,7 +181,9 @@ export function createWikiTranslationMiddleware( * marked pending, and any failure (a model or backend error) is caught, the page * is stamped for retry, and the run continues rather than aborting. All failures * are reported once through `onWarning`, so a single bad page degrades the update - * gracefully instead of crashing it. + * gracefully instead of crashing it. `onStatus` announces the pass once, lazily, + * just before the first page is translated, so a sweep that finds nothing to do + * prints no status at all. */ async function translateWiki( backend: BackendProtocolV2, @@ -156,9 +191,11 @@ async function translateWiki( model: BaseChatModel, plan: TranslationPlan, onWarning: (message: string) => void, + onStatus: (message: string) => void, ): Promise { const root = outputMode === "local-wiki" ? "/" : "/openwiki"; const failures: string[] = []; + let announced = false; for (const filePath of await collectMarkdownFiles(backend, root)) { let content: string; @@ -178,6 +215,13 @@ async function translateWiki( undefined; if (!plan.translateAll && !pending) continue; + // Announce the pass on the first page that will actually be translated, so a + // no-op sweep stays silent and the raw token stream stays suppressed. + if (!announced) { + onStatus(TRANSLATION_STATUS_MESSAGE); + announced = true; + } + try { await translatePage(backend, model, filePath, content, plan); } catch (error) { @@ -272,6 +316,10 @@ async function markPending( /** * Asks the model to translate a Markdown document, returning its raw text. + * + * The call is tagged with {@link NOSTREAM_TAG} so its tokens are excluded from + * the agent's `messages` stream: the translated Markdown is written back through + * the backend rather than streamed to the TUI token by token. */ async function translateMarkdown( model: BaseChatModel, @@ -279,10 +327,13 @@ async function translateMarkdown( from: string, to: string, ): Promise { - const response = await model.invoke([ - new SystemMessage(buildTranslationPrompt(from, to)), - new HumanMessage(content), - ]); + const response = await model.invoke( + [ + new SystemMessage(buildTranslationPrompt(from, to)), + new HumanMessage(content), + ], + { tags: [NOSTREAM_TAG] }, + ); return extractText(response.content); } diff --git a/test/translation-middleware.test.ts b/test/translation-middleware.test.ts index 0547cdb1..621e794e 100644 --- a/test/translation-middleware.test.ts +++ b/test/translation-middleware.test.ts @@ -30,13 +30,17 @@ function sweep(language: string): TranslationPlan { * for each page body, without any network access. */ function fakeModel(translate: (content: string) => string) { - const calls: { system: string; human: string }[] = []; + const calls: { system: string; human: string; tags: string[] }[] = []; const asText = (message: BaseMessage): string => typeof message.content === "string" ? message.content : ""; const model = { - invoke: (messages: BaseMessage[]) => { + invoke: (messages: BaseMessage[], options?: { tags?: string[] }) => { const [system, human] = messages; - calls.push({ system: asText(system), human: asText(human) }); + calls.push({ + system: asText(system), + human: asText(human), + tags: options?.tags ?? [], + }); return Promise.resolve(new AIMessage(translate(asText(human)))); }, }; @@ -157,6 +161,55 @@ describe("createWikiTranslationMiddleware beforeAgent", () => { expect(calls[0].system).toContain( '"title", "description", "type", and "tags"', ); + // Every translation call is tagged so its tokens are excluded from the + // agent's messages stream and never scroll past in the TUI. + for (const call of calls) { + expect(call.tags).toContain("langsmith:nostream"); + } + }); + + test("announces the pass once when at least one page is translated", async () => { + const { backend } = await setup(); + await backend.write("/openwiki/one.md", "# One\n\nBody.\n"); + await backend.write("/openwiki/two.md", "# Two\n\nBody.\n"); + + const status: string[] = []; + const { model } = fakeModel((content) => `T\n${content}`); + await runBeforeAgent( + createWikiTranslationMiddleware( + backend, + "repository", + model, + switchTo("zh-CN"), + () => {}, + (message) => status.push(message), + ), + ); + + // A single line for the whole pass, not one per page. + expect(status).toEqual(["Translating wiki docs..."]); + }); + + test("stays silent on a no-op sweep with nothing to translate", async () => { + const { backend } = await setup(); + await backend.write("/openwiki/one.md", "# One\n\nBody.\n"); + + const status: string[] = []; + const { model, calls } = fakeModel((content) => `T\n${content}`); + await runBeforeAgent( + createWikiTranslationMiddleware( + backend, + "repository", + model, + sweep("en"), + () => {}, + (message) => status.push(message), + ), + ); + + // No pending pages means no model calls and no announcement. + expect(calls).toHaveLength(0); + expect(status).toEqual([]); }); test("keeps translating other pages when one page's write fails and reports it", async () => { From 1f032f818793c4897a232247b6763fade89b409a Mon Sep 17 00:00:00 2001 From: Colin Francis Date: Mon, 27 Jul 2026 10:19:52 -0700 Subject: [PATCH 10/13] formatting --- src/agent/index.ts | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/agent/index.ts b/src/agent/index.ts index 094d04ee..596aa1da 100644 --- a/src/agent/index.ts +++ b/src/agent/index.ts @@ -332,8 +332,15 @@ async function runOpenWikiAgentCore( // The pass announces itself with one line in place of the // suppressed per-token translation output. It is routine // progress, so unlike a warning it is not mirrored to stderr. + // The trailing blank line keeps it a distinct Markdown block: + // the TUI coalesces consecutive text events into one + // block-lexed log item, so without it the status would run + // straight into the agent's first streamed line. (message) => { - options.onEvent?.({ type: "text", text: message }); + options.onEvent?.({ + type: "text", + text: `${message}\n\n`, + }); }, ), ] From a7ef1750ab169d3aa5a13d996698f80b34f4347b Mon Sep 17 00:00:00 2001 From: Colin Francis Date: Mon, 27 Jul 2026 10:46:13 -0700 Subject: [PATCH 11/13] keep OKF tags canonical, translate only reader-facing front matter --- src/agent/prompt.ts | 2 +- src/agent/translation-middleware.ts | 2 +- test/prompt.test.ts | 4 +++- test/translation-middleware.test.ts | 12 +++++++----- 4 files changed, 12 insertions(+), 8 deletions(-) diff --git a/src/agent/prompt.ts b/src/agent/prompt.ts index 6b84cf70..eb451c14 100644 --- a/src/agent/prompt.ts +++ b/src/agent/prompt.ts @@ -212,7 +212,7 @@ function createLanguageInstructions(language: string | undefined): string { Output language: - Write generated wiki prose, headings, table content, and documentation in ${language}. -- In each page's YAML front matter, write the human-readable "title", "description", "type", and "tags" values in ${language}. Keep the YAML keys and any URL, file-path, timestamp, or identifier-like values unchanged. +- In each page's YAML front matter, write the human-readable "title", "description", and "type" values in ${language}. Write the "tags" values in English so they stay stable across languages as cross-cutting aggregation keys. Keep the YAML keys as written, and copy any URL, file path, timestamp, or identifier-like value byte-for-byte. - Apply this language only to generated wiki files. Do not translate OpenWiki CLI text or runtime messages. - Keep code identifiers, file paths, commands, API names, URLs, and code blocks unchanged where translation would reduce technical accuracy or usability.`; } diff --git a/src/agent/translation-middleware.ts b/src/agent/translation-middleware.ts index c5b35a58..d496c029 100644 --- a/src/agent/translation-middleware.ts +++ b/src/agent/translation-middleware.ts @@ -360,7 +360,7 @@ Translate the Markdown document provided by the user into ${describeLanguage( Rules: - Translate prose, headings, list items, blockquotes, and table cell text. -- In the YAML front matter, translate the human-readable "title", "description", "type", and "tags" values. Keep every front matter key and all other values (URLs, file paths, identifiers, timestamps) byte-for-byte unchanged. +- In the YAML front matter, translate the human-readable "title", "description", and "type" values. Leave the "tags" values in English so they stay stable across pages as cross-cutting aggregation keys. Keep every front matter key as written, and copy all other values (URLs, file paths, identifiers, timestamps) byte-for-byte. - Do NOT translate code identifiers, file paths, commands, API names, URLs, or anything inside inline code spans or fenced code blocks. - Preserve all Markdown syntax, link targets, mermaid fences, and the document's whitespace and structure. - Return ONLY the translated document text, with no explanation, commentary, or surrounding code fences.`; diff --git a/test/prompt.test.ts b/test/prompt.test.ts index c3e758a4..dc0d1291 100644 --- a/test/prompt.test.ts +++ b/test/prompt.test.ts @@ -13,8 +13,10 @@ describe("createSystemPrompt output language", () => { "Write generated wiki prose, headings, table content, and documentation in zh-CN.", ); expect(prompt).toContain( - 'write the human-readable "title", "description", "type", and "tags" values in zh-CN', + 'write the human-readable "title", "description", and "type" values in zh-CN', ); + // Tags stay canonical (an aggregation key), so they are written in English. + expect(prompt).toContain('Write the "tags" values in English'); expect(prompt).toContain( "Apply this language only to generated wiki files.", ); diff --git a/test/translation-middleware.test.ts b/test/translation-middleware.test.ts index 621e794e..491f678a 100644 --- a/test/translation-middleware.test.ts +++ b/test/translation-middleware.test.ts @@ -156,11 +156,13 @@ describe("createWikiTranslationMiddleware beforeAgent", () => { ]); expect(calls[0].system).toContain("Chinese (China)"); expect(calls[0].system).toContain("English"); - // The front matter instruction now covers type and tags, not just title and - // description, so no structural value is left in the source language. - expect(calls[0].system).toContain( - '"title", "description", "type", and "tags"', - ); + // The front matter instruction covers the human-readable title, + // description, and type values, so no reader-facing value is left in the + // source language. + expect(calls[0].system).toContain('"title", "description", and "type"'); + // Tags are a cross-cutting aggregation key, so they stay canonical: the + // prompt tells the model to leave the tags values in English. + expect(calls[0].system).toContain('Leave the "tags" values in English'); // Every translation call is tagged so its tokens are excluded from the // agent's messages stream and never scroll past in the TUI. for (const call of calls) { From b624a06a335d480e953267a625d7a44e24bd0a66 Mon Sep 17 00:00:00 2001 From: Colin Francis Date: Mon, 27 Jul 2026 11:05:00 -0700 Subject: [PATCH 12/13] keep OKF tags canonical, translate only reader-facing front matter --- src/agent/prompt.ts | 1 + test/prompt.test.ts | 7 +++++++ 2 files changed, 8 insertions(+) diff --git a/src/agent/prompt.ts b/src/agent/prompt.ts index eb451c14..9bd3d8d4 100644 --- a/src/agent/prompt.ts +++ b/src/agent/prompt.ts @@ -212,6 +212,7 @@ function createLanguageInstructions(language: string | undefined): string { Output language: - Write generated wiki prose, headings, table content, and documentation in ${language}. +- OpenWiki has already brought existing pages into ${language} in a separate deterministic pass before you run, so treat the wiki as already in ${language}. Do not translate or rewrite an existing page just because it, or the recorded run metadata, still shows a different language; that whole-wiki reconciliation is code-owned. Write only your own new or changed content in ${language} and leave otherwise-accurate pages alone. - In each page's YAML front matter, write the human-readable "title", "description", and "type" values in ${language}. Write the "tags" values in English so they stay stable across languages as cross-cutting aggregation keys. Keep the YAML keys as written, and copy any URL, file path, timestamp, or identifier-like value byte-for-byte. - Apply this language only to generated wiki files. Do not translate OpenWiki CLI text or runtime messages. - Keep code identifiers, file paths, commands, API names, URLs, and code blocks unchanged where translation would reduce technical accuracy or usability.`; diff --git a/test/prompt.test.ts b/test/prompt.test.ts index dc0d1291..a3ca52a7 100644 --- a/test/prompt.test.ts +++ b/test/prompt.test.ts @@ -17,6 +17,13 @@ describe("createSystemPrompt output language", () => { ); // Tags stay canonical (an aggregation key), so they are written in English. expect(prompt).toContain('Write the "tags" values in English'); + // Whole-wiki language reconciliation is code-owned: the agent must not + // re-translate existing pages on a switch, so it never fights the separate + // deterministic translation pass or acts on stale language metadata. + expect(prompt).toContain( + "brought existing pages into zh-CN in a separate deterministic pass", + ); + expect(prompt).toContain("that whole-wiki reconciliation is code-owned"); expect(prompt).toContain( "Apply this language only to generated wiki files.", ); From dbe78941ca2a15300d7231dfac7cfed758554762 Mon Sep 17 00:00:00 2001 From: Colin Francis Date: Mon, 27 Jul 2026 11:41:32 -0700 Subject: [PATCH 13/13] prompt tuning --- src/agent/prompt.ts | 2 +- src/agent/translation-middleware.ts | 2 +- test/prompt.test.ts | 5 +++++ test/translation-middleware.test.ts | 5 +++++ 4 files changed, 12 insertions(+), 2 deletions(-) diff --git a/src/agent/prompt.ts b/src/agent/prompt.ts index 9bd3d8d4..3220391e 100644 --- a/src/agent/prompt.ts +++ b/src/agent/prompt.ts @@ -213,7 +213,7 @@ function createLanguageInstructions(language: string | undefined): string { Output language: - Write generated wiki prose, headings, table content, and documentation in ${language}. - OpenWiki has already brought existing pages into ${language} in a separate deterministic pass before you run, so treat the wiki as already in ${language}. Do not translate or rewrite an existing page just because it, or the recorded run metadata, still shows a different language; that whole-wiki reconciliation is code-owned. Write only your own new or changed content in ${language} and leave otherwise-accurate pages alone. -- In each page's YAML front matter, write the human-readable "title", "description", and "type" values in ${language}. Write the "tags" values in English so they stay stable across languages as cross-cutting aggregation keys. Keep the YAML keys as written, and copy any URL, file path, timestamp, or identifier-like value byte-for-byte. +- In each page's YAML front matter, write the human-readable "title", "description", and "type" values in ${language}. Do this even when the value is dense with product names, feature names, or technical terminology; within those values keep unchanged only literal code identifiers, file paths, commands, and URLs. Write the "tags" values in English so they stay stable across languages as cross-cutting aggregation keys. Keep the YAML keys as written, and copy any URL, file path, timestamp, or identifier-like value byte-for-byte. - Apply this language only to generated wiki files. Do not translate OpenWiki CLI text or runtime messages. - Keep code identifiers, file paths, commands, API names, URLs, and code blocks unchanged where translation would reduce technical accuracy or usability.`; } diff --git a/src/agent/translation-middleware.ts b/src/agent/translation-middleware.ts index d496c029..574c2801 100644 --- a/src/agent/translation-middleware.ts +++ b/src/agent/translation-middleware.ts @@ -360,7 +360,7 @@ Translate the Markdown document provided by the user into ${describeLanguage( Rules: - Translate prose, headings, list items, blockquotes, and table cell text. -- In the YAML front matter, translate the human-readable "title", "description", and "type" values. Leave the "tags" values in English so they stay stable across pages as cross-cutting aggregation keys. Keep every front matter key as written, and copy all other values (URLs, file paths, identifiers, timestamps) byte-for-byte. +- In the YAML front matter, fully translate the human-readable "title", "description", and "type" values, even when they are dense with product names, feature names, or technical terminology; within those values keep unchanged only literal code identifiers, file paths, commands, and URLs. Leave the "tags" values in English so they stay stable across pages as cross-cutting aggregation keys. Keep every front matter key as written, and copy all other values (URLs, file paths, identifiers, timestamps) byte-for-byte. - Do NOT translate code identifiers, file paths, commands, API names, URLs, or anything inside inline code spans or fenced code blocks. - Preserve all Markdown syntax, link targets, mermaid fences, and the document's whitespace and structure. - Return ONLY the translated document text, with no explanation, commentary, or surrounding code fences.`; diff --git a/test/prompt.test.ts b/test/prompt.test.ts index a3ca52a7..16b2eabf 100644 --- a/test/prompt.test.ts +++ b/test/prompt.test.ts @@ -15,6 +15,11 @@ describe("createSystemPrompt output language", () => { expect(prompt).toContain( 'write the human-readable "title", "description", and "type" values in zh-CN', ); + // The field rule must dominate the "keep technical terms unchanged" rule, or + // a technical-term-dense description gets left in the source language. + expect(prompt).toContain( + "dense with product names, feature names, or technical terminology", + ); // Tags stay canonical (an aggregation key), so they are written in English. expect(prompt).toContain('Write the "tags" values in English'); // Whole-wiki language reconciliation is code-owned: the agent must not diff --git a/test/translation-middleware.test.ts b/test/translation-middleware.test.ts index 491f678a..533c9da0 100644 --- a/test/translation-middleware.test.ts +++ b/test/translation-middleware.test.ts @@ -160,6 +160,11 @@ describe("createWikiTranslationMiddleware beforeAgent", () => { // description, and type values, so no reader-facing value is left in the // source language. expect(calls[0].system).toContain('"title", "description", and "type"'); + // The field rule must dominate the "keep technical terms unchanged" rule, or + // a technical-term-dense description gets left in the source language. + expect(calls[0].system).toContain( + "dense with product names, feature names, or technical terminology", + ); // Tags are a cross-cutting aggregation key, so they stay canonical: the // prompt tells the model to leave the tags values in English. expect(calls[0].system).toContain('Leave the "tags" values in English');