diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7b359f24..bf1de18f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -32,6 +32,9 @@ jobs: - name: Verify action pins run: bash scripts/verify-action-pins.sh + - name: Verify sources contain no unapproved non-English text + run: go run scripts/verify-english-only.go + - name: Check formatting run: | unformatted="$(gofmt -s -l .)" diff --git a/AGENTS.md b/AGENTS.md index 68aae24b..359983c4 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -22,8 +22,10 @@ open-code-review (`ocr`) is an AI-powered code review CLI tool written in Go (mo ## Code Style -- After writing code, run `make check` to format and check the code. -- `make check` runs: license check, `go mod tidy`, `gofmt -s -w .`, and `go vet`. +- After writing code, run `make check`. It formats and tidies in place, so there is no need to run `gofmt` or `go vet` separately. +- **Source files are written in English** — comments, identifiers and strings alike. `make english-check` enforces this in CI. It flags any letter outside ASCII, whichever the writing system (Han, kana, Hangul, Cyrillic, and equally the diacritics that spell German or Vietnamese), plus combining accents and fullwidth punctuation (`:`, `(`), which is easy to leave behind in an otherwise English sentence. Symbols and emoji (`─ → ≥ ✅`) pass, since they are not letters. Prose spelled entirely in ASCII (`Loeschen der Datei`, or a romanised transcription) takes a dictionary to spot and stays a matter for review. +- **Translated prose has its own homes, none of them scanned.** `README..md` and `CONTRIBUTING..md` (`zh-CN`, `ja-JP`, `ko-KR`, `ru-RU`); the doc pages under `pages/src/content/docs//` (`en`, `zh`, `ja`, `ru`, Markdown throughout); and the UI copy tables in `pages/src/i18n/.ts`. Markdown is out of scope by extension, so translations go there freely. The i18n tables are `.ts` and would be scanned, so they are exempt by prefix instead — translated UI strings belong in those tables rather than inline in a component. +- **Two escape hatches for the exceptional case, narrower one preferred.** Append an `allow-non-english: ` marker comment to the offending line — the right choice for a handful of lines, such as an encoding fixture or a language-switcher label, and it leaves the rest of the file protected. Only for a whole tree that is inherently non-English, add a prefix to `allowedPrefixes` in `scripts/verify-english-only.go`; it currently holds just `pages/src/i18n/` and `extensions/vscode/`, the latter temporary until the extension's Chinese comments are translated. ## Testing diff --git a/Makefile b/Makefile index 9c75a738..121c5377 100644 --- a/Makefile +++ b/Makefile @@ -2,7 +2,7 @@ build-all dist sha256sum version-info \ build-linux-amd64 build-linux-arm64 build-darwin-amd64 build-darwin-arm64 \ build-windows-amd64 build-windows-arm64 \ - license-check license-add + license-check license-add english-check BINARY_NAME := opencodereview GO := go @@ -64,7 +64,7 @@ fmt: vet: LC_ALL=C $(GO) vet $(PACKAGES) -check: license-check +check: license-check english-check $(GO) mod tidy gofmt -s -w . LC_ALL=C $(GO) vet $(PACKAGES) @@ -73,6 +73,9 @@ check: license-check license-check: @bash scripts/verify-license.sh +english-check: + @$(GO) run scripts/verify-english-only.go + license-add: @bash scripts/add-license.sh diff --git a/action.yml b/action.yml index 74aa5689..883d7bd1 100644 --- a/action.yml +++ b/action.yml @@ -36,7 +36,7 @@ inputs: language: description: >- Review output language, written via `ocr config set language` - (e.g. English, 中文). No env var exists for this. + (e.g. English, Chinese). No env var exists for this. required: false default: 'English' llm_timeout: diff --git a/cmd/opencodereview/output_test.go b/cmd/opencodereview/output_test.go index 2ddadf6f..2742cb67 100644 --- a/cmd/opencodereview/output_test.go +++ b/cmd/opencodereview/output_test.go @@ -94,7 +94,7 @@ func TestSanitizeTerminal(t *testing.T) { {"strips carriage return", "fake\rreal", "fakereal"}, {"empty string", "", ""}, {"only control chars", "\x1b\x07\x00\x7f", ""}, - {"unicode preserved", "代码审查 レビュー 🔍", "代码审查 レビュー 🔍"}, + {"unicode preserved", "代码审查 レビュー 🔍", "代码审查 レビュー 🔍"}, // allow-non-english: fixture asserts non-ASCII output is preserved verbatim {"mixed safe and unsafe", "path\x1b[0m/file.go", "path[0m/file.go"}, {"strips C1 CSI (U+009B)", "before\u009bafter", "beforeafter"}, {"strips C1 OSC (U+009D)", "before\u009dafter", "beforeafter"}, diff --git a/cmd/opencodereview/session_cmd_test.go b/cmd/opencodereview/session_cmd_test.go index ce9a570a..421c97a4 100644 --- a/cmd/opencodereview/session_cmd_test.go +++ b/cmd/opencodereview/session_cmd_test.go @@ -287,11 +287,11 @@ func TestRunSessionShow_MissingID(t *testing.T) { } func TestTruncateUnicode(t *testing.T) { - got := truncate("错误原因:超过限制", 6) + got := truncate("错误原因:超过限制", 6) // allow-non-english: fixture exercises rune-boundary truncation if !strings.HasSuffix(got, "…") { t.Fatalf("expected ellipsis suffix, got %q", got) } - if !strings.Contains(got, "错误") { + if !strings.Contains(got, "错误") { // allow-non-english: fixture exercises rune-boundary truncation t.Fatalf("expected valid truncated unicode text, got %q", got) } } diff --git a/examples/gerrit_ci/post_review_test.py b/examples/gerrit_ci/post_review_test.py index 56ad327e..3b930868 100644 --- a/examples/gerrit_ci/post_review_test.py +++ b/examples/gerrit_ci/post_review_test.py @@ -134,10 +134,10 @@ def test_suggestion_without_existing(self): self.assertNotIn("**Suggestion:**", msg) def test_unicode_comment_roundtrip(self): - content = "空指针解引用:y 可能为 nil" - ri = build([comment(path="pkg/服务.go", content=content)]) - self.assertIn("pkg/服务.go", ri["comments"]) - self.assertIn(content, entry_of(ri, "pkg/服务.go")["message"]) + content = "空指针解引用:y 可能为 nil" # allow-non-english: fixture exercises UTF-8 comment bodies + ri = build([comment(path="pkg/服务.go", content=content)]) # allow-non-english: fixture exercises UTF-8 file paths + self.assertIn("pkg/服务.go", ri["comments"]) # allow-non-english: fixture exercises UTF-8 file paths + self.assertIn(content, entry_of(ri, "pkg/服务.go")["message"]) # allow-non-english: fixture exercises UTF-8 file paths self.assertEqual(json.loads(json.dumps(ri, ensure_ascii=False)), ri) def test_path_with_spaces(self): @@ -539,7 +539,7 @@ def fake_urlopen(req, timeout=None): def test_preemptive_basic_auth_and_utf8_body(self): import base64 - req, _parsed = self.post({"message": "空指针解引用:y 可能为 nil"}) + req, _parsed = self.post({"message": "空指针解引用:y 可能为 nil"}) # allow-non-english: fixture exercises UTF-8 request payloads auth = req.get_header("Authorization") self.assertIsNotNone(auth, "Authorization header must be set preemptively") self.assertTrue(auth.startswith("Basic ")) @@ -547,7 +547,7 @@ def test_preemptive_basic_auth_and_utf8_body(self): base64.b64decode(auth[len("Basic "):]).decode("utf-8"), "review-bot:s3cret-pass", ) - self.assertIn("空指针解引用".encode("utf-8"), req.data) + self.assertIn("空指针解引用".encode("utf-8"), req.data) # allow-non-english: fixture exercises UTF-8 request payloads self.assertIn("application/json", req.get_header("Content-type")) def test_xssi_response_parses(self): diff --git a/internal/config/toolsconfig/tools.json b/internal/config/toolsconfig/tools.json index 4701cf73..59dda84f 100644 --- a/internal/config/toolsconfig/tools.json +++ b/internal/config/toolsconfig/tools.json @@ -98,7 +98,7 @@ "main_task": true, "definition": { "name": "file_read", - "description": "Use this tool to read file content when you need to get context for git diff. You can specify start_line and end_line to view specific parts of the file.\n\n**Line Range Strategy:**\n- Git diff hunk header provides guidance on how to get more relevant context.\n- Git diff hunk header \"@@-x,y +m,n@@\" indicates that the old file has y lines starting from line x, and the new file has n lines starting from line m.\n- For example, when you need to read 50 lines above and below the current changed code block in the new file, set start_line = m - 50, end_line = m + n + 50.\n\n**Example output:**\nFile:path/to/example.go (Total lines: 50)\nIS_TRUNCATED: false\nLINE_RANGE: 10-12\n// The following is the original content of the file\nfunc main() {\n fmt.Println(\"Hello, World!\")\n}\n\n**Limitations:**\n- If the specified range exceeds 500 lines, only 500 lines will be returned with a truncation notice.\n- This tool can only read file content from the modified version (after changes) in git diff.", + "description": "Use this tool to read file content when you need to get context for git diff. You can specify start_line and end_line to view specific parts of the file.\n\n**Line Range Strategy:**\n- Git diff hunk header provides guidance on how to get more relevant context.\n- Git diff hunk header \"@@-x,y +m,n@@\" indicates that the old file has y lines starting from line x, and the new file has n lines starting from line m.\n- For example, when you need to read 50 lines above and below the current changed code block in the new file, set start_line = m - 50, end_line = m + n + 50.\n\n**Example output:**\nFile: path/to/example.go (Total lines: 50)\nIS_TRUNCATED: false\nLINE_RANGE: 10-12\n// The following is the original content of the file\nfunc main() {\n fmt.Println(\"Hello, World!\")\n}\n\n**Limitations:**\n- If the specified range exceeds 500 lines, only 500 lines will be returned with a truncation notice.\n- This tool can only read file content from the modified version (after changes) in git diff.", "parameters": { "type": "object", "properties": { diff --git a/internal/diff/git_test.go b/internal/diff/git_test.go index 4b527506..8a6a35f6 100644 --- a/internal/diff/git_test.go +++ b/internal/diff/git_test.go @@ -81,7 +81,7 @@ func initRepoWithNonASCIIChange(t *testing.T) (string, string) { runGitTest(t, repo, "config", "commit.gpgsign", "false") runGitTest(t, repo, "config", "core.quotepath", "true") - relPath := "src/café/(authenticated)/文件.ts" + relPath := "src/café/(authenticated)/文件.ts" // allow-non-english: fixture exercises non-ASCII paths file := filepath.Join(repo, filepath.FromSlash(relPath)) if err := os.MkdirAll(filepath.Dir(file), 0o755); err != nil { t.Fatalf("create non-ASCII path: %v", err) @@ -153,7 +153,7 @@ func TestWorkspaceDiffPreservesNonASCIIUntrackedPath(t *testing.T) { repo, trackedPath := initRepoWithNonASCIIChange(t) runGitTest(t, repo, "checkout", "--", trackedPath) - untrackedPath := "src/café/(authenticated)/新增.ts" + untrackedPath := "src/café/(authenticated)/新增.ts" // allow-non-english: fixture exercises non-ASCII paths if err := os.WriteFile(filepath.Join(repo, filepath.FromSlash(untrackedPath)), []byte("untracked\n"), 0o644); err != nil { t.Fatalf("write non-ASCII untracked file: %v", err) } diff --git a/internal/session/manifest_test.go b/internal/session/manifest_test.go index 6800a7d5..575b59ea 100644 --- a/internal/session/manifest_test.go +++ b/internal/session/manifest_test.go @@ -692,7 +692,7 @@ func TestSanitizeReasonTruncatesAndSingleLine(t *testing.T) { t.Fatal("newlines not collapsed") } // Multibyte input must not be cut mid-rune. - multibyte := strings.Repeat("世", maxReasonLen+50) + multibyte := strings.Repeat("世", maxReasonLen+50) // allow-non-english: fixture exercises multibyte truncation if !utf8.ValidString(sanitizeReason(multibyte)) { t.Fatal("truncation produced invalid UTF-8") } diff --git a/internal/viewer/server_test.go b/internal/viewer/server_test.go index 6c19e7d8..f8552797 100644 --- a/internal/viewer/server_test.go +++ b/internal/viewer/server_test.go @@ -20,8 +20,8 @@ func TestTruncateText(t *testing.T) { {"truncated with ellipsis", 3, "hello", "hel…"}, {"empty string", 5, "", ""}, {"n=0 always truncates non-empty", 0, "hi", "…"}, - {"unicode shorter than n bytes", 20, "你好世界", "你好世界"}, - {"unicode truncated at byte boundary", 6, "你好世界", "你好…"}, + {"unicode shorter than n bytes", 20, "你好世界", "你好世界"}, // allow-non-english: fixture exercises rune-boundary truncation + {"unicode truncated at byte boundary", 6, "你好世界", "你好…"}, // allow-non-english: fixture exercises rune-boundary truncation } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { diff --git a/pages/src/components/Footer.tsx b/pages/src/components/Footer.tsx index c034fa52..4ecf0ca3 100644 --- a/pages/src/components/Footer.tsx +++ b/pages/src/components/Footer.tsx @@ -10,9 +10,9 @@ import type { Language } from '../i18n/types'; const LANG_OPTIONS: { value: Language; label: string }[] = [ { value: 'en', label: 'English' }, - { value: 'zh', label: '中文' }, - { value: 'ja', label: '日本語' }, - { value: 'ru', label: 'Русский' }, + { value: 'zh', label: '中文' }, // allow-non-english: language options are labelled in their own language + { value: 'ja', label: '日本語' }, // allow-non-english: language options are labelled in their own language + { value: 'ru', label: 'Русский' }, // allow-non-english: language options are labelled in their own language ]; const Footer: React.FC = () => { diff --git a/pages/src/components/HeroSection.tsx b/pages/src/components/HeroSection.tsx index 6a67dbb1..c3b96962 100644 --- a/pages/src/components/HeroSection.tsx +++ b/pages/src/components/HeroSection.tsx @@ -119,7 +119,7 @@ const terminalLines = [ { num: 9, content:   }, { num: 10, content: ─── internal/auth/login.go:42-45 ─── }, { num: 11, content: Consider using bcrypt cost factor ≥ 12 for password hashing. }, - { num: 12, content: }, + { num: 12, content: }, // allow-non-english: fullwidth bar renders the terminal cursor ]; const INSTALL_CHANNELS = [ diff --git a/pages/src/components/MarkdownRenderer.test.tsx b/pages/src/components/MarkdownRenderer.test.tsx index 9977dff1..c9ca582d 100644 --- a/pages/src/components/MarkdownRenderer.test.tsx +++ b/pages/src/components/MarkdownRenderer.test.tsx @@ -6,17 +6,24 @@ import { describe, expect, it } from 'vitest'; import { LanguageProvider } from '../i18n'; import MarkdownRenderer from './MarkdownRenderer'; +// The headings are Russian on purpose: explicit heading IDs exist for text that +// cannot produce a usable ASCII slug on its own. Held in constants rather than +// inline, because a marker comment on the JSX line would render as heading text. +const H2 = 'Что делает навык'; // allow-non-english: fixture heading that cannot produce an ASCII slug +const H4 = 'Публикация'; // allow-non-english: fixture heading that cannot produce an ASCII slug +const CONTENT = `## ${H2} {#what-the-skill-does}\n\n#### ${H4} {#service-account}`; + describe('MarkdownRenderer heading IDs', () => { it('renders an explicit heading ID without displaying its marker', () => { render( - + , ); - const heading = screen.getByRole('heading', { name: 'Что делает навык' }); + const heading = screen.getByRole('heading', { name: H2 }); expect(heading.getAttribute('id')).toBe('what-the-skill-does'); expect(heading.textContent).not.toContain('{#what-the-skill-does}'); - expect(screen.getByRole('heading', { name: 'Публикация', level: 4 }).getAttribute('id')).toBe('service-account'); + expect(screen.getByRole('heading', { name: H4, level: 4 }).getAttribute('id')).toBe('service-account'); }); }); diff --git a/pages/src/components/Navbar.tsx b/pages/src/components/Navbar.tsx index 970adbf4..74a435c1 100644 --- a/pages/src/components/Navbar.tsx +++ b/pages/src/components/Navbar.tsx @@ -12,15 +12,15 @@ import type { Language } from '../i18n/types'; const LANG_OPTIONS: { value: Language; label: string }[] = [ { value: 'en', label: 'English' }, - { value: 'zh', label: '中文' }, - { value: 'ja', label: '日本語' }, - { value: 'ru', label: 'Русский' }, + { value: 'zh', label: '中文' }, // allow-non-english: language options are labelled in their own language + { value: 'ja', label: '日本語' }, // allow-non-english: language options are labelled in their own language + { value: 'ru', label: 'Русский' }, // allow-non-english: language options are labelled in their own language ]; const LANG_BADGE: Record = { en: 'En', - zh: '中', - ja: 'あ', + zh: '中', // allow-non-english: single-glyph locale badge + ja: 'あ', // allow-non-english: single-glyph locale badge ru: 'Ru', }; diff --git a/pages/src/utils/headingId.test.ts b/pages/src/utils/headingId.test.ts index 7e0810e9..8df6887e 100644 --- a/pages/src/utils/headingId.test.ts +++ b/pages/src/utils/headingId.test.ts @@ -5,10 +5,14 @@ import { describe, expect, it } from 'vitest'; import { extractHeadings } from './extractHeadings'; import { parseExplicitHeadingId } from './headingId'; +// Russian on purpose: explicit heading IDs exist for text that cannot produce a +// usable ASCII slug on its own. +const HEADING = 'Что делает навк'; // allow-non-english: fixture heading that cannot produce an ASCII slug + describe('explicit heading IDs', () => { it('separates a trailing explicit ID from the visible heading text', () => { - expect(parseExplicitHeadingId('Что делает навк {#what-the-skill-does}')).toEqual({ - text: 'Что делает навк', + expect(parseExplicitHeadingId(`${HEADING} {#what-the-skill-does}`)).toEqual({ + text: HEADING, id: 'what-the-skill-does', }); }); @@ -18,8 +22,8 @@ describe('explicit heading IDs', () => { }); it('uses the explicit ID in the table of contents without exposing its marker', () => { - expect(extractHeadings('## Что делает навк {#what-the-skill-does}')).toEqual([ - { id: 'what-the-skill-does', text: 'Что делает навк', level: 2 }, + expect(extractHeadings(`## ${HEADING} {#what-the-skill-does}`)).toEqual([ + { id: 'what-the-skill-does', text: HEADING, level: 2 }, ]); }); }); diff --git a/scripts/github-actions/check-translation-sync.test.js b/scripts/github-actions/check-translation-sync.test.js index 44f0bfac..e8aaf587 100644 --- a/scripts/github-actions/check-translation-sync.test.js +++ b/scripts/github-actions/check-translation-sync.test.js @@ -91,8 +91,8 @@ function testIdenticalStructurePasses() { // Same outline, DIFFERENT heading text (simulating translations). Must pass: // the check compares structure, not text. const en = readme(["## What is it?", "### Details", "## Usage"]); - const zh = readme(["## 这是什么?", "### 细节", "## 使用方法"]); - const ja = readme(["## これは何ですか?", "### 詳細", "## 使い方"]); + const zh = readme(["## 这是什么?", "### 细节", "## 使用方法"]); // allow-non-english: fixture mimics translated README headings + const ja = readme(["## これは何ですか?", "### 詳細", "## 使い方"]); // allow-non-english: fixture mimics translated README headings const { ok, errors } = compareReadmeStructures([ { name: "README.md", content: en }, { name: "README.zh-CN.md", content: zh }, diff --git a/scripts/verify-english-only.go b/scripts/verify-english-only.go new file mode 100644 index 00000000..1d521fa3 --- /dev/null +++ b/scripts/verify-english-only.go @@ -0,0 +1,275 @@ +// SPDX-License-Identifier: Apache-2.0 +// Copyright 2026 alibaba/open-code-review Contributors + +//go:build ignore + +// Command verify-english-only fails when unapproved non-English text appears in +// source files. +// +// Comments, identifiers and user-visible strings in this repository are written +// in English so that any contributor can review and maintain every file, +// whichever languages they happen to read. Translated content belongs in the +// locale-specific docs (README.zh-CN.md, pages/src/content/docs/zh/…) and in +// the i18n tables, not in code. +// +// What it detects, and the one thing it cannot: +// +// - Detected: every letter outside ASCII, whichever the writing system. Han, +// kana, Hangul, Cyrillic, Greek, Arabic, Hebrew and Devanagari, and equally +// the diacritics that spell German, French, Turkish or Vietnamese. Plus CJK +// and fullwidth punctuation, and combining accents. +// - Not detected: another language spelled entirely in ASCII — a romanised +// transcription, or German with its umlauts written out ("Loeschen der +// Datei"). Telling that from English needs a dictionary rather than a +// character test, so it stays a matter for review. +// +// Symbols are deliberately left alone: box drawing, arrows, emoji and maths +// (─ → ≥ ≈ ×) are not letters and appear throughout the TUI output on purpose. +// +// Markdown is not scanned: the translated READMEs, CONTRIBUTING files and doc +// pages are legitimately non-English. +// +// Run it directly (the build tag keeps it out of ./... so it does not affect +// go vet, go build or the coverage threshold): +// +// go run scripts/verify-english-only.go +// +// Two escape hatches exist, in order of preference: +// +// 1. Append an "allow-non-english: " marker comment to the offending +// line — the right choice for a handful of lines, e.g. an encoding fixture +// or a language-switcher label. The rest of the file stays protected. +// +// 2. Add a prefix to allowedPrefixes below, for whole trees that are +// inherently non-English (i18n tables) — or, temporarily, for a backlog +// that has not been translated yet. +package main + +import ( + "bufio" + "errors" + "fmt" + "os" + "os/exec" + // path, not path/filepath: every path here comes from git ls-files, which + // always emits forward slashes — on Windows too, since that is how the + // index stores them. The allowedPrefixes entries assume the same. + "path" + "strings" + "unicode" +) + +// scannedExts lists the extensions treated as source files. +var scannedExts = map[string]bool{ + ".go": true, ".ts": true, ".tsx": true, ".js": true, ".cjs": true, + ".mjs": true, ".py": true, ".sh": true, ".ps1": true, ".css": true, + ".html": true, ".yml": true, ".yaml": true, ".json": true, +} + +// scannedNames lists extension-less files that are still source files. +var scannedNames = map[string]bool{"Makefile": true} + +// allowedPrefixes exempts paths whose non-English content is expected. Keep +// each entry narrow and justified; a temporary entry must say what removes it. +var allowedPrefixes = []struct{ prefix, reason string }{ + {"pages/src/i18n/", "translated UI copy for the docs site"}, + {"extensions/vscode/", "TEMPORARY: the extension's comments, test names and zh-cn NLS bundle are still Chinese; drop this entry once they are translated"}, +} + +// exemptMarker on a line suppresses the report for that line. The trailing +// colon is part of the marker so that a bare "allow-non-english" cannot exempt +// a line without saying why. +const exemptMarker = "allow-non-english:" + +// isNonEnglish reports whether r is a letter no English word is written with, +// or one of the CJK/fullwidth punctuation forms. +// +// The rule is "a letter outside ASCII", not "a letter outside Latin". Written +// English needs no letter beyond the ASCII 26, so anything past that is another +// language: Cyrillic and Han as obviously as the diacritics of German, French or +// Turkish. Scripts are not enumerated, which keeps the rule stable as the +// contributor base grows — one that nobody has contributed in yet is covered on +// the day it arrives, with no edit here. +// +// Testing for letters, rather than for non-ASCII bytes, is what keeps symbols +// out of scope: the box drawing, arrows, emoji and maths in the TUI output are +// not letters, and neither are the em dashes used throughout these comments. A +// plain non-ASCII test would flag every one of them. +// +// Common and Inherited are the exception. Those two scripts hold the characters +// belonging to no writing system in particular, and the letterlike symbols among +// them are letters only by Unicode category: the information source (U+2139, +// category Ll) that renders as an info icon, the script small l (U+2113), the +// capitals of the maths alphabets. None of them writes a word in any language. +// +// Letterlike forms that Unicode does assign to a real script stay in scope, so +// the ohm sign (U+2126, script Greek because it is equivalent to U+03A9) is +// reported like any other Greek letter. A comment that spells sigma or omega as +// a glyph therefore needs a marker — deliberate, since exempting Greek to allow +// maths notation would exempt Greek prose with it. +// +// Punctuation is checked separately, and matters as much as letters: a +// fullwidth colon (U+FF1A) or comma (U+FF0C) left in an English sentence is a +// typo that reads as correct and is invisible in review. Vertical forms +// (U+FE10–U+FE19), CJK compatibility forms (U+FE30–U+FE4F) and small form +// variants (U+FE50–U+FE6F) are covered alongside the fullwidth block. +func isNonEnglish(r rune) bool { + switch { + case r < 0x80: // ASCII, the overwhelming majority of every scanned line + return false + case unicode.IsLetter(r) && + !unicode.Is(unicode.Common, r) && + !unicode.Is(unicode.Inherited, r): + return true + case r >= 0x0300 && r <= 0x036F: + // Combining diacritical marks, so that the decomposed spelling of an + // accented letter is caught too: NFD writes e-acute as "e" plus U+0301, + // where the letter itself is plain ASCII and the accent carries the + // language. Variation selectors (U+FE0F, which follows an emoji) are + // combining marks as well, but sit outside this block and pass. + return true + case r >= 0x3000 && r <= 0x303F: // CJK Symbols and Punctuation + return true + case r >= 0xFE10 && r <= 0xFE19: // Vertical Forms + return true + case r >= 0xFE30 && r <= 0xFE6F: // CJK Compatibility Forms + Small Form Variants + return true + case r >= 0xFF00 && r <= 0xFFEF: // Halfwidth and Fullwidth Forms + return true + } + return false +} + +func isScanned(file string) bool { + if scannedNames[path.Base(file)] { + return true + } + return scannedExts[path.Ext(file)] +} + +func allowedPrefix(file string) bool { + for _, a := range allowedPrefixes { + if strings.HasPrefix(file, a.prefix) { + return true + } + } + return false +} + +// errReported marks a failure that run has already written to stderr in full, +// so main does not print a redundant one-line summary after the report. +var errReported = errors.New("findings already reported") + +type finding struct { + file string + line int + text string + char rune +} + +func scan(file string) ([]finding, error) { + f, err := os.Open(file) + if err != nil { + return nil, err + } + defer f.Close() + + var found []finding + sc := bufio.NewScanner(f) + sc.Buffer(make([]byte, 0, 64*1024), 8*1024*1024) + for n := 1; sc.Scan(); n++ { + line := sc.Text() + if strings.Contains(line, exemptMarker) { + continue + } + for _, r := range line { + if isNonEnglish(r) { + found = append(found, finding{file: file, line: n, text: strings.TrimSpace(line), char: r}) + break + } + } + } + return found, sc.Err() +} + +// trim shortens a reported line so the report stays readable. +func trim(s string) string { + const max = 100 + if len([]rune(s)) <= max { + return s + } + return string([]rune(s)[:max]) + "…" +} + +func run() error { + // --others --exclude-standard includes files that are not committed yet, so + // a new file is checked before it lands rather than the run after. Ignored + // paths (dist/, node_modules/) stay out. + // + // -z separates paths with NUL and emits them verbatim; without it git quotes + // and escapes any path that is not plain ASCII — exactly the kind of path + // internal/diff/git_test.go has fixtures for. + out, err := exec.Command("git", "ls-files", "-z", "--cached", "--others", "--exclude-standard").Output() + if err != nil { + // Output() fills ExitError.Stderr; without it the error reads as a bare + // "exit status 128" and the CI log never shows what git complained about. + var exitErr *exec.ExitError + if errors.As(err, &exitErr) && len(exitErr.Stderr) > 0 { + return fmt.Errorf("git ls-files: %w: %s", err, strings.TrimSpace(string(exitErr.Stderr))) + } + return fmt.Errorf("git ls-files: %w", err) + } + + var findings []finding + var scanned int + // NUL-terminated, so the final element is empty; the file == "" guard below + // drops it. Do not trim the output: a path may legitimately end in a space. + for _, file := range strings.Split(string(out), "\x00") { + if file == "" || !isScanned(file) || allowedPrefix(file) { + continue + } + if _, err := os.Stat(file); err != nil { + continue // deleted but still indexed + } + scanned++ + found, err := scan(file) + if err != nil { + return fmt.Errorf("scan %s: %w", file, err) + } + findings = append(findings, found...) + } + + if len(findings) > 0 { + fmt.Fprintf(os.Stderr, "ERROR: unapproved non-English text found in %d line(s):\n", len(findings)) + for _, f := range findings { + fmt.Fprintf(os.Stderr, " %s:%d: %q in %s\n", f.file, f.line, f.char, trim(f.text)) + } + fmt.Fprintf(os.Stderr, ` +Source files are English-only: comments, identifiers and strings alike. +Translated prose belongs in README..md, pages/src/content/docs// +or an i18n table. + +If the non-English text is intentional — an encoding fixture, a +language-switcher label — append a marker comment, including the reason, to +that line: + + {"multibyte truncation", 6, "..."}, // %s fixture exercises rune boundaries + +For a whole tree that is inherently non-English, add a prefix to +allowedPrefixes in scripts/verify-english-only.go instead. +`, exemptMarker) + return errReported + } + + fmt.Printf("No unapproved non-English text in %d scanned source files.\n", scanned) + return nil +} + +func main() { + if err := run(); err != nil { + if !errors.Is(err, errReported) { + fmt.Fprintln(os.Stderr, "verify-english-only:", err) + } + os.Exit(1) + } +}