From 6f5b6e4dc6decca06f5876688c44bb8fb726487e Mon Sep 17 00:00:00 2001 From: CTO Agent Date: Wed, 8 Jul 2026 11:47:41 -0400 Subject: [PATCH 1/2] perf: anonymize files in parallel with a process pool Each file gets its own TraceAnonymizer, so files are fully independent and can be processed concurrently. Directory anonymization now fans out over a ProcessPoolExecutor sized to the machine's CPU count, falling back to the sequential path for small batches where process startup would cost more than it saves. Co-Authored-By: Claude Fable 5 --- src/teich/anonymize.py | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/src/teich/anonymize.py b/src/teich/anonymize.py index 4df3627..8b2890f 100644 --- a/src/teich/anonymize.py +++ b/src/teich/anonymize.py @@ -2,9 +2,11 @@ from __future__ import annotations +from concurrent.futures import ProcessPoolExecutor from dataclasses import dataclass, field from pathlib import Path import hashlib +import os import json import re import shutil @@ -24,6 +26,10 @@ ".log", } +# ponytail: process startup isn't free — only fan out when there are enough +# files for the parallelism to pay for itself. +_MIN_FILES_FOR_PARALLEL = 8 + @dataclass class AnonymizeFileReport: @@ -73,11 +79,17 @@ def anonymize_path(input_path: Path, output_path: Path, *, in_place: bool = Fals report.files.append(file_report) return report - for source_file in sorted(path for path in input_path.rglob("*") if path.is_file()): - relative_path = source_file.relative_to(input_path) - destination = source_file if in_place else output_path / relative_path - file_report = _anonymize_file(source_file, destination) - report.files.append(file_report) + source_files = sorted(path for path in input_path.rglob("*") if path.is_file()) + destinations = [source if in_place else output_path / source.relative_to(input_path) for source in source_files] + workers = min(os.cpu_count() or 1, len(source_files)) + if workers > 1 and len(source_files) >= _MIN_FILES_FOR_PARALLEL: + # Each file is anonymized independently (fresh TraceAnonymizer per + # file), so files can be processed in parallel safely. + with ProcessPoolExecutor(max_workers=workers) as executor: + report.files.extend(executor.map(_anonymize_file, source_files, destinations)) + else: + for source_file, destination in zip(source_files, destinations): + report.files.append(_anonymize_file(source_file, destination)) return report From 3d83cfaa2d36aa4074230b87fd6e737ff121751a Mon Sep 17 00:00:00 2001 From: Arman Rafiee Date: Fri, 10 Jul 2026 02:42:46 -0400 Subject: [PATCH 2/2] fix Windows anonymizer worker limit --- src/teich/anonymize.py | 11 ++++++++++- tests/test_extract_anonymize_cli.py | 8 ++++++++ 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/src/teich/anonymize.py b/src/teich/anonymize.py index 8b2890f..beac800 100644 --- a/src/teich/anonymize.py +++ b/src/teich/anonymize.py @@ -11,6 +11,7 @@ import re import shutil import string +import sys from tempfile import NamedTemporaryFile from typing import Any @@ -29,6 +30,14 @@ # ponytail: process startup isn't free — only fan out when there are enough # files for the parallelism to pay for itself. _MIN_FILES_FOR_PARALLEL = 8 +_WINDOWS_MAX_PROCESS_WORKERS = 61 + + +def _process_worker_count(file_count: int) -> int: + workers = min(os.cpu_count() or 1, file_count) + if sys.platform == "win32": + workers = min(workers, _WINDOWS_MAX_PROCESS_WORKERS) + return workers @dataclass @@ -81,7 +90,7 @@ def anonymize_path(input_path: Path, output_path: Path, *, in_place: bool = Fals source_files = sorted(path for path in input_path.rglob("*") if path.is_file()) destinations = [source if in_place else output_path / source.relative_to(input_path) for source in source_files] - workers = min(os.cpu_count() or 1, len(source_files)) + workers = _process_worker_count(len(source_files)) if workers > 1 and len(source_files) >= _MIN_FILES_FOR_PARALLEL: # Each file is anonymized independently (fresh TraceAnonymizer per # file), so files can be processed in parallel safely. diff --git a/tests/test_extract_anonymize_cli.py b/tests/test_extract_anonymize_cli.py index 01de554..c765458 100644 --- a/tests/test_extract_anonymize_cli.py +++ b/tests/test_extract_anonymize_cli.py @@ -7,6 +7,7 @@ from typer.testing import CliRunner +from teich import anonymize as anonymize_module from teich.converter import convert_traces_to_training_data from teich import extract as extract_module from teich.extract import CURSOR_EXTRACTION_NOTICE, default_session_sources @@ -1642,6 +1643,13 @@ def test_anonymize_replaces_emails_keys_and_home_usernames_consistently(tmp_path assert "api_key=1" in result.output +def test_anonymize_parallel_worker_count_caps_windows(monkeypatch): + monkeypatch.setattr(anonymize_module.os, "cpu_count", lambda: 128) + monkeypatch.setattr(anonymize_module.sys, "platform", "win32") + + assert anonymize_module._process_worker_count(100) == 61 + + def test_anonymize_jsonl_preserves_valid_json_after_escaped_path_replacements(tmp_path: Path): input_dir = tmp_path / "input" input_dir.mkdir()