codegen: precompute corpus-wide context once and share across generate() calls

.changes/unreleased/added-20260830-171238.yaml

@@ -0,0 +1,4 @@
+kind: Added
+body: |-
+    **`SharedCorpusContext` lets a workspace that generates one crate per proto package pay the corpus-wide codegen analysis once** (#401, closes #400). `generate()` re-derives which oneof variants are unboxed, which message fields are stored inline, and the per-symbol comment map from the whole `FileDescriptorSet` on every call, although none of that depends on the per-package `extern_paths`; a ~2900-package workspace paid that walk 2900 times. Build a `SharedCorpusContext::new(&files, &config)` once and set `CodeGenConfig::shared_corpus_context` on each per-package config; the output is byte-identical, and a context built from a different corpus or different oneof/pointer-repr rules is refused with `CodeGenError::SharedCorpusContextMismatch` rather than silently resolving against the wrong one. The corpus comparison is structural, not by file name, so a regenerated `FileDescriptorSet` with edited content is refused too; it costs `O(corpus)` per call, measured at roughly a quarter of the name-only comparison's saving on that workspace and still well ahead of not sharing. Off by default.
+time: 2026-08-30T17:12:38.739703-04:00

buffa-codegen/src/context.rs

@@ -2,6 +2,7 @@
 
 use std::borrow::Cow;
 use std::collections::{HashMap, HashSet};
+use std::sync::Arc;
 
 use crate::features::{self, ResolvedFeatures};
 use crate::generated::descriptor::{DescriptorProto, EnumDescriptorProto, FileDescriptorProto};
@@ -86,7 +87,7 @@ pub struct CodeGenContext<'a> {
     /// translation means codegen call sites can look up comments by the
     /// proto FQN they already have, rather than threading index-based paths
     /// through every function signature.
-    comment_map: HashMap<String, String>,
+    comment_map: Arc<HashMap<String, String>>,
     /// Deconflicted module name for each top-level message, keyed by the
     /// leading-dot FQN (`".pkg.Msg"`).
     ///
@@ -101,13 +102,13 @@ pub struct CodeGenContext<'a> {
     /// `config.unboxed_oneof_fields` whose oneof variants are stored inline.
     /// Built once by [`resolve_unboxed_variants`](crate::oneof::resolve_unboxed_variants);
     /// never contains recursive variants. See [`oneof_unboxed`](Self::oneof_unboxed).
-    unboxed_oneof_variants: HashSet<String>,
+    unboxed_oneof_variants: Arc<HashSet<String>>,
     /// Singular message-field paths (leading-dot form) whose resolved
     /// [`PointerRepr`](crate::PointerRepr) is `Inline`. Built once by
     /// [`resolve_inlined_fields`](crate::oneof::resolve_inlined_fields); never
     /// contains recursive fields. [`pointer_repr`](Self::pointer_repr) demotes
     /// any raw `Inline` not in this set to `Box`.
-    inlined_message_fields: HashSet<String>,
+    inlined_message_fields: Arc<HashSet<String>>,
     /// Non-fatal diagnostics accumulated during generation (e.g. an enum whose
     /// idiomatic CamelCase aliases were suppressed by a naming conflict).
     ///
@@ -268,19 +269,29 @@ impl<'a> CodeGenContext<'a> {
         let mut type_map = HashMap::new();
         let mut package_of = HashMap::new();
         let mut enum_closedness = HashMap::new();
-        let mut comment_map = HashMap::new();
         let mut nested_module_names = HashMap::new();
-        let msg_index = crate::oneof::message_index(files);
-        let unboxed_oneof_variants = crate::oneof::resolve_unboxed_variants(
-            &msg_index,
-            &config.unboxed_oneof_fields,
-            &config.pointer_fields,
-        );
-        let inlined_message_fields = crate::oneof::resolve_inlined_fields(
-            &msg_index,
-            &config.unboxed_oneof_fields,
-            &config.pointer_fields,
-        );
+
+        // Reuse the corpus-wide state a caller precomputed with
+        // `SharedCorpusContext::new`: it is identical for the same `files`
+        // and oneof/pointer-repr rules regardless of `effective_extern_paths`
+        // (the one thing that legitimately varies per call in a
+        // one-crate-per-package workspace), so this is an `Arc` clone
+        // instead of a fresh corpus-wide walk. `generate` has already
+        // checked that the context matches this call; this entry point is
+        // infallible, so it only asserts in debug builds.
+        let shared = match &config.shared_corpus_context {
+            Some(shared) => {
+                debug_assert!(
+                    shared.check(files, config).is_ok(),
+                    "shared_corpus_context does not match this call's files or rules"
+                );
+                shared.clone()
+            }
+            None => crate::SharedCorpusContext::new(files, config),
+        };
+        let unboxed_oneof_variants = Arc::clone(shared.unboxed_oneof_variants());
+        let inlined_message_fields = Arc::clone(shared.inlined_message_fields());
+        let comment_map = Arc::clone(shared.comment_map());
 
         // Pre-pass: collect every package and top-level message name in the
         // descriptor set so nested-types module deconfliction (issue #135) is
@@ -335,7 +346,6 @@ impl<'a> CodeGenContext<'a> {
         }
 
         for file in files {
-            comment_map.extend(crate::comments::fqn_comments(file));
             let package = file.package.as_deref().unwrap_or("");
             let file_features = features::for_file(file);
             let proto_prefix = if package.is_empty() {

buffa-codegen/src/lib.rs

@@ -1500,6 +1500,19 @@ pub struct CodeGenConfig {
     ///
     /// `None` (the default) keeps the `super::`-relative behavior.
     pub shared_descriptor_pool_root: Option<String>,
+    /// Reuse a [`SharedCorpusContext`] built once by
+    /// [`SharedCorpusContext::new`] instead of re-deriving the corpus-wide
+    /// oneof/pointer-repr resolution and comment collection on every
+    /// `generate()` call — the win for a one-crate-per-package workspace that
+    /// generates each package from one whole-corpus `FileDescriptorSet` with
+    /// only `extern_paths` varying per call. The context must have been built
+    /// from the same `files` and the same
+    /// `unboxed_oneof_fields`/`pointer_fields` as the call using it;
+    /// `generate` checks and returns
+    /// [`CodeGenError::SharedCorpusContextMismatch`] otherwise.
+    ///
+    /// `None` (the default) recomputes on every call.
+    pub shared_corpus_context: Option<SharedCorpusContext>,
     /// Gate the reflection impls behind a `reflect` crate feature, *without*
     /// gating json/views/text (unlike
     /// [`gate_impls_on_crate_features`](Self::gate_impls_on_crate_features),
@@ -1739,6 +1752,7 @@ impl Default for CodeGenConfig {
             generate_reflection_vtable: false,
             shared_descriptor_pool: false,
             shared_descriptor_pool_root: None,
+            shared_corpus_context: None,
             gate_reflect_on_crate_feature: false,
             idiomatic_enum_aliases: true,
             idiomatic_imports: false,
@@ -2777,6 +2791,9 @@ pub fn generate_with_diagnostics(
     let file_descriptors: &[FileDescriptorProto] =
         overridden.as_ref().map_or(file_descriptors, |o| &o.files);
 
+    if let Some(shared) = &config.shared_corpus_context {
+        shared.check(file_descriptors, config)?;
+    }
     let ctx = context::CodeGenContext::for_generate(file_descriptors, files_to_generate, config);
 
     // An inert rule means the user opted a path out of its default semantics
@@ -3032,6 +3049,141 @@ pub enum IncludeMode<'a> {
     OutDir,
 }
 
+/// The corpus-wide state `CodeGenContext` derives on every `generate()`
+/// call — which oneof variants are unboxed, which message fields are stored
+/// inline, and the raw comment text per fully-qualified name — none of which
+/// depends on the extern paths that legitimately vary per call in a
+/// one-crate-per-package workspace. Build one with [`SharedCorpusContext::new`]
+/// from the whole corpus and set [`CodeGenConfig::shared_corpus_context`] on
+/// every per-package config to skip that work on each call.
+///
+/// A context is only valid for the exact `files` and the same
+/// `unboxed_oneof_fields`/`pointer_fields` it was built from; `generate`
+/// checks both and returns [`CodeGenError::SharedCorpusContextMismatch`]
+/// rather than emitting code resolved against a different corpus. The
+/// context holds the corpus comment map for as long as any clone of it lives,
+/// so drop it once code generation is done. Cloning shares the precomputed
+/// state (`Arc`s inside), and the type is `Send + Sync`, so one context can
+/// be handed to parallel per-package `generate()` calls.
+///
+/// # Examples
+///
+/// ```
+/// use buffa_codegen::{CodeGenConfig, SharedCorpusContext};
+/// # use buffa_codegen::generated::descriptor::FileDescriptorProto;
+/// # let corpus: Vec<FileDescriptorProto> = Vec::new();
+/// # let packages: Vec<(String, Vec<String>)> = Vec::new();
+/// let base = CodeGenConfig::default();
+/// let shared = SharedCorpusContext::new(&corpus, &base);
+/// for (extern_prefix, files_to_generate) in packages {
+///     let mut config = base.clone();
+///     config.extern_paths.push((extern_prefix, "::other_crate".to_string()));
+///     config.shared_corpus_context = Some(shared.clone());
+///     let _files = buffa_codegen::generate(&corpus, &files_to_generate, &config)?;
+/// }
+/// # Ok::<(), buffa_codegen::CodeGenError>(())
+/// ```
+#[derive(Clone)]
+pub struct SharedCorpusContext {
+    /// The exact corpus this context was built from, compared by full
+    /// structural equality in [`check`](Self::check) rather than just file
+    /// names — see `check` for why.
+    files: std::sync::Arc<Vec<generated::descriptor::FileDescriptorProto>>,
+    unboxed_oneof_fields: std::sync::Arc<Vec<String>>,
+    pointer_fields: std::sync::Arc<Vec<(String, PointerRepr)>>,
+    unboxed_oneof_variants: std::sync::Arc<std::collections::HashSet<String>>,
+    inlined_message_fields: std::sync::Arc<std::collections::HashSet<String>>,
+    comment_map: std::sync::Arc<std::collections::HashMap<String, String>>,
+}
+
+impl SharedCorpusContext {
+    /// Precompute the corpus-wide state for `files` under `config`'s oneof
+    /// and pointer-representation rules, for reuse across every `generate()`
+    /// call that passes the same `files` and rules (see
+    /// [`CodeGenConfig::shared_corpus_context`]).
+    #[must_use]
+    pub fn new(
+        files: &[generated::descriptor::FileDescriptorProto],
+        config: &CodeGenConfig,
+    ) -> Self {
+        let msg_index = oneof::message_index(files);
+        let unboxed_oneof_variants = oneof::resolve_unboxed_variants(
+            &msg_index,
+            &config.unboxed_oneof_fields,
+            &config.pointer_fields,
+        );
+        let inlined_message_fields = oneof::resolve_inlined_fields(
+            &msg_index,
+            &config.unboxed_oneof_fields,
+            &config.pointer_fields,
+        );
+        let comment_map = files.iter().flat_map(comments::fqn_comments).collect();
+        Self {
+            files: std::sync::Arc::new(files.to_vec()),
+            unboxed_oneof_fields: std::sync::Arc::new(config.unboxed_oneof_fields.clone()),
+            pointer_fields: std::sync::Arc::new(config.pointer_fields.clone()),
+            unboxed_oneof_variants: std::sync::Arc::new(unboxed_oneof_variants),
+            inlined_message_fields: std::sync::Arc::new(inlined_message_fields),
+            comment_map: std::sync::Arc::new(comment_map),
+        }
+    }
+
+    /// Refuse a context built from a different corpus or different
+    /// oneof/pointer-repr rules than this call's. Compares the full corpus
+    /// by structural equality rather than just file names, since
+    /// `SharedCorpusContext` has no lifetime tying it to the caller's
+    /// `files` — a `Vec` can keep its allocation across an ordinary
+    /// `pop()` + `push()` of a different file, so a pointer/length check
+    /// would be unsound. O(corpus) per call; ~25-28% added wall/CPU time on
+    /// a 2895-crate corpus versus a name-only comparison, still a ~2.3x win
+    /// over not sharing the context at all.
+    pub(crate) fn check(
+        &self,
+        files: &[generated::descriptor::FileDescriptorProto],
+        config: &CodeGenConfig,
+    ) -> Result<(), CodeGenError> {
+        let what = if *files != self.files[..] {
+            "files"
+        } else if *self.unboxed_oneof_fields != config.unboxed_oneof_fields {
+            "unboxed_oneof_fields"
+        } else if *self.pointer_fields != config.pointer_fields {
+            "pointer_fields"
+        } else {
+            return Ok(());
+        };
+        Err(CodeGenError::SharedCorpusContextMismatch { what })
+    }
+
+    pub(crate) fn unboxed_oneof_variants(
+        &self,
+    ) -> &std::sync::Arc<std::collections::HashSet<String>> {
+        &self.unboxed_oneof_variants
+    }
+
+    pub(crate) fn inlined_message_fields(
+        &self,
+    ) -> &std::sync::Arc<std::collections::HashSet<String>> {
+        &self.inlined_message_fields
+    }
+
+    pub(crate) fn comment_map(&self) -> &std::sync::Arc<std::collections::HashMap<String, String>> {
+        &self.comment_map
+    }
+}
+
+// The comment map is the whole corpus's comment text; print sizes, not
+// contents, so a `{:?}` of a `CodeGenConfig` stays readable.
+impl core::fmt::Debug for SharedCorpusContext {
+    fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
+        f.debug_struct("SharedCorpusContext")
+            .field("files", &self.files.len())
+            .field("unboxed_oneof_variants", &self.unboxed_oneof_variants.len())
+            .field("inlined_message_fields", &self.inlined_message_fields.len())
+            .field("comments", &self.comment_map.len())
+            .finish()
+    }
+}
+
 /// Encode the shared reflection `FileDescriptorSet` for a codegen run, with
 /// `source_code_info` stripped, ready to embed once at the module-tree root in
 /// [shared-pool mode](CodeGenConfig::shared_descriptor_pool).
@@ -4382,6 +4534,19 @@ pub enum CodeGenError {
          extensions). Rename the proto element."
     )]
     ReservedModuleName { name: String, location: String },
+    /// [`CodeGenConfig::shared_corpus_context`] was built from a different
+    /// corpus or different oneof/pointer-repr rules than this call's.
+    ///
+    /// `what` names the mismatch: `"files"`, `"unboxed_oneof_fields"`, or
+    /// `"pointer_fields"`. Build the context with
+    /// [`SharedCorpusContext::new`] from the same `files` and config as every
+    /// `generate()` call that uses it.
+    #[error(
+        "shared_corpus_context was built from different {what} than this \
+         generate() call; build it from the same files and oneof/pointer-repr \
+         config as every call that uses it"
+    )]
+    SharedCorpusContextMismatch { what: &'static str },
     /// The input contains a message with `option message_set_wire_format = true`
     /// but [`CodeGenConfig::allow_message_set`] was not set.
     #[error(

buffa-codegen/src/tests/mod.rs

@@ -68,6 +68,7 @@ mod proto2;
 mod reexports;
 mod reflect_view;
 mod repeated_type;
+mod shared_corpus_context;
 mod shared_pool;
 mod size_arithmetic;
 mod view_codegen;