[1/n] add git ref storage for older API versions

.config/nextest.toml

@@ -1,2 +1,6 @@
+# Required for NEXTEST_BIN_EXE_ with hyphens in binary names replaced by
+# underscores.
+nextest-version = "0.9.113"
+
 [profile.ci]
 fail-fast = false

clippy.toml

@@ -0,0 +1,11 @@
+# Clippy configuration for dropshot-api-manager.
+
+# Ban non-atomic file writes. Use atomicwrites::AtomicFile instead to prevent
+# corruption on interruption.
+[[disallowed-methods]]
+path = "std::fs::write"
+reason = "use atomicwrites instead to prevent corruption on interruption"
+
+[[disallowed-methods]]
+path = "fs_err::write"
+reason = "use atomicwrites instead to prevent corruption on interruption"

crates/dropshot-api-manager-types/src/validation.rs

@@ -130,42 +130,75 @@ impl ApiSpecFileName {
     }
 
     /// Returns the path of this file relative to the root of the OpenAPI
-    /// documents
+    /// documents.
     pub fn path(&self) -> Utf8PathBuf {
         match &self.kind {
             ApiSpecFileNameKind::Lockstep => {
                 Utf8PathBuf::from_iter([self.basename()])
             }
-            ApiSpecFileNameKind::Versioned { .. } => Utf8PathBuf::from_iter([
-                self.ident.deref().clone(),
-                self.basename(),
-            ]),
+            ApiSpecFileNameKind::Versioned { .. }
+            | ApiSpecFileNameKind::VersionedGitRef { .. } => {
+                Utf8PathBuf::from_iter([
+                    self.ident.deref().clone(),
+                    self.basename(),
+                ])
+            }
         }
     }
 
-    /// Returns the base name of this file path
+    /// Returns the base name of this file path.
     pub fn basename(&self) -> String {
         match &self.kind {
             ApiSpecFileNameKind::Lockstep => format!("{}.json", self.ident),
             ApiSpecFileNameKind::Versioned { version, hash } => {
                 format!("{}-{}-{}.json", self.ident, version, hash)
             }
+            ApiSpecFileNameKind::VersionedGitRef { version, hash } => {
+                format!("{}-{}-{}.json.gitref", self.ident, version, hash)
+            }
         }
     }
 
-    /// For versioned APIs, returns the version part of the filename
+    /// For versioned APIs, returns the version part of the filename.
     pub fn version(&self) -> Option<&semver::Version> {
         match &self.kind {
             ApiSpecFileNameKind::Lockstep => None,
-            ApiSpecFileNameKind::Versioned { version, .. } => Some(version),
+            ApiSpecFileNameKind::Versioned { version, .. }
+            | ApiSpecFileNameKind::VersionedGitRef { version, .. } => {
+                Some(version)
+            }
         }
     }
 
-    /// For versioned APIs, returns the hash part of the filename
+    /// For versioned APIs, returns the hash part of the filename.
     pub fn hash(&self) -> Option<&str> {
         match &self.kind {
             ApiSpecFileNameKind::Lockstep => None,
-            ApiSpecFileNameKind::Versioned { hash, .. } => Some(hash),
+            ApiSpecFileNameKind::Versioned { hash, .. }
+            | ApiSpecFileNameKind::VersionedGitRef { hash, .. } => Some(hash),
+        }
+    }
+
+    /// Returns true if this is a git ref file.
+    pub fn is_git_ref(&self) -> bool {
+        matches!(self.kind, ApiSpecFileNameKind::VersionedGitRef { .. })
+    }
+
+    /// Converts a `VersionedGitRef` to its `Versioned` equivalent.
+    ///
+    /// For non-git ref files, returns a clone of self.
+    pub fn to_json_filename(&self) -> ApiSpecFileName {
+        match &self.kind {
+            ApiSpecFileNameKind::VersionedGitRef { version, hash } => {
+                ApiSpecFileName::new(
+                    self.ident.clone(),
+                    ApiSpecFileNameKind::Versioned {
+                        version: version.clone(),
+                        hash: hash.clone(),
+                    },
+                )
+            }
+            _ => self.clone(),
         }
     }
 }
@@ -182,6 +215,17 @@ pub enum ApiSpecFileNameKind {
         /// The hash of the file contents.
         hash: String,
     },
+    /// The file's path implies a versioned API stored as a git ref.
+    ///
+    /// Instead of storing the full JSON content, a `.gitref` file contains a
+    /// reference in the format `commit:path` that can be used to retrieve the
+    /// content via `git show`.
+    VersionedGitRef {
+        /// The version of the API this document describes.
+        version: semver::Version,
+        /// The hash of the file contents (from the original file).
+        hash: String,
+    },
 }
 
 /// Newtype for API identifiers

crates/dropshot-api-manager/README.md

@@ -75,6 +75,7 @@ In addition, if you have versioned APIs (see below):
 
 * The OpenAPI manager must be run within a Git repository.
 * You must also ensure that `git` is available on the command line, or that the `GIT` environment variable is set to the location of the Git binary.
+* Shallow clones don't work; full Git history is required. See [_CI and shallow clones_](#ci-and-shallow-clones) below.
 
 #### API crates
 
@@ -227,6 +228,30 @@ Again, we assume you're starting from a fresh branch from "main".
 
 As of this writing, every API has exactly one Rust client package and it's always generated from the latest version of the API.  Per RFD 532, this is sufficient for APIs that are server-side-only versioned.  For APIs that will be client-side versioned, you may need to create additional Rust packages that use Progenitor to generate clients based on older OpenAPI documents.  This has not been done before but is believed to be straightforward.
 
+## Git ref storage
+
+For versioned APIs, the Dropshot API manager can optionally store older API versions as *Git ref files* instead of full JSON files. A Git ref file is a small text file (with a `.gitref` extension) that points to the JSON content at a specific Git commit.
+
+### Benefits
+
+- **Meaningful diffs on GitHub.** New API versions appear as renames of the previous version, so GitHub shows what actually changed rather than thousands of added lines.
+- **Blame works across versions.** `git blame` on the latest version traces history back through previous versions.
+- **Smaller repository checkout on disk.** Git ref files are ~100 bytes each, replacing large JSON files in the working copy.
+
+For more background, see [RFD 634](https://rfd.shared.oxide.computer/rfd/0634).
+
+### Tradeoffs
+
+- **Requires Git history.** Shallow clones won't work, so CI must use `fetch-depth: 0`. See [_CI and shallow clones_](#ci-and-shallow-clones) below.
+- **Rename-rename conflicts.** Parallel branches adding versions to the same API produce merge conflicts. See [_Git ref merge conflicts_](#git-ref-merge-conflicts) below.
+- **Tools must dereference Git ref files.** Tools that need older API versions must know how to read Git ref files, though most workflows only use the `-latest.json` symlink.
+
+### Enabling Git ref storage
+
+To enable Git ref storage for an API, use the `.with_git_ref_storage()` builder method when configuring the API in your integration point. You can also call `.with_git_ref_storage()` on a combined `ManagedApi` to turn Git ref storage on by default. Use `.disable_git_ref_storage()` to opt an API out of default Git ref storage.
+
+For details on the file format and conversion rules, see [_Git ref storage details_](#git-ref-storage-details) below.
+
 ## More about versioned APIs
 
 The idea behind versioned APIs is:
@@ -293,6 +318,18 @@ You generally don't need to think about any of this to use the tool.  Like with
 2. You defined a new version, but forgot to annotate the API endpoints with what version they were added or removed in.  Again, you'll get an error about having changed a blessed version and you'll need to follow the steps above to fix it.
 3. You merge with an upstream that adds new versions.
 
+### CI and shallow clones
+
+Versioned APIs require access to Git history: the tool loads blessed versions from the merge-base between `HEAD` and `main`, which requires that history to be available. Shallow clones (e.g., `git clone --depth 1`) typically lack the necessary history.
+
+For GitHub Actions, use a full clone:
+
+```yaml
+- uses: actions/checkout@v6
+  with:
+    fetch-depth: 0
+```
+
 ### Merging with upstream changes to versioned APIs
 
 When you merge with commits that added one or more versions to the same API that you also changed locally:
@@ -347,6 +384,82 @@ That should be it!  Now, when iterating on the API, you'll need to follow the pr
 
 In principle, this process could be reversed to convert an API from versioned to lockstep, but this almost certainly has runtime implications that would need to be considered.
 
+### Git ref storage details
+
+#### What changes on disk
+
+With Git ref storage enabled, the directory structure changes from:
+
+```
+openapi/sled-agent/
+├── sled-agent-1.0.0-2da304.json
+├── sled-agent-2.0.0-a3e161.json
+├── sled-agent-3.0.0-f44f77.json
+└── sled-agent-latest.json -> sled-agent-3.0.0-f44f77.json
+```
+
+To:
+
+```
+openapi/sled-agent/
+├── sled-agent-1.0.0-2da304.json.gitref
+├── sled-agent-2.0.0-a3e161.json.gitref
+├── sled-agent-3.0.0-f44f77.json
+└── sled-agent-latest.json -> sled-agent-3.0.0-f44f77.json
+```
+
+The latest version remains a full JSON file, and the `-latest.json` symlink continues to work. Older blessed versions become `.gitref` files.
+
+#### Git ref file format
+
+A `.gitref` file contains a single line:
+
+```
+99c3f3ef97f80d1401c54ce0c625af125d4faef3:openapi/sled-agent/sled-agent-2.0.0-a3e161.json
+```
+
+The format is `<commit-hash>:<path>`, where the commit hash is when that version was introduced.
+
+#### Reading Git ref file contents
+
+To view the contents of a Git ref file:
+
+```sh
+git show $(cat sled-agent-2.0.0-a3e161.json.gitref)
+```
+
+For Jujutsu:
+
+```sh
+IFS=: read -r commit path < sled-agent-2.0.0-a3e161.json.gitref
+jj file show -r "$commit" "root:$path"
+```
+
+#### When versions are converted
+
+The API manager automatically converts versions between JSON and Git ref formats. A version is stored as a Git ref when all of the following are true:
+
+- Git ref storage is enabled for the API.
+- The version is blessed (present in the upstream branch).
+- The version is not the latest.
+- The version was not introduced in the same commit as the latest version.
+
+When you add a new version locally, the previous latest version is converted to a Git ref. If you remove that new version, the conversion is reversed.
+
+#### Git ref merge conflicts
+
+With Git ref storage, Git detects new API versions as renames of the previous version. If parallel branches both add new versions, Git produces a rename-rename conflict.
+
+To resolve, run `cargo openapi generate` (or your equivalent alias). The tool regenerates the correct files from your resolved `api_versions!` macro.
+
+If you use Jujutsu, the `-latest.json` symlink becomes a regular file during conflicts. The API manager detects this and corrects it when you run `generate`.
+
+#### Progenitor and client generation
+
+Progenitor-generated clients should continue to reference the `-latest.json` symlink, which always points to a real JSON file. No changes are needed for typical client generation.
+
+If you need to generate a client for an older version stored as a Git ref, you will currently need to disable Git ref storage for that API.
+
 ## Contributing
 
 Bugfixes and other minor fixes are welcome! Before working on a major feature, please [open an issue](https://github.com/oxidecomputer/dropshot-api-manager/issues/new) to discuss it.