feat(localembed): pin local embedding models to a resolved revision, load offline

.gitignore

@@ -14,7 +14,12 @@ dist/
 bin/
 
 # Build artifacts
+# Both patterns are anchored on purpose. A bare "codamigo" would also match the
+# cmd/codamigo source directory and hide every file added under it.
 /codamigo
+# `go build ./cmd/codamigo/` run from inside that directory, rather than the
+# `make build` target, leaves the binary here.
+/cmd/codamigo/codamigo
 /build/
 
 # Test artifacts

README.md

@@ -262,7 +262,11 @@ codamigo download-model --model all-MiniLM-L6-v2
 | `--force` | Re-download files that are already present and verified |
 | `--hf-token` | HuggingFace token; only needed for gated or private models |
 
-Re-running is a no-op: files already present with a matching size and hash are skipped. A file that fails verification is deleted before the error is reported, so a retry starts clean.
+Files already present with a matching size and hash are skipped, and a file that fails verification is deleted before the error is reported, so a retry starts clean.
+
+Re-running always re-resolves the revision upstream, which is how you move a model to a newer one. For a built-in model that changes nothing — its revision is fixed. For a bare repository id it means a re-run after upstream's `main` has moved downloads the new revision in full — a fresh 133 MB at `bge-small-en-v1.5`'s size — into a new snapshot. The old snapshot is reported with its path and size and left alone; delete it by hand if you want the space.
+
+Each download also records the resolved commit in `codamigo-pin.json` in the model directory, which is what lets later loads run offline, and prints the model's real `embedding_dimensions` for you to paste into the config.
 
 ---
 
@@ -295,7 +299,9 @@ codamigo reset && codamigo index  # the store records its vector width, so switc
 
 Both are pinned to a fixed revision with per-file checksums, which is what makes `download-model` reproducible rather than just a corruption check.
 
-Any HuggingFace sentence-transformers repository id also works (`embedding_model: some-org/some-model`), but it is **not** checksum-verified, tracks `main`, and requires you to set `embedding_dimensions` yourself. Not every architecture loads: `nomic-ai/nomic-embed-text-v1.5`, for instance, is rejected by the current go-huggingface loader.
+Any HuggingFace sentence-transformers repository id also works (`embedding_model: some-org/some-model`), but it is **not** checksum-verified and requires you to set `embedding_dimensions` yourself — the value `download-model` prints. It resolves `main` at download time and is then pinned to that exact commit, so it never re-downloads on its own; only another `download-model` moves it. Not every architecture loads: `nomic-ai/nomic-embed-text-v1.5`, for instance, is rejected by the current go-huggingface loader.
+
+Loading a model never touches the network: the revision it would otherwise have to look up comes from `codamigo-pin.json` instead. Model directories that predate that file still load offline, deriving the revision from go-huggingface's own cached repository info with no re-download. `codamigo doctor` reports which of the two a model is using.
 
 ### Compute backends and speed
 

cmd/codamigo/doctor_cmd.go

@@ -230,18 +230,31 @@ func reportProvider(cfg *config.Config, emb embedder.Embedder) {
 		fmt.Printf("[FAIL] Model: %v\n", err)
 		return
 	}
-	fmt.Printf("       Model: %s (%s)\n", model.DisplayName(), model.RepoID)
-	if !model.Pinned() {
-		fmt.Printf("[WARN] %s is not a built-in model, so its files are not checksum-verified\n", model.DisplayName())
-	}
-
 	root, err := localModelsRoot(cfg)
 	if err != nil {
 		fmt.Printf("[FAIL] Models directory: %v\n", err)
 		return
 	}
+	fmt.Printf("       Model: %s (%s)\n", model.DisplayName(), model.RepoID)
+	if !model.Pinned() {
+		fmt.Printf("[WARN] %s is not a built-in model, so its files are not checksum-verified\n", model.DisplayName())
+	}
+
 	if dir, err := localembed.ModelDir(root, model); err == nil {
-		if missing, err := localembed.MissingFiles(dir, model); err == nil && len(missing) == 0 {
+		switch pin, err := localembed.ReadPin(dir); {
+		case err == nil:
+			fmt.Printf("       Revision: %s (pinned %s)\n", pin.CommitHash, pin.ResolvedFrom)
+		case errors.Is(err, localembed.ErrNoPin):
+			fmt.Printf("       Revision: no pin file; derived from the cached repository info\n")
+			fmt.Printf("       Run 'codamigo download-model' to record one.\n")
+		default:
+			fmt.Printf("[WARN] Revision: %v\n", err)
+		}
+
+		resolved, _, err := localembed.ResolvePin(dir, model)
+		if err != nil {
+			warnIfModelMissing(root, model)
+		} else if missing, err := localembed.MissingFiles(dir, resolved); err == nil && len(missing) == 0 {
 			fmt.Printf("[OK]  Model files present: %s\n", dir)
 		} else {
 			warnIfModelMissing(root, model)

cmd/codamigo/download_cmd.go

@@ -105,6 +105,14 @@ func downloadModelCmd() *cli.Command {
 			}
 			fmt.Printf("      Model directory: %s\n", res.ModelDir)
 
+			if stale, err := localembed.SupersededSnapshots(modelDir, model, res.CommitHash); err == nil && len(stale) > 0 {
+				fmt.Printf("\n[WARN] %d superseded snapshot(s) remain in this model directory.\n", len(stale))
+				fmt.Print("       They are no longer used. Remove them by hand if you want the space:\n")
+				for _, s := range stale {
+					fmt.Printf("         %s (%s)\n", s.Path, humanBytes(s.Bytes))
+				}
+			}
+
 			if cmd.Bool("xla") || cmd.String("cuda") != "" {
 				if err := installPlugins(cmd); err != nil {
 					// Not fatal: the pure-Go backend still works, just slowly.
@@ -114,7 +122,7 @@ func downloadModelCmd() *cli.Command {
 				}
 			}
 
-			printLocalConfigSnippet(model)
+			printLocalConfigSnippet(model, res.Dimensions)
 			return nil
 		},
 	}
@@ -149,15 +157,19 @@ func installPlugins(cmd *cli.Command) error {
 
 // printLocalConfigSnippet tells the user exactly what to add to switch over,
 // including embedding_dimensions, which must match the model or the store will
-// refuse to open.
-func printLocalConfigSnippet(model localembed.Model) {
+// refuse to open. dimensions comes from the model's own config.json, so the
+// unpinned case no longer leaves the user to work it out.
+func printLocalConfigSnippet(model localembed.Model, dimensions int) {
 	name := model.DisplayName()
 	fmt.Printf("\nTo use it, add this to ~/.codamigo/global_settings.yml:\n\n")
 	fmt.Printf("  embedding_provider: local\n")
 	fmt.Printf("  embedding_model: %s\n", name)
-	if model.Dimensions > 0 {
+	switch {
+	case model.Dimensions > 0:
 		fmt.Printf("  embedding_dimensions: %d\n", model.Dimensions)
-	} else {
+	case dimensions > 0:
+		fmt.Printf("  embedding_dimensions: %d\n", dimensions)
+	default:
 		fmt.Printf("  embedding_dimensions: <the model's hidden size — codamigo will tell you>\n")
 	}
 	fmt.Printf("\nThe index stores its vector width, so switch providers with:\n")
@@ -188,7 +200,12 @@ func isModelDownloaded(root string, model localembed.Model) (bool, error) {
 	if err != nil {
 		return false, err
 	}
-	return localembed.IsDownloaded(dir, model)
+	resolved, _, err := localembed.ResolvePin(dir, model)
+	if err != nil {
+		// No resolvable revision means nothing usable is on disk.
+		return false, nil
+	}
+	return localembed.IsDownloaded(dir, resolved)
 }
 
 // warnIfModelMissing prints an actionable hint when the local provider is
@@ -198,7 +215,13 @@ func warnIfModelMissing(root string, model localembed.Model) {
 	if err != nil {
 		return
 	}
-	missing, err := localembed.MissingFiles(dir, model)
+	resolved, _, err := localembed.ResolvePin(dir, model)
+	if err != nil {
+		fmt.Printf("[FAIL] Model %s is not downloaded (%v)\n", model.DisplayName(), err)
+		fmt.Printf("       Run: codamigo download-model --model %s\n", model.DisplayName())
+		return
+	}
+	missing, err := localembed.MissingFiles(dir, resolved)
 	if err != nil {
 		if !errors.Is(err, os.ErrNotExist) {
 			fmt.Printf("[WARN] Could not inspect %s: %v\n", dir, err)

localembed/cache.go

@@ -71,41 +71,17 @@ func flatRepoDir(repoID string) string {
 	return "models--" + strings.ReplaceAll(repoID, "/", "--")
 }
 
-// SnapshotDir returns the directory holding m's actual files inside modelDir.
+// SnapshotDir returns the directory holding m's files inside modelDir.
 //
-// A pinned model names its revision, so the path is exact. An unpinned model
-// tracks "main", whose commit hash is only known after the download, so the
-// single directory under snapshots/ is used instead. Returns
-// [ErrModelNotDownloaded] when there is nothing there yet.
+// m.Revision must already be a concrete commit hash — [ResolvePin] is what
+// guarantees that. Naming the path needs no filesystem access; whether the
+// files are actually present is [MissingFiles]' question.
 func SnapshotDir(modelDir string, m Model) (string, error) {
-	snapshots := filepath.Join(modelDir, flatRepoDir(m.RepoID), "snapshots")
-	if m.Revision != "" && m.Revision != "main" {
-		return filepath.Join(snapshots, m.Revision), nil
-	}
-	entries, err := os.ReadDir(snapshots)
-	if errors.Is(err, fs.ErrNotExist) {
-		return "", fmt.Errorf("%w: %s has no snapshot under %s", ErrModelNotDownloaded, m.DisplayName(), snapshots)
-	}
-	if err != nil {
-		return "", fmt.Errorf("reading %s: %w", snapshots, err)
-	}
-	var dirs []string
-	for _, e := range entries {
-		if e.IsDir() {
-			dirs = append(dirs, e.Name())
-		}
-	}
-	switch len(dirs) {
-	case 0:
-		return "", fmt.Errorf("%w: %s has no snapshot under %s", ErrModelNotDownloaded, m.DisplayName(), snapshots)
-	case 1:
-		return filepath.Join(snapshots, dirs[0]), nil
-	default:
-		// Several revisions of an unpinned model. Refuse rather than pick, since
-		// silently loading the wrong weights is worse than an actionable error.
-		return "", fmt.Errorf("%s has %d revisions under %s; pin embedding_model to a registry model "+
-			"or remove the directory and re-download", m.DisplayName(), len(dirs), snapshots)
+	if !isCommitHash(m.Revision) {
+		return "", fmt.Errorf("%s has unresolved revision %q; call ResolvePin first",
+			m.DisplayName(), m.Revision)
 	}
+	return filepath.Join(modelDir, flatRepoDir(m.RepoID), "snapshots", m.Revision), nil
 }
 
 // MissingFiles returns the manifest paths that are absent from modelDir or that
@@ -116,13 +92,6 @@ func SnapshotDir(modelDir string, m Model) (string, error) {
 // used rather than os.Lstat: a dangling link must count as missing.
 func MissingFiles(modelDir string, m Model) ([]string, error) {
 	snapshot, err := SnapshotDir(modelDir, m)
-	if errors.Is(err, ErrModelNotDownloaded) {
-		paths := make([]string, len(m.Files))
-		for i, f := range m.Files {
-			paths[i] = f.Path
-		}
-		return paths, nil
-	}
 	if err != nil {
 		return nil, err
 	}
@@ -151,3 +120,111 @@ func IsDownloaded(modelDir string, m Model) (bool, error) {
 	}
 	return len(missing) == 0, nil
 }
+
+// SnapshotInfo describes one snapshot directory on disk.
+type SnapshotInfo struct {
+	Path  string
+	Bytes int64
+}
+
+// SupersededSnapshots returns the snapshot directories under modelDir other
+// than keep, with their sizes.
+//
+// Moving an unpinned model to a newer upstream revision leaves the previous
+// snapshot behind — often gigabytes of it. Reporting them is deliberate:
+// deleting a working set of weights on the user's behalf is not this package's
+// decision. A missing snapshots directory is not an error, just an empty
+// result.
+func SupersededSnapshots(modelDir string, m Model, keep string) ([]SnapshotInfo, error) {
+	snapshots := filepath.Join(modelDir, flatRepoDir(m.RepoID), "snapshots")
+	entries, err := os.ReadDir(snapshots)
+	if errors.Is(err, fs.ErrNotExist) {
+		return nil, nil
+	}
+	if err != nil {
+		return nil, fmt.Errorf("reading %s: %w", snapshots, err)
+	}
+	// Resolved once: every blob the kept snapshot still references. Any
+	// superseded snapshot entry pointing at one of these frees nothing when
+	// deleted, since removing a snapshot only removes its symlinks.
+	keepBlobs := blobTargets(filepath.Join(snapshots, keep))
+	var out []SnapshotInfo
+	for _, e := range entries {
+		if !e.IsDir() || e.Name() == keep {
+			continue
+		}
+		path := filepath.Join(snapshots, e.Name())
+		out = append(out, SnapshotInfo{Path: path, Bytes: dirSize(path, keepBlobs)})
+	}
+	return out, nil
+}
+
+// blobTargets resolves every symlink under snapshotDir to its target's
+// absolute, cleaned path. go-huggingface deduplicates identical file content
+// by etag: every entry in a snapshot directory is normally a symlink into a
+// shared blobs/ directory, so this is how [dirSize] tells which blobs the kept
+// snapshot still needs. snapshotDir need not exist (e.g. an already-deleted
+// keep); errors are swallowed and simply yield an empty set, since this is
+// only ever used to decide what a human-facing size figure should say.
+func blobTargets(snapshotDir string) map[string]struct{} {
+	targets := make(map[string]struct{})
+	_ = filepath.WalkDir(snapshotDir, func(path string, d fs.DirEntry, err error) error {
+		if err != nil || d.IsDir() {
+			return nil //nolint:nilerr // an unreadable entry just contributes nothing
+		}
+		if resolved, ok := resolveSymlink(path); ok {
+			targets[resolved] = struct{}{}
+		}
+		return nil
+	})
+	return targets
+}
+
+// resolveSymlink reports the absolute, cleaned target of path if path is a
+// symlink, resolving a relative target against path's own directory the way
+// go-huggingface's snapshot entries do. ok is false for anything else,
+// including a plain file or any read error, so callers can just skip it.
+func resolveSymlink(path string) (string, bool) {
+	info, err := os.Lstat(path)
+	if err != nil || info.Mode()&os.ModeSymlink == 0 {
+		return "", false
+	}
+	target, err := os.Readlink(path)
+	if err != nil {
+		return "", false
+	}
+	if !filepath.IsAbs(target) {
+		target = filepath.Join(filepath.Dir(path), target)
+	}
+	return filepath.Clean(target), true
+}
+
+// dirSize reports the bytes actually reclaimable if the superseded snapshot
+// directory root were deleted.
+//
+// Snapshot entries are normally symlinks into a shared blobs/ directory,
+// deduplicated by etag: two snapshots of the same model share a blob for
+// every file whose content did not change between revisions. Deleting a
+// snapshot directory only removes its symlinks, never the blobs — so an entry
+// whose target is also referenced by the kept snapshot (keepBlobs) frees
+// nothing and must not be counted. A plain (non-symlinked) file is always
+// counted, since there is nothing else referencing it. Errors are swallowed:
+// this figure is reported to a human, never acted on.
+func dirSize(root string, keepBlobs map[string]struct{}) int64 {
+	var total int64
+	_ = filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error {
+		if err != nil || d.IsDir() {
+			return nil //nolint:nilerr // an unreadable entry just does not count
+		}
+		if resolved, ok := resolveSymlink(path); ok {
+			if _, shared := keepBlobs[resolved]; shared {
+				return nil
+			}
+		}
+		if info, err := os.Stat(path); err == nil {
+			total += info.Size()
+		}
+		return nil
+	})
+	return total
+}