diff --git a/.gitignore b/.gitignore index 7ebd618..0880516 100644 --- a/.gitignore +++ b/.gitignore @@ -14,7 +14,12 @@ dist/ bin/ # Build artifacts +# Both patterns are anchored on purpose. A bare "codamigo" would also match the +# cmd/codamigo source directory and hide every file added under it. /codamigo +# `go build ./cmd/codamigo/` run from inside that directory, rather than the +# `make build` target, leaves the binary here. +/cmd/codamigo/codamigo /build/ # Test artifacts diff --git a/README.md b/README.md index 2a0abbc..94e6ff0 100644 --- a/README.md +++ b/README.md @@ -262,7 +262,11 @@ codamigo download-model --model all-MiniLM-L6-v2 | `--force` | Re-download files that are already present and verified | | `--hf-token` | HuggingFace token; only needed for gated or private models | -Re-running is a no-op: files already present with a matching size and hash are skipped. A file that fails verification is deleted before the error is reported, so a retry starts clean. +Files already present with a matching size and hash are skipped, and a file that fails verification is deleted before the error is reported, so a retry starts clean. + +Re-running always re-resolves the revision upstream, which is how you move a model to a newer one. For a built-in model that changes nothing — its revision is fixed. For a bare repository id it means a re-run after upstream's `main` has moved downloads the new revision in full — a fresh 133 MB at `bge-small-en-v1.5`'s size — into a new snapshot. The old snapshot is reported with its path and size and left alone; delete it by hand if you want the space. + +Each download also records the resolved commit in `codamigo-pin.json` in the model directory, which is what lets later loads run offline, and prints the model's real `embedding_dimensions` for you to paste into the config. --- @@ -295,7 +299,9 @@ codamigo reset && codamigo index # the store records its vector width, so switc Both are pinned to a fixed revision with per-file checksums, which is what makes `download-model` reproducible rather than just a corruption check. -Any HuggingFace sentence-transformers repository id also works (`embedding_model: some-org/some-model`), but it is **not** checksum-verified, tracks `main`, and requires you to set `embedding_dimensions` yourself. Not every architecture loads: `nomic-ai/nomic-embed-text-v1.5`, for instance, is rejected by the current go-huggingface loader. +Any HuggingFace sentence-transformers repository id also works (`embedding_model: some-org/some-model`), but it is **not** checksum-verified and requires you to set `embedding_dimensions` yourself — the value `download-model` prints. It resolves `main` at download time and is then pinned to that exact commit, so it never re-downloads on its own; only another `download-model` moves it. Not every architecture loads: `nomic-ai/nomic-embed-text-v1.5`, for instance, is rejected by the current go-huggingface loader. + +Loading a model never touches the network: the revision it would otherwise have to look up comes from `codamigo-pin.json` instead. Model directories that predate that file still load offline, deriving the revision from go-huggingface's own cached repository info with no re-download. `codamigo doctor` reports which of the two a model is using. ### Compute backends and speed diff --git a/cmd/codamigo/doctor_cmd.go b/cmd/codamigo/doctor_cmd.go index c893917..f3892d4 100644 --- a/cmd/codamigo/doctor_cmd.go +++ b/cmd/codamigo/doctor_cmd.go @@ -230,18 +230,31 @@ func reportProvider(cfg *config.Config, emb embedder.Embedder) { fmt.Printf("[FAIL] Model: %v\n", err) return } - fmt.Printf(" Model: %s (%s)\n", model.DisplayName(), model.RepoID) - if !model.Pinned() { - fmt.Printf("[WARN] %s is not a built-in model, so its files are not checksum-verified\n", model.DisplayName()) - } - root, err := localModelsRoot(cfg) if err != nil { fmt.Printf("[FAIL] Models directory: %v\n", err) return } + fmt.Printf(" Model: %s (%s)\n", model.DisplayName(), model.RepoID) + if !model.Pinned() { + fmt.Printf("[WARN] %s is not a built-in model, so its files are not checksum-verified\n", model.DisplayName()) + } + if dir, err := localembed.ModelDir(root, model); err == nil { - if missing, err := localembed.MissingFiles(dir, model); err == nil && len(missing) == 0 { + switch pin, err := localembed.ReadPin(dir); { + case err == nil: + fmt.Printf(" Revision: %s (pinned %s)\n", pin.CommitHash, pin.ResolvedFrom) + case errors.Is(err, localembed.ErrNoPin): + fmt.Printf(" Revision: no pin file; derived from the cached repository info\n") + fmt.Printf(" Run 'codamigo download-model' to record one.\n") + default: + fmt.Printf("[WARN] Revision: %v\n", err) + } + + resolved, _, err := localembed.ResolvePin(dir, model) + if err != nil { + warnIfModelMissing(root, model) + } else if missing, err := localembed.MissingFiles(dir, resolved); err == nil && len(missing) == 0 { fmt.Printf("[OK] Model files present: %s\n", dir) } else { warnIfModelMissing(root, model) diff --git a/cmd/codamigo/download_cmd.go b/cmd/codamigo/download_cmd.go index 1669138..7bf310a 100644 --- a/cmd/codamigo/download_cmd.go +++ b/cmd/codamigo/download_cmd.go @@ -105,6 +105,14 @@ func downloadModelCmd() *cli.Command { } fmt.Printf(" Model directory: %s\n", res.ModelDir) + if stale, err := localembed.SupersededSnapshots(modelDir, model, res.CommitHash); err == nil && len(stale) > 0 { + fmt.Printf("\n[WARN] %d superseded snapshot(s) remain in this model directory.\n", len(stale)) + fmt.Print(" They are no longer used. Remove them by hand if you want the space:\n") + for _, s := range stale { + fmt.Printf(" %s (%s)\n", s.Path, humanBytes(s.Bytes)) + } + } + if cmd.Bool("xla") || cmd.String("cuda") != "" { if err := installPlugins(cmd); err != nil { // Not fatal: the pure-Go backend still works, just slowly. @@ -114,7 +122,7 @@ func downloadModelCmd() *cli.Command { } } - printLocalConfigSnippet(model) + printLocalConfigSnippet(model, res.Dimensions) return nil }, } @@ -149,15 +157,19 @@ func installPlugins(cmd *cli.Command) error { // printLocalConfigSnippet tells the user exactly what to add to switch over, // including embedding_dimensions, which must match the model or the store will -// refuse to open. -func printLocalConfigSnippet(model localembed.Model) { +// refuse to open. dimensions comes from the model's own config.json, so the +// unpinned case no longer leaves the user to work it out. +func printLocalConfigSnippet(model localembed.Model, dimensions int) { name := model.DisplayName() fmt.Printf("\nTo use it, add this to ~/.codamigo/global_settings.yml:\n\n") fmt.Printf(" embedding_provider: local\n") fmt.Printf(" embedding_model: %s\n", name) - if model.Dimensions > 0 { + switch { + case model.Dimensions > 0: fmt.Printf(" embedding_dimensions: %d\n", model.Dimensions) - } else { + case dimensions > 0: + fmt.Printf(" embedding_dimensions: %d\n", dimensions) + default: fmt.Printf(" embedding_dimensions: