diff --git a/cmd/bd/config.go b/cmd/bd/config.go index d1d3712a21..366556a215 100644 --- a/cmd/bd/config.go +++ b/cmd/bd/config.go @@ -5,7 +5,6 @@ import ( "os" "os/exec" "path/filepath" - "regexp" "sort" "strings" @@ -13,12 +12,10 @@ import ( "github.com/spf13/viper" "github.com/steveyegge/beads/cmd/bd/doctor" "github.com/steveyegge/beads/internal/config" + "github.com/steveyegge/beads/internal/remotecache" "github.com/steveyegge/beads/internal/types" ) -// gitSSHRemotePattern matches standard git SSH remote URLs (user@host:path) -var gitSSHRemotePattern = regexp.MustCompile(`^[a-zA-Z0-9._-]+@[a-zA-Z0-9][a-zA-Z0-9._-]*:.+$`) - var configCmd = &cobra.Command{ Use: "config", GroupID: "setup", @@ -503,27 +500,10 @@ func validateSyncConfig(repoPath string) []string { return issues } -// isValidRemoteURL validates remote URL formats for sync configuration +// isValidRemoteURL validates remote URL formats for sync configuration. +// Delegates to remotecache.IsRemoteURL for consistent URL classification. func isValidRemoteURL(url string) bool { - // Valid URL schemes for beads remotes - validSchemes := []string{ - "dolthub://", - "gs://", - "s3://", - "file://", - "https://", - "http://", - "ssh://", - } - - for _, scheme := range validSchemes { - if strings.HasPrefix(url, scheme) { - return true - } - } - - // Also allow standard git remote patterns (user@host:path) - return gitSSHRemotePattern.MatchString(url) + return remotecache.IsRemoteURL(url) } // findBeadsRepoRoot walks up from the given path to find the repo root (containing .beads) diff --git a/cmd/bd/create.go b/cmd/bd/create.go index 4b46f1a759..ea816c569f 100644 --- a/cmd/bd/create.go +++ b/cmd/bd/create.go @@ -14,6 +14,7 @@ import ( "github.com/steveyegge/beads/internal/config" "github.com/steveyegge/beads/internal/debug" "github.com/steveyegge/beads/internal/hooks" + "github.com/steveyegge/beads/internal/remotecache" "github.com/steveyegge/beads/internal/routing" "github.com/steveyegge/beads/internal/storage" "github.com/steveyegge/beads/internal/storage/dolt" @@ -423,21 +424,38 @@ var createCmd = &cobra.Command{ // Switch to target repo for multi-repo support (bd-6x6g) // When routing to a different repo, we use direct storage access var targetStore storage.DoltStorage + var remoteCache *remotecache.Cache // non-nil when routing to a remote URL if repoPath != "." { - targetBeadsDir := routing.ExpandPath(repoPath) - debug.Logf("DEBUG: Routing to target repo: %s\n", targetBeadsDir) + if remotecache.IsRemoteURL(repoPath) { + // Remote URL: pull into cache, open store, push explicitly after create + var err error + remoteCache, err = remotecache.DefaultCache() + if err != nil { + FatalError("failed to initialize remote cache: %v", err) + } + if _, err := remoteCache.Ensure(rootCtx, repoPath); err != nil { + FatalError("failed to sync remote %s: %v", repoPath, err) + } + targetStore, err = remoteCache.OpenStore(rootCtx, repoPath, newDoltStoreFromConfig) + if err != nil { + FatalError("failed to open remote store: %v", err) + } + } else { + targetBeadsDir := routing.ExpandPath(repoPath) + debug.Logf("DEBUG: Routing to target repo: %s\n", targetBeadsDir) - // Ensure target beads directory exists with prefix inheritance - if err := ensureBeadsDirForPath(rootCtx, targetBeadsDir, store); err != nil { - FatalError("failed to initialize target repo: %v", err) - } + // Ensure target beads directory exists with prefix inheritance + if err := ensureBeadsDirForPath(rootCtx, targetBeadsDir, store); err != nil { + FatalError("failed to initialize target repo: %v", err) + } - // Open new store for target repo using factory to respect backend config - targetBeadsDirPath := filepath.Join(targetBeadsDir, ".beads") - var err error - targetStore, err = newDoltStoreFromConfig(rootCtx, targetBeadsDirPath) - if err != nil { - FatalError("failed to open target store: %v", err) + // Open new store for target repo using factory to respect backend config + targetBeadsDirPath := filepath.Join(targetBeadsDir, ".beads") + var err error + targetStore, err = newDoltStoreFromConfig(rootCtx, targetBeadsDirPath) + if err != nil { + FatalError("failed to open target store: %v", err) + } } // Close the original store before replacing it (it won't be used anymore) @@ -742,6 +760,15 @@ var createCmd = &cobra.Command{ } } + // Push to remote if this was a remote-routed create. + // Done explicitly (not via defer) because FatalError calls os.Exit, + // which skips deferred functions. + if remoteCache != nil { + if pushErr := remoteCache.Push(rootCtx, repoPath); pushErr != nil { + FatalError("failed to push to %s: %v\nThe issue was created locally but not synced to the remote.", repoPath, pushErr) + } + } + // Run create hook if hookRunner != nil { hookRunner.Run(hooks.EventCreate, issue) diff --git a/cmd/bd/repo.go b/cmd/bd/repo.go index 33a4f9f370..57ee5ab244 100644 --- a/cmd/bd/repo.go +++ b/cmd/bd/repo.go @@ -9,6 +9,7 @@ import ( "github.com/spf13/cobra" "github.com/steveyegge/beads/internal/config" + "github.com/steveyegge/beads/internal/remotecache" "github.com/steveyegge/beads/internal/storage" "github.com/steveyegge/beads/internal/types" ) @@ -52,19 +53,23 @@ shared across all clones of this repository.`, RunE: func(cmd *cobra.Command, args []string) error { repoPath := args[0] - // Expand ~ to home directory for validation and display - expandedPath := repoPath - if len(repoPath) > 0 && repoPath[0] == '~' { - home, err := os.UserHomeDir() - if err == nil { - expandedPath = filepath.Join(home, repoPath[1:]) + if remotecache.IsRemoteURL(repoPath) { + // Remote URL: skip local .beads directory validation + fmt.Fprintf(os.Stderr, "Adding remote repository: %s\n", repoPath) + } else { + // Local path: validate .beads directory exists + expandedPath := repoPath + if len(repoPath) > 0 && repoPath[0] == '~' { + home, err := os.UserHomeDir() + if err == nil { + expandedPath = filepath.Join(home, repoPath[1:]) + } } - } - // Validate the repo path exists and has .beads - beadsDir := filepath.Join(expandedPath, ".beads") - if _, err := os.Stat(beadsDir); os.IsNotExist(err) { - return fmt.Errorf("no .beads directory found at %s - is this a beads repository?", expandedPath) + beadsDir := filepath.Join(expandedPath, ".beads") + if _, err := os.Stat(beadsDir); os.IsNotExist(err) { + return fmt.Errorf("no .beads directory found at %s - is this a beads repository?", expandedPath) + } } // Find config.yaml @@ -137,6 +142,13 @@ that came from the removed repository.`, return fmt.Errorf("failed to remove repository: %w", err) } + // Evict remote cache if applicable + if remotecache.IsRemoteURL(repoPath) { + if cache, err := remotecache.DefaultCache(); err == nil { + _ = cache.Evict(repoPath) + } + } + // Embedded mode: flush Dolt commit before output. if isEmbeddedDolt && store != nil { if _, err := store.CommitPending(ctx, actor); err != nil { @@ -244,7 +256,50 @@ Also triggers Dolt push/pull if a remote is configured.`, // Hydrate issues from each additional repository for _, repoPath := range repos.Additional { - // Expand tilde + // Remote URL: pull into cache, read issues from SQL store + if remotecache.IsRemoteURL(repoPath) { + cache, err := remotecache.DefaultCache() + if err != nil { + fmt.Fprintf(os.Stderr, "Warning: failed to init cache for %s: %v\n", repoPath, err) + continue + } + if _, err = cache.Ensure(ctx, repoPath); err != nil { + fmt.Fprintf(os.Stderr, "Warning: failed to sync remote %s: %v\n", repoPath, err) + continue + } + remoteStore, err := cache.OpenStore(ctx, repoPath, newDoltStoreFromConfig) + if err != nil { + fmt.Fprintf(os.Stderr, "Warning: failed to open remote store %s: %v\n", repoPath, err) + continue + } + + issues, err := remoteStore.SearchIssues(ctx, "", types.IssueFilter{}) + _ = remoteStore.Close() // close eagerly — defer in a loop would leak connections + if err != nil { + fmt.Fprintf(os.Stderr, "Warning: failed to read issues from %s: %v\n", repoPath, err) + continue + } + + for _, issue := range issues { + issue.SourceRepo = repoPath + } + if len(issues) > 0 { + if importErr := store.CreateIssuesWithFullOptions(ctx, issues, "repo-sync", storage.BatchCreateOptions{ + OrphanHandling: storage.OrphanAllow, + SkipPrefixValidation: true, + }); importErr != nil { + fmt.Fprintf(os.Stderr, "Warning: failed to import from %s: %v\n", repoPath, importErr) + continue + } + totalImported += len(issues) + if verbose { + fmt.Fprintf(os.Stderr, "Imported %d issue(s) from remote %s\n", len(issues), repoPath) + } + } + continue + } + + // Local path: expand tilde expandedPath := repoPath if len(repoPath) > 0 && repoPath[0] == '~' { home, err := os.UserHomeDir() diff --git a/internal/remotecache/cache.go b/internal/remotecache/cache.go new file mode 100644 index 0000000000..11ee1fdf92 --- /dev/null +++ b/internal/remotecache/cache.go @@ -0,0 +1,281 @@ +package remotecache + +import ( + "context" + "encoding/json" + "fmt" + "os" + "os/exec" + "path/filepath" + "time" + + "github.com/steveyegge/beads/internal/configfile" + "github.com/steveyegge/beads/internal/debug" + "github.com/steveyegge/beads/internal/lockfile" + "github.com/steveyegge/beads/internal/storage" +) + +// staleLockAge is the maximum age of a lock file before it's considered stale. +const staleLockAge = 5 * time.Minute + +// StoreOpener is a function that opens a DoltStorage from a beads directory. +// This is injected by the cmd layer to abstract over build-tag-specific +// store construction (embedded vs server). +type StoreOpener func(ctx context.Context, beadsDir string) (storage.DoltStorage, error) + +// Cache manages local clones of remote Dolt databases. +// Each remote URL maps to a directory under Dir named by CacheKey(url). +type Cache struct { + Dir string // e.g., ~/.cache/beads/remotes + FreshFor time.Duration // skip pull if last pull was within this duration; 0 means always pull +} + +// CacheMeta stores metadata about a cached remote clone. +type CacheMeta struct { + RemoteURL string `json:"remote_url"` + LastPull int64 `json:"last_pull_ns"` + LastPush int64 `json:"last_push_ns"` +} + +// defaultFreshFor is the default TTL for cached clones. Ensure() skips +// pulling when the last pull was within this duration. +const defaultFreshFor = 30 * time.Second + +// DefaultCache returns a Cache using the XDG-conventional cache directory. +func DefaultCache() (*Cache, error) { + cacheDir, err := os.UserCacheDir() + if err != nil { + return nil, fmt.Errorf("failed to determine cache directory: %w", err) + } + dir := filepath.Join(cacheDir, "beads", "remotes") + return &Cache{Dir: dir, FreshFor: defaultFreshFor}, nil +} + +// entryDir returns the cache entry directory for a remote URL. +func (c *Cache) entryDir(remoteURL string) string { + return filepath.Join(c.Dir, CacheKey(remoteURL)) +} + +// cloneTarget returns the dolt database directory within a cache entry. +// dolt clone creates /.dolt/ directly, so the target is named +// after the database (default "beads") to match the embedded driver layout. +func (c *Cache) cloneTarget(remoteURL string) string { + return filepath.Join(c.entryDir(remoteURL), configfile.DefaultDoltDatabase) +} + +// metaPath returns the path to the metadata file for a cache entry. +func (c *Cache) metaPath(remoteURL string) string { + return filepath.Join(c.entryDir(remoteURL), ".meta.json") +} + +// lockPath returns the path to the lock file for a cache entry. +func (c *Cache) lockPath(remoteURL string) string { + return filepath.Join(c.entryDir(remoteURL), ".lock") +} + +// Ensure clones the remote if not cached (cold start), or pulls if already +// cached (warm start). Returns the cache entry directory path. +// +// Auth credentials are inherited from environment variables: +// DOLT_REMOTE_USER, DOLT_REMOTE_PASSWORD, or DoltHub credentials +// configured via `dolt creds`. +func (c *Cache) Ensure(ctx context.Context, remoteURL string) (string, error) { + if _, err := exec.LookPath("dolt"); err != nil { + return "", fmt.Errorf("dolt CLI not found (required for remote cache): %w", err) + } + + entry := c.entryDir(remoteURL) + if err := os.MkdirAll(entry, 0o750); err != nil { + return "", fmt.Errorf("failed to create cache entry dir: %w", err) + } + + // Acquire exclusive lock for clone/pull + lock, err := c.acquireLock(ctx, remoteURL) + if err != nil { + return "", fmt.Errorf("failed to acquire cache lock: %w", err) + } + defer c.releaseLock(lock) + + target := c.cloneTarget(remoteURL) + if c.doltExists(target) { + // Warm start: skip pull if the cache is still fresh + if c.FreshFor > 0 { + meta := c.readMeta(remoteURL) + age := time.Since(time.Unix(0, meta.LastPull)) + if age < c.FreshFor { + debug.Logf("remotecache: skipping pull for %s (%.1fs old, fresh for %.0fs)\n", + remoteURL, age.Seconds(), c.FreshFor.Seconds()) + return entry, nil + } + } + if err := c.doltPull(ctx, target); err != nil { + return "", fmt.Errorf("dolt pull failed for %s: %w", remoteURL, err) + } + } else { + // Cold start: clone + if err := c.doltClone(ctx, remoteURL, target); err != nil { + return "", fmt.Errorf("dolt clone failed for %s: %w", remoteURL, err) + } + } + + // Write metadata + meta := CacheMeta{ + RemoteURL: remoteURL, + LastPull: time.Now().UnixNano(), + } + c.writeMeta(remoteURL, &meta) + + return entry, nil +} + +// Push pushes local commits in the cached clone back to the remote. +func (c *Cache) Push(ctx context.Context, remoteURL string) error { + target := c.cloneTarget(remoteURL) + if !c.doltExists(target) { + return fmt.Errorf("no cached clone for %s", remoteURL) + } + + lock, err := c.acquireLock(ctx, remoteURL) + if err != nil { + return fmt.Errorf("failed to acquire cache lock: %w", err) + } + defer c.releaseLock(lock) + + cmd := exec.CommandContext(ctx, "dolt", "push", "origin", "main") + cmd.Dir = target + if output, err := cmd.CombinedOutput(); err != nil { + return fmt.Errorf("dolt push failed: %w\nOutput: %s", err, output) + } + + // Update push timestamp + meta := c.readMeta(remoteURL) + meta.LastPush = time.Now().UnixNano() + c.writeMeta(remoteURL, meta) + + return nil +} + +// OpenStore opens a DoltStorage from the cached clone using the provided +// StoreOpener. The cache entry directory is used as the beads directory. +// The caller is responsible for calling Close() on the returned store. +// +// Note: OpenStore does not acquire a cache lock. The caller must ensure +// no concurrent Ensure() or Push() is running against the same remoteURL, +// as those modify the underlying dolt database. This is safe for single- +// process CLI use but not for concurrent multi-process access. +func (c *Cache) OpenStore(ctx context.Context, remoteURL string, opener StoreOpener) (storage.DoltStorage, error) { + entry := c.entryDir(remoteURL) + if !c.doltExists(c.cloneTarget(remoteURL)) { + return nil, fmt.Errorf("no cached clone for %s — run Ensure first", remoteURL) + } + return opener(ctx, entry) +} + +// Evict removes a cached remote clone entirely. +func (c *Cache) Evict(remoteURL string) error { + entry := c.entryDir(remoteURL) + return os.RemoveAll(entry) +} + +// doltExists checks if a dolt database exists at the given path. +func (c *Cache) doltExists(dbPath string) bool { + doltDir := filepath.Join(dbPath, ".dolt") + info, err := os.Stat(doltDir) + return err == nil && info.IsDir() +} + +// doltClone clones a remote into the target directory. +func (c *Cache) doltClone(ctx context.Context, remoteURL, target string) error { + cmd := exec.CommandContext(ctx, "dolt", "clone", remoteURL, target) + if output, err := cmd.CombinedOutput(); err != nil { + return fmt.Errorf("%w\nOutput: %s", err, output) + } + return nil +} + +// doltPull pulls from origin in the given database directory. +func (c *Cache) doltPull(ctx context.Context, dbDir string) error { + cmd := exec.CommandContext(ctx, "dolt", "pull", "origin", "main") + cmd.Dir = dbDir + if output, err := cmd.CombinedOutput(); err != nil { + return fmt.Errorf("%w\nOutput: %s", err, output) + } + return nil +} + +// acquireLock acquires an exclusive file lock for a cache entry. +func (c *Cache) acquireLock(ctx context.Context, remoteURL string) (*os.File, error) { + lp := c.lockPath(remoteURL) + + // Clean up stale locks + if info, err := os.Stat(lp); err == nil { + if time.Since(info.ModTime()) > staleLockAge { + _ = os.Remove(lp) + } + } + + // #nosec G304 - controlled path + f, err := os.OpenFile(lp, os.O_CREATE|os.O_RDWR, 0o600) + if err != nil { + return nil, err + } + + // Poll with timeout + deadline := time.Now().Add(2 * time.Minute) + for { + err := lockfile.FlockExclusiveNonBlocking(f) + if err == nil { + return f, nil + } + if !lockfile.IsLocked(err) { + _ = f.Close() + return nil, err + } + if time.Now().After(deadline) { + _ = f.Close() + return nil, fmt.Errorf("timeout waiting for cache lock on %s", remoteURL) + } + select { + case <-ctx.Done(): + _ = f.Close() + return nil, fmt.Errorf("interrupted waiting for cache lock on %s: %w", remoteURL, ctx.Err()) + case <-time.After(100 * time.Millisecond): + } + } +} + +// releaseLock releases a cache entry file lock. +// The lock file is intentionally NOT removed: deleting it after unlock creates +// a TOCTOU race where another process's newly-acquired lock gets deleted. +// Stale lock files are cleaned up by acquireLock's age check instead. +func (c *Cache) releaseLock(f *os.File) { + if f != nil { + _ = lockfile.FlockUnlock(f) + _ = f.Close() + } +} + +// readMeta reads the cache metadata for a remote URL. +func (c *Cache) readMeta(remoteURL string) *CacheMeta { + data, err := os.ReadFile(c.metaPath(remoteURL)) + if err != nil { + return &CacheMeta{RemoteURL: remoteURL} + } + var meta CacheMeta + if err := json.Unmarshal(data, &meta); err != nil { + return &CacheMeta{RemoteURL: remoteURL} + } + return &meta +} + +// writeMeta writes cache metadata for a remote URL. +func (c *Cache) writeMeta(remoteURL string, meta *CacheMeta) { + data, err := json.MarshalIndent(meta, "", " ") + if err != nil { + debug.Logf("remotecache: failed to marshal meta for %s: %v\n", remoteURL, err) + return + } + if err := os.WriteFile(c.metaPath(remoteURL), data, 0o600); err != nil { + debug.Logf("remotecache: failed to write meta for %s: %v\n", remoteURL, err) + } +} diff --git a/internal/remotecache/cache_test.go b/internal/remotecache/cache_test.go new file mode 100644 index 0000000000..0d3eab43cf --- /dev/null +++ b/internal/remotecache/cache_test.go @@ -0,0 +1,287 @@ +package remotecache + +import ( + "context" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + "time" +) + +// skipIfNoDolt skips the test if the dolt CLI is not installed. +func skipIfNoDolt(t *testing.T) { + t.Helper() + if _, err := exec.LookPath("dolt"); err != nil { + t.Skip("dolt CLI not found, skipping integration test") + } +} + +// initDoltRemote creates a file:// dolt remote by initializing a dolt repo, +// adding a file:// remote, and pushing to it. Returns the file:// URL that +// can be used with dolt clone. +func initDoltRemote(t *testing.T, dir string) string { + t.Helper() + + // Create the "source" repo that we'll push from + srcDir := filepath.Join(dir, "src") + if err := os.MkdirAll(srcDir, 0o750); err != nil { + t.Fatal(err) + } + + // dolt init + cmd := exec.Command("dolt", "init", "--name", "test", "--email", "test@test.com") + cmd.Dir = srcDir + if out, err := cmd.CombinedOutput(); err != nil { + t.Fatalf("dolt init failed: %v\n%s", err, out) + } + + // Create a table so there's data to clone + cmd = exec.Command("dolt", "sql", "-q", "CREATE TABLE test_table (id INT PRIMARY KEY, name VARCHAR(100))") + cmd.Dir = srcDir + if out, err := cmd.CombinedOutput(); err != nil { + t.Fatalf("create table failed: %v\n%s", err, out) + } + + cmd = exec.Command("dolt", "add", ".") + cmd.Dir = srcDir + if out, err := cmd.CombinedOutput(); err != nil { + t.Fatalf("dolt add failed: %v\n%s", err, out) + } + + cmd = exec.Command("dolt", "commit", "-m", "init") + cmd.Dir = srcDir + if out, err := cmd.CombinedOutput(); err != nil { + t.Fatalf("dolt commit failed: %v\n%s", err, out) + } + + // Create the remote directory and add it as a file:// remote + remoteDir := filepath.Join(dir, "remote-storage") + if err := os.MkdirAll(remoteDir, 0o750); err != nil { + t.Fatal(err) + } + remoteURL := "file://" + remoteDir + + cmd = exec.Command("dolt", "remote", "add", "origin", remoteURL) + cmd.Dir = srcDir + if out, err := cmd.CombinedOutput(); err != nil { + t.Fatalf("dolt remote add failed: %v\n%s", err, out) + } + + // Push to create the remote storage + cmd = exec.Command("dolt", "push", "origin", "main") + cmd.Dir = srcDir + if out, err := cmd.CombinedOutput(); err != nil { + t.Fatalf("dolt push failed: %v\n%s", err, out) + } + + return remoteURL +} + +func TestEnsureColdStart(t *testing.T) { + skipIfNoDolt(t) + ctx := context.Background() + + tmpDir := t.TempDir() + remoteURL := initDoltRemote(t, filepath.Join(tmpDir, "remote")) + + cache := &Cache{Dir: filepath.Join(tmpDir, "cache")} + entryDir, err := cache.Ensure(ctx, remoteURL) + if err != nil { + t.Fatalf("Ensure (cold) failed: %v", err) + } + + // Verify the clone exists + target := cache.cloneTarget(remoteURL) + if !cache.doltExists(target) { + t.Errorf("expected .dolt directory at %s", target) + } + + // Verify metadata was written + meta := cache.readMeta(remoteURL) + if meta.RemoteURL != remoteURL { + t.Errorf("meta.RemoteURL = %q, want %q", meta.RemoteURL, remoteURL) + } + if meta.LastPull == 0 { + t.Error("meta.LastPull should be set after Ensure") + } + + // Entry dir should be the parent of the clone target + if entryDir != cache.entryDir(remoteURL) { + t.Errorf("entryDir = %q, want %q", entryDir, cache.entryDir(remoteURL)) + } +} + +func TestEnsureWarmStart(t *testing.T) { + skipIfNoDolt(t) + ctx := context.Background() + + tmpDir := t.TempDir() + remoteURL := initDoltRemote(t, filepath.Join(tmpDir, "remote")) + + cache := &Cache{Dir: filepath.Join(tmpDir, "cache")} + + // Cold start + if _, err := cache.Ensure(ctx, remoteURL); err != nil { + t.Fatalf("Ensure (cold) failed: %v", err) + } + + firstMeta := cache.readMeta(remoteURL) + + // Warm start (should pull, not clone) + if _, err := cache.Ensure(ctx, remoteURL); err != nil { + t.Fatalf("Ensure (warm) failed: %v", err) + } + + secondMeta := cache.readMeta(remoteURL) + if secondMeta.LastPull <= firstMeta.LastPull { + t.Error("LastPull should update on warm start") + } +} + +func TestEvict(t *testing.T) { + skipIfNoDolt(t) + ctx := context.Background() + + tmpDir := t.TempDir() + remoteURL := initDoltRemote(t, filepath.Join(tmpDir, "remote")) + + cache := &Cache{Dir: filepath.Join(tmpDir, "cache")} + if _, err := cache.Ensure(ctx, remoteURL); err != nil { + t.Fatalf("Ensure failed: %v", err) + } + + // Verify cache exists + if !cache.doltExists(cache.cloneTarget(remoteURL)) { + t.Fatal("expected cache entry to exist before eviction") + } + + // Evict + if err := cache.Evict(remoteURL); err != nil { + t.Fatalf("Evict failed: %v", err) + } + + // Verify gone + if cache.doltExists(cache.cloneTarget(remoteURL)) { + t.Error("expected cache entry to be gone after eviction") + } +} + +func TestPush(t *testing.T) { + skipIfNoDolt(t) + ctx := context.Background() + + tmpDir := t.TempDir() + remoteURL := initDoltRemote(t, filepath.Join(tmpDir, "remote")) + + cache := &Cache{Dir: filepath.Join(tmpDir, "cache")} + + // Clone the remote + if _, err := cache.Ensure(ctx, remoteURL); err != nil { + t.Fatalf("Ensure failed: %v", err) + } + + // Make a local change in the cached clone + target := cache.cloneTarget(remoteURL) + cmd := exec.Command("dolt", "sql", "-q", "INSERT INTO test_table VALUES (1, 'pushed')") + cmd.Dir = target + if out, err := cmd.CombinedOutput(); err != nil { + t.Fatalf("insert failed: %v\n%s", err, out) + } + + cmd = exec.Command("dolt", "add", ".") + cmd.Dir = target + if out, err := cmd.CombinedOutput(); err != nil { + t.Fatalf("dolt add failed: %v\n%s", err, out) + } + + cmd = exec.Command("dolt", "commit", "-m", "add row") + cmd.Dir = target + if out, err := cmd.CombinedOutput(); err != nil { + t.Fatalf("dolt commit failed: %v\n%s", err, out) + } + + // Push back to remote + if err := cache.Push(ctx, remoteURL); err != nil { + t.Fatalf("Push failed: %v", err) + } + + // Verify push timestamp was recorded + meta := cache.readMeta(remoteURL) + if meta.LastPush == 0 { + t.Error("meta.LastPush should be set after Push") + } + + // Verify the data made it to the remote by cloning into a fresh dir + verifyDir := filepath.Join(tmpDir, "verify") + cmd = exec.Command("dolt", "clone", remoteURL, verifyDir) + if out, err := cmd.CombinedOutput(); err != nil { + t.Fatalf("verification clone failed: %v\n%s", err, out) + } + + cmd = exec.Command("dolt", "sql", "-q", "SELECT name FROM test_table WHERE id = 1", "-r", "csv") + cmd.Dir = verifyDir + out, err := cmd.Output() + if err != nil { + t.Fatalf("verification query failed: %v", err) + } + if !strings.Contains(string(out), "pushed") { + t.Errorf("expected 'pushed' in verification output, got: %s", out) + } +} + +func TestEnsureFreshFor(t *testing.T) { + skipIfNoDolt(t) + ctx := context.Background() + + tmpDir := t.TempDir() + remoteURL := initDoltRemote(t, filepath.Join(tmpDir, "remote")) + + cache := &Cache{ + Dir: filepath.Join(tmpDir, "cache"), + FreshFor: 1 * time.Hour, // very long TTL so second call skips pull + } + + // Cold start (always clones) + if _, err := cache.Ensure(ctx, remoteURL); err != nil { + t.Fatalf("Ensure (cold) failed: %v", err) + } + + firstMeta := cache.readMeta(remoteURL) + + // Second call should skip pull because of FreshFor + if _, err := cache.Ensure(ctx, remoteURL); err != nil { + t.Fatalf("Ensure (warm, fresh) failed: %v", err) + } + + secondMeta := cache.readMeta(remoteURL) + if secondMeta.LastPull != firstMeta.LastPull { + t.Error("LastPull should NOT update when cache is still fresh") + } + + // With FreshFor=0, should always pull + cache.FreshFor = 0 + if _, err := cache.Ensure(ctx, remoteURL); err != nil { + t.Fatalf("Ensure (warm, FreshFor=0) failed: %v", err) + } + + thirdMeta := cache.readMeta(remoteURL) + if thirdMeta.LastPull <= firstMeta.LastPull { + t.Error("LastPull should update when FreshFor=0") + } +} + +func TestDefaultCache(t *testing.T) { + cache, err := DefaultCache() + if err != nil { + t.Fatalf("DefaultCache failed: %v", err) + } + if cache.Dir == "" { + t.Error("cache.Dir should not be empty") + } + // Should end with beads/remotes + if filepath.Base(filepath.Dir(cache.Dir)) != "beads" || filepath.Base(cache.Dir) != "remotes" { + t.Errorf("unexpected cache dir: %s", cache.Dir) + } +} diff --git a/internal/remotecache/url.go b/internal/remotecache/url.go new file mode 100644 index 0000000000..0e90bc24a4 --- /dev/null +++ b/internal/remotecache/url.go @@ -0,0 +1,46 @@ +package remotecache + +import ( + "crypto/sha256" + "fmt" + "regexp" + "strings" +) + +// remoteSchemes lists URL scheme prefixes recognized as dolt remote URLs. +var remoteSchemes = []string{ + "dolthub://", + "gs://", + "s3://", + "file://", + "https://", + "http://", + "ssh://", + "git+ssh://", + "git+https://", +} + +// gitSSHPattern matches SCP-style git remote URLs (user@host:path). +var gitSSHPattern = regexp.MustCompile(`^[a-zA-Z0-9._-]+@[a-zA-Z0-9][a-zA-Z0-9._-]*:.+$`) + +// IsRemoteURL returns true if s looks like a dolt remote URL rather than +// a local filesystem path. Recognized schemes: dolthub://, https://, http://, +// s3://, gs://, file://, ssh://, git+ssh://, git+https://, and SCP-style +// git@host:path. +func IsRemoteURL(s string) bool { + for _, scheme := range remoteSchemes { + if strings.HasPrefix(s, scheme) { + return true + } + } + return gitSSHPattern.MatchString(s) +} + +// CacheKey returns a filesystem-safe identifier for a remote URL. +// It uses the first 16 hex characters (64 bits) of the SHA-256 hash. +// Birthday-bound collision risk is negligible for a local cache: 50% at +// ~4.3 billion entries, well beyond any realistic number of remotes. +func CacheKey(remoteURL string) string { + h := sha256.Sum256([]byte(remoteURL)) + return fmt.Sprintf("%x", h[:8]) +} diff --git a/internal/remotecache/url_test.go b/internal/remotecache/url_test.go new file mode 100644 index 0000000000..232ee45353 --- /dev/null +++ b/internal/remotecache/url_test.go @@ -0,0 +1,65 @@ +package remotecache + +import ( + "testing" +) + +func TestIsRemoteURL(t *testing.T) { + tests := []struct { + input string + want bool + }{ + // Remote URLs — should return true + {"dolthub://org/backend", true}, + {"dolthub://myorg/myrepo", true}, + {"https://doltremoteapi.dolthub.com/org/repo", true}, + {"http://localhost:50051/mydb", true}, + {"s3://my-bucket/beads", true}, + {"gs://my-bucket/beads", true}, + {"file:///tmp/dolt-remote", true}, + {"ssh://git@github.com/org/repo", true}, + {"git+ssh://git@github.com/org/repo", true}, + {"git+https://github.com/org/repo", true}, + {"git@github.com:org/repo.git", true}, + {"deploy@myserver.com:beads/data", true}, + + // Local paths — should return false + {".", false}, + {"..", false}, + {"~/beads-planning", false}, + {"/absolute/path/to/repo", false}, + {"../relative/path", false}, + {"relative/path", false}, + {"", false}, + {"/", false}, + } + + for _, tt := range tests { + t.Run(tt.input, func(t *testing.T) { + got := IsRemoteURL(tt.input) + if got != tt.want { + t.Errorf("IsRemoteURL(%q) = %v, want %v", tt.input, got, tt.want) + } + }) + } +} + +func TestCacheKey(t *testing.T) { + // Deterministic + k1 := CacheKey("dolthub://org/backend") + k2 := CacheKey("dolthub://org/backend") + if k1 != k2 { + t.Errorf("CacheKey not deterministic: %q != %q", k1, k2) + } + + // Different URLs produce different keys + k3 := CacheKey("dolthub://org/frontend") + if k1 == k3 { + t.Errorf("CacheKey collision: %q and %q both produce %q", "dolthub://org/backend", "dolthub://org/frontend", k1) + } + + // Length is 16 hex chars + if len(k1) != 16 { + t.Errorf("CacheKey length = %d, want 16", len(k1)) + } +}