Skip to content

Commit 541b684

Browse files
mrmaxsteelclaude
mrmaxsteel
and
claude
authored
feat: add remote URL support to multi-repo hydration (PR #2827)
* /{cmd,internal}: add remote URL support to multi-repo hydration Multi-repo hydration (`bd repo sync`) reads issues.jsonl from local filesystem paths, predating beads' dolt remote support. This adds remote URL support so hydration, `--repo`, and `bd repo add` can accept dolt remote URLs (dolthub://, https://, s3://, etc.) alongside local paths. New internal/remotecache package manages cached dolt clones at ~/.cache/beads/remotes/<hash>/ with clone/pull/push lifecycle and file-based locking for concurrent access. Closes #2826 Co-Authored-By: Claude Opus 4.6 (1M context) <[email protected]> * fix: address review issues in remote hydration PR - Fix gosec G306: tighten meta file permissions from 0644 to 0600 - Make lock polling context-aware (respects Ctrl+C / ctx cancellation) - Replace defer cache.Push with explicit call (FatalError calls os.Exit, which skips deferred functions — silent data loss on remote creates) - Log remoteStore.Close() errors as warnings instead of discarding - Quantify CacheKey birthday-bound collision risk in comment Co-Authored-By: Claude Opus 4.6 (1M context) <[email protected]> * fix(remotecache): address code review findings from #2827 - Fix lock file race: remove os.Remove in releaseLock to prevent TOCTOU race where another process's lock gets deleted; stale lock cleanup in acquireLock handles orphaned files - Add debug.Logf to writeMeta for error visibility - Rename cacheErr to idiomatic err in repo sync remote block - Escalate remote push failure to FatalError in create (silent warning meant the remote never received the issue) Co-Authored-By: Claude Opus 4.6 (1M context) <[email protected]> * fix(remotecache): defer store close, add Push test, add freshness TTL - Replace defer-in-loop with eager close in repo sync (defer inside a for loop leaks connections until function return) - Add FreshFor TTL to Cache (default 30s) — Ensure() skips pull when last pull is within the window, FreshFor=0 preserves always-pull - Add TestPush: full round-trip integration test (clone → insert → push → re-clone → verify data) - Add TestEnsureFreshFor: validates TTL skip and FreshFor=0 bypass - Add concurrency doc comment to OpenStore Co-Authored-By: Claude Opus 4.6 (1M context) <[email protected]> --------- Co-authored-by: Claude Opus 4.6 (1M context) <[email protected]>
1 parent 5297286 commit 541b684

7 files changed

Lines changed: 789 additions & 48 deletions

File tree

‎cmd/bd/config.go‎

Lines changed: 4 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -5,20 +5,17 @@ import (
55
"os"
66
"os/exec"
77
"path/filepath"
8-
"regexp"
98
"sort"
109
"strings"
1110

1211
"github.com/spf13/cobra"
1312
"github.com/spf13/viper"
1413
"github.com/steveyegge/beads/cmd/bd/doctor"
1514
"github.com/steveyegge/beads/internal/config"
15+
"github.com/steveyegge/beads/internal/remotecache"
1616
"github.com/steveyegge/beads/internal/types"
1717
)
1818

19-
// gitSSHRemotePattern matches standard git SSH remote URLs (user@host:path)
20-
var gitSSHRemotePattern = regexp.MustCompile(`^[a-zA-Z0-9._-]+@[a-zA-Z0-9][a-zA-Z0-9._-]*:.+$`)
21-
2219
var configCmd = &cobra.Command{
2320
Use: "config",
2421
GroupID: "setup",
@@ -503,27 +500,10 @@ func validateSyncConfig(repoPath string) []string {
503500
return issues
504501
}
505502

506-
// isValidRemoteURL validates remote URL formats for sync configuration
503+
// isValidRemoteURL validates remote URL formats for sync configuration.
504+
// Delegates to remotecache.IsRemoteURL for consistent URL classification.
507505
func isValidRemoteURL(url string) bool {
508-
// Valid URL schemes for beads remotes
509-
validSchemes := []string{
510-
"dolthub://",
511-
"gs://",
512-
"s3://",
513-
"file://",
514-
"https://",
515-
"http://",
516-
"ssh://",
517-
}
518-
519-
for _, scheme := range validSchemes {
520-
if strings.HasPrefix(url, scheme) {
521-
return true
522-
}
523-
}
524-
525-
// Also allow standard git remote patterns (user@host:path)
526-
return gitSSHRemotePattern.MatchString(url)
506+
return remotecache.IsRemoteURL(url)
527507
}
528508

529509
// findBeadsRepoRoot walks up from the given path to find the repo root (containing .beads)

‎cmd/bd/create.go‎

Lines changed: 39 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@ import (
1414
"github.com/steveyegge/beads/internal/config"
1515
"github.com/steveyegge/beads/internal/debug"
1616
"github.com/steveyegge/beads/internal/hooks"
17+
"github.com/steveyegge/beads/internal/remotecache"
1718
"github.com/steveyegge/beads/internal/routing"
1819
"github.com/steveyegge/beads/internal/storage"
1920
"github.com/steveyegge/beads/internal/storage/dolt"
@@ -423,21 +424,38 @@ var createCmd = &cobra.Command{
423424
// Switch to target repo for multi-repo support (bd-6x6g)
424425
// When routing to a different repo, we use direct storage access
425426
var targetStore storage.DoltStorage
427+
var remoteCache *remotecache.Cache // non-nil when routing to a remote URL
426428
if repoPath != "." {
427-
targetBeadsDir := routing.ExpandPath(repoPath)
428-
debug.Logf("DEBUG: Routing to target repo: %s\n", targetBeadsDir)
429+
if remotecache.IsRemoteURL(repoPath) {
430+
// Remote URL: pull into cache, open store, push explicitly after create
431+
var err error
432+
remoteCache, err = remotecache.DefaultCache()
433+
if err != nil {
434+
FatalError("failed to initialize remote cache: %v", err)
435+
}
436+
if _, err := remoteCache.Ensure(rootCtx, repoPath); err != nil {
437+
FatalError("failed to sync remote %s: %v", repoPath, err)
438+
}
439+
targetStore, err = remoteCache.OpenStore(rootCtx, repoPath, newDoltStoreFromConfig)
440+
if err != nil {
441+
FatalError("failed to open remote store: %v", err)
442+
}
443+
} else {
444+
targetBeadsDir := routing.ExpandPath(repoPath)
445+
debug.Logf("DEBUG: Routing to target repo: %s\n", targetBeadsDir)
429446

430-
// Ensure target beads directory exists with prefix inheritance
431-
if err := ensureBeadsDirForPath(rootCtx, targetBeadsDir, store); err != nil {
432-
FatalError("failed to initialize target repo: %v", err)
433-
}
447+
// Ensure target beads directory exists with prefix inheritance
448+
if err := ensureBeadsDirForPath(rootCtx, targetBeadsDir, store); err != nil {
449+
FatalError("failed to initialize target repo: %v", err)
450+
}
434451

435-
// Open new store for target repo using factory to respect backend config
436-
targetBeadsDirPath := filepath.Join(targetBeadsDir, ".beads")
437-
var err error
438-
targetStore, err = newDoltStoreFromConfig(rootCtx, targetBeadsDirPath)
439-
if err != nil {
440-
FatalError("failed to open target store: %v", err)
452+
// Open new store for target repo using factory to respect backend config
453+
targetBeadsDirPath := filepath.Join(targetBeadsDir, ".beads")
454+
var err error
455+
targetStore, err = newDoltStoreFromConfig(rootCtx, targetBeadsDirPath)
456+
if err != nil {
457+
FatalError("failed to open target store: %v", err)
458+
}
441459
}
442460

443461
// Close the original store before replacing it (it won't be used anymore)
@@ -742,6 +760,15 @@ var createCmd = &cobra.Command{
742760
}
743761
}
744762

763+
// Push to remote if this was a remote-routed create.
764+
// Done explicitly (not via defer) because FatalError calls os.Exit,
765+
// which skips deferred functions.
766+
if remoteCache != nil {
767+
if pushErr := remoteCache.Push(rootCtx, repoPath); pushErr != nil {
768+
FatalError("failed to push to %s: %v\nThe issue was created locally but not synced to the remote.", repoPath, pushErr)
769+
}
770+
}
771+
745772
// Run create hook
746773
if hookRunner != nil {
747774
hookRunner.Run(hooks.EventCreate, issue)

‎cmd/bd/repo.go‎

Lines changed: 67 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@ import (
99

1010
"github.com/spf13/cobra"
1111
"github.com/steveyegge/beads/internal/config"
12+
"github.com/steveyegge/beads/internal/remotecache"
1213
"github.com/steveyegge/beads/internal/storage"
1314
"github.com/steveyegge/beads/internal/types"
1415
)
@@ -52,19 +53,23 @@ shared across all clones of this repository.`,
5253
RunE: func(cmd *cobra.Command, args []string) error {
5354
repoPath := args[0]
5455

55-
// Expand ~ to home directory for validation and display
56-
expandedPath := repoPath
57-
if len(repoPath) > 0 && repoPath[0] == '~' {
58-
home, err := os.UserHomeDir()
59-
if err == nil {
60-
expandedPath = filepath.Join(home, repoPath[1:])
56+
if remotecache.IsRemoteURL(repoPath) {
57+
// Remote URL: skip local .beads directory validation
58+
fmt.Fprintf(os.Stderr, "Adding remote repository: %s\n", repoPath)
59+
} else {
60+
// Local path: validate .beads directory exists
61+
expandedPath := repoPath
62+
if len(repoPath) > 0 && repoPath[0] == '~' {
63+
home, err := os.UserHomeDir()
64+
if err == nil {
65+
expandedPath = filepath.Join(home, repoPath[1:])
66+
}
6167
}
62-
}
6368

64-
// Validate the repo path exists and has .beads
65-
beadsDir := filepath.Join(expandedPath, ".beads")
66-
if _, err := os.Stat(beadsDir); os.IsNotExist(err) {
67-
return fmt.Errorf("no .beads directory found at %s - is this a beads repository?", expandedPath)
69+
beadsDir := filepath.Join(expandedPath, ".beads")
70+
if _, err := os.Stat(beadsDir); os.IsNotExist(err) {
71+
return fmt.Errorf("no .beads directory found at %s - is this a beads repository?", expandedPath)
72+
}
6873
}
6974

7075
// Find config.yaml
@@ -137,6 +142,13 @@ that came from the removed repository.`,
137142
return fmt.Errorf("failed to remove repository: %w", err)
138143
}
139144

145+
// Evict remote cache if applicable
146+
if remotecache.IsRemoteURL(repoPath) {
147+
if cache, err := remotecache.DefaultCache(); err == nil {
148+
_ = cache.Evict(repoPath)
149+
}
150+
}
151+
140152
// Embedded mode: flush Dolt commit before output.
141153
if isEmbeddedDolt && store != nil {
142154
if _, err := store.CommitPending(ctx, actor); err != nil {
@@ -244,7 +256,50 @@ Also triggers Dolt push/pull if a remote is configured.`,
244256

245257
// Hydrate issues from each additional repository
246258
for _, repoPath := range repos.Additional {
247-
// Expand tilde
259+
// Remote URL: pull into cache, read issues from SQL store
260+
if remotecache.IsRemoteURL(repoPath) {
261+
cache, err := remotecache.DefaultCache()
262+
if err != nil {
263+
fmt.Fprintf(os.Stderr, "Warning: failed to init cache for %s: %v\n", repoPath, err)
264+
continue
265+
}
266+
if _, err = cache.Ensure(ctx, repoPath); err != nil {
267+
fmt.Fprintf(os.Stderr, "Warning: failed to sync remote %s: %v\n", repoPath, err)
268+
continue
269+
}
270+
remoteStore, err := cache.OpenStore(ctx, repoPath, newDoltStoreFromConfig)
271+
if err != nil {
272+
fmt.Fprintf(os.Stderr, "Warning: failed to open remote store %s: %v\n", repoPath, err)
273+
continue
274+
}
275+
276+
issues, err := remoteStore.SearchIssues(ctx, "", types.IssueFilter{})
277+
_ = remoteStore.Close() // close eagerly — defer in a loop would leak connections
278+
if err != nil {
279+
fmt.Fprintf(os.Stderr, "Warning: failed to read issues from %s: %v\n", repoPath, err)
280+
continue
281+
}
282+
283+
for _, issue := range issues {
284+
issue.SourceRepo = repoPath
285+
}
286+
if len(issues) > 0 {
287+
if importErr := store.CreateIssuesWithFullOptions(ctx, issues, "repo-sync", storage.BatchCreateOptions{
288+
OrphanHandling: storage.OrphanAllow,
289+
SkipPrefixValidation: true,
290+
}); importErr != nil {
291+
fmt.Fprintf(os.Stderr, "Warning: failed to import from %s: %v\n", repoPath, importErr)
292+
continue
293+
}
294+
totalImported += len(issues)
295+
if verbose {
296+
fmt.Fprintf(os.Stderr, "Imported %d issue(s) from remote %s\n", len(issues), repoPath)
297+
}
298+
}
299+
continue
300+
}
301+
302+
// Local path: expand tilde
248303
expandedPath := repoPath
249304
if len(repoPath) > 0 && repoPath[0] == '~' {
250305
home, err := os.UserHomeDir()

0 commit comments

Comments
 (0)