-
Notifications
You must be signed in to change notification settings - Fork 9
Expand file tree
/
Copy pathcli.go
More file actions
2311 lines (2004 loc) · 71.6 KB
/
Copy pathcli.go
File metadata and controls
2311 lines (2004 loc) · 71.6 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
package main
import (
"context"
"encoding/json"
"fmt"
"io"
"os"
"os/exec"
"path/filepath"
"strings"
"syscall"
"time"
"github.com/charmbracelet/lipgloss"
"github.com/charmbracelet/log"
"github.com/mattn/go-runewidth"
"github.com/modelcontextprotocol/go-sdk/mcp"
"github.com/spf13/cobra"
"github.com/unbound-force/dewey/v3/chunker"
"github.com/unbound-force/dewey/v3/client"
"github.com/unbound-force/dewey/v3/curate"
"github.com/unbound-force/dewey/v3/embed"
"github.com/unbound-force/dewey/v3/ignore"
"github.com/unbound-force/dewey/v3/llm"
"github.com/unbound-force/dewey/v3/sanitize"
"github.com/unbound-force/dewey/v3/source"
"github.com/unbound-force/dewey/v3/store"
"github.com/unbound-force/dewey/v3/tools"
"github.com/unbound-force/dewey/v3/types"
"github.com/unbound-force/dewey/v3/vault"
)
// newJournalCmd creates the `dewey journal` subcommand.
// Appends a block to today's (or a specified date's) journal page.
func newJournalCmd() *cobra.Command {
var date string
cmd := &cobra.Command{
Use: "journal [flags] TEXT",
Short: "Append block to today's journal",
Long: `Appends a block to a Logseq journal page.
Prints the created block UUID on success.
Content can be provided as arguments or piped via stdin:
dewey journal "my note"
echo "my note" | dewey journal`,
RunE: func(cmd *cobra.Command, args []string) error {
c := client.New("", "")
content := readContentFromArgs(args)
if content == "" {
return fmt.Errorf("no content provided")
}
var t time.Time
if date != "" {
var err error
t, err = time.Parse("2006-01-02", date)
if err != nil {
return fmt.Errorf("invalid date %q (use YYYY-MM-DD)", date)
}
} else {
t = time.Now()
}
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
pageName := findJournalPage(ctx, c, t)
if pageName == "" {
// No existing page found — use ordinal format (most common Logseq default).
pageName = ordinalDate(t)
}
block, err := c.AppendBlockInPage(ctx, pageName, content)
if err != nil {
return fmt.Errorf("journal: %w", err)
}
if block != nil {
fmt.Println(block.UUID)
}
return nil
},
}
cmd.Flags().StringVarP(&date, "date", "d", "", "Journal date (YYYY-MM-DD). Default: today")
return cmd
}
// newAddCmd creates the `dewey add` subcommand.
// Appends a block to a named page.
func newAddCmd() *cobra.Command {
var page string
cmd := &cobra.Command{
Use: "add [flags] TEXT",
Short: "Append block to a page",
Long: `Appends a block to a Logseq page (creates page if needed).
Prints the created block UUID on success.
Content can be provided as arguments or piped via stdin:
dewey add -p "My Page" "content here"
echo "content" | dewey add --page "My Page"`,
RunE: func(cmd *cobra.Command, args []string) error {
if page == "" {
return fmt.Errorf("--page is required")
}
c := client.New("", "")
content := readContentFromArgs(args)
if content == "" {
return fmt.Errorf("no content provided")
}
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
block, err := c.AppendBlockInPage(ctx, page, content)
if err != nil {
return fmt.Errorf("add: %w", err)
}
if block != nil {
fmt.Println(block.UUID)
}
return nil
},
}
cmd.Flags().StringVarP(&page, "page", "p", "", "Page name (required)")
return cmd
}
// newSearchCmd creates the `dewey search` subcommand.
// Performs full-text search using the vault backend (same data path as dewey serve).
func newSearchCmd() *cobra.Command {
var limit int
var vaultPath string
cmd := &cobra.Command{
Use: "search [flags] QUERY",
Short: "Full-text search across the graph",
Long: "Full-text search across all blocks in the knowledge graph.",
RunE: func(cmd *cobra.Command, args []string) error {
query := strings.Join(args, " ")
if query == "" {
return fmt.Errorf("query is required")
}
// Resolve vault path using the shared resolver.
vp, err := resolveVaultPath(vaultPath)
if err != nil {
return err
}
// Create vault client and load local .md files.
var opts []vault.Option
vc := vault.New(vp, opts...)
if err := vc.Load(); err != nil {
return fmt.Errorf("search: load vault: %w", err)
}
// If persistent store exists, load external-source pages from graph.db.
deweyDir := filepath.Join(vp, deweyWorkspaceDir)
if _, err := os.Stat(deweyDir); err == nil {
dbPath := filepath.Join(deweyDir, "graph.db")
s, err := store.New(dbPath)
if err == nil {
defer func() { _ = s.Close() }()
vs := vault.NewVaultStore(s, vp, "disk-local")
if n, err := vs.LoadExternalPages(vc); err != nil {
logger.Warn("failed to load external pages", "err", err)
} else if n > 0 {
logger.Info("loaded external pages", "count", n)
}
}
}
// Build backlinks and search index.
vc.BuildBacklinks()
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
hits, err := vc.FullTextSearch(ctx, query, limit)
if err != nil {
return fmt.Errorf("search: %w", err)
}
if len(hits) == 0 {
return fmt.Errorf("no results for %q", query)
}
for _, hit := range hits {
fmt.Printf("%s | %s\n", hit.PageName, strings.ReplaceAll(hit.Content, "\n", " "))
}
return nil
},
}
cmd.Flags().IntVar(&limit, "limit", 10, "Max results")
cmd.Flags().StringVar(&vaultPath, "vault", "", "Path to Obsidian vault")
return cmd
}
// newInitCmd creates the `dewey init` subcommand.
// Initializes a .uf/dewey/ directory with default configuration.
// Idempotent — running twice does not error (per CLI contract).
func newInitCmd() *cobra.Command {
var vaultPath string
cmd := &cobra.Command{
Use: "init",
Short: "Initialize Dewey configuration",
Long: "Create .uf/dewey/ directory with default config.yaml and sources.yaml.",
RunE: func(cmd *cobra.Command, args []string) error {
if vaultPath == "" {
var err error
vaultPath, err = os.Getwd()
if err != nil {
return fmt.Errorf("get working directory: %w", err)
}
}
deweyDir := filepath.Join(vaultPath, deweyWorkspaceDir)
// Check if already initialized. If so, skip config/sources
// creation but still run slash command scaffolding below.
alreadyInitialized := false
if _, err := os.Stat(deweyDir); err == nil {
alreadyInitialized = true
logger.Info("already initialized", "path", deweyDir)
}
if !alreadyInitialized {
// Create .uf/dewey/ directory (MkdirAll creates .uf/ parent too — D3).
if err := os.MkdirAll(deweyDir, 0o755); err != nil {
return fmt.Errorf("create .uf/dewey/ directory: %w", err)
}
// Write default config.yaml.
configPath := filepath.Join(deweyDir, "config.yaml")
configContent := `# Dewey configuration
# See: https://github.com/unbound-force/dewey
embedding:
model: granite-embedding:30m # Override with DEWEY_EMBEDDING_MODEL env var
# endpoint: http://localhost:11434 # Uncomment to override OLLAMA_HOST
# max_chunk_chars: 12288 # Override with DEWEY_CHUNK_MAX_CHARS env var
`
if err := os.WriteFile(configPath, []byte(configContent), 0o644); err != nil {
return fmt.Errorf("write config.yaml: %w", err)
}
// Write default sources.yaml.
sourcesPath := filepath.Join(deweyDir, "sources.yaml")
sourcesContent := `# Dewey content sources
# Each source provides documents for the knowledge graph index.
sources:
- id: disk-local
type: disk
name: local
config:
path: "."
# ignore: [pattern1, pattern2] # additional patterns beyond .gitignore
# recursive: true # set false to index only top-level files
`
if err := os.WriteFile(sourcesPath, []byte(sourcesContent), 0o644); err != nil {
return fmt.Errorf("write sources.yaml: %w", err)
}
// Write default knowledge-stores.yaml (T008, 015-curated-knowledge-stores).
// Scaffolds a commented-out example store. Follows the same idempotency
// pattern as sources.yaml — don't overwrite if file exists.
ksPath := filepath.Join(deweyDir, "knowledge-stores.yaml")
ksContent := `# Knowledge store configuration
# Each store curates knowledge from indexed sources.
# Uncomment and customize the example below.
# stores:
# - name: team-decisions
# sources: [disk-local]
# # path: .uf/dewey/knowledge/team-decisions # default
# # curate_on_index: false # default
# # curation_interval: 10m # default
`
if err := os.WriteFile(ksPath, []byte(ksContent), 0o644); err != nil {
return fmt.Errorf("write knowledge-stores.yaml: %w", err)
}
// Append granular .uf/dewey/ runtime artifact patterns to .gitignore.
// Only runtime artifacts (db, log, lock) are ignored — sources.yaml
// and config.yaml remain trackable for team sharing.
gitignorePath := filepath.Join(vaultPath, ".gitignore")
if _, err := os.Stat(gitignorePath); err == nil {
content, err := os.ReadFile(gitignorePath)
if err == nil {
text := string(content)
switch {
case strings.Contains(text, ".uf/dewey/graph.db"):
// Current granular patterns already present — skip.
case strings.Contains(text, ".dewey/graph.db"):
// Old granular patterns — inform user to update.
logger.Info("old .dewey/ gitignore patterns found — update to .uf/dewey/ patterns")
case strings.Contains(text, ".dewey/"):
// Legacy blanket pattern — don't modify, inform user.
logger.Info("existing .dewey/ gitignore pattern found — update to .uf/dewey/ patterns")
default:
// No dewey patterns — append granular patterns.
f, err := os.OpenFile(gitignorePath, os.O_APPEND|os.O_WRONLY, 0o644)
if err == nil {
defer func() { _ = f.Close() }()
if len(content) > 0 && content[len(content)-1] != '\n' {
_, _ = f.WriteString("\n")
}
_, _ = f.WriteString(".uf/dewey/graph.db\n")
_, _ = f.WriteString(".uf/dewey/graph.db-shm\n")
_, _ = f.WriteString(".uf/dewey/graph.db-wal\n")
_, _ = f.WriteString(".uf/dewey/dewey.log\n")
_, _ = f.WriteString(".uf/dewey/dewey.lock\n")
}
}
}
}
} // end if !alreadyInitialized
// Scaffold Dewey-specific slash commands into .opencode/command/
// if the .opencode/ directory exists (composability — only scaffold
// when OpenCode is present). Idempotent — skip files that already
// exist to avoid overwriting user customizations.
opencodeCmdDir := filepath.Join(vaultPath, ".opencode", "command")
if _, err := os.Stat(filepath.Join(vaultPath, ".opencode")); err == nil {
if err := os.MkdirAll(opencodeCmdDir, 0o755); err == nil {
for name, content := range deweySlashCommands {
cmdPath := filepath.Join(opencodeCmdDir, name)
if _, err := os.Stat(cmdPath); err != nil {
// File doesn't exist — scaffold it.
if writeErr := os.WriteFile(cmdPath, []byte(content), 0o644); writeErr == nil {
logger.Info("scaffolded slash command", "path", cmdPath)
}
}
}
}
}
if !alreadyInitialized {
logger.Info("initialized", "path", deweyDir)
logger.Info("run 'dewey index' to build the initial index")
}
return nil
},
}
cmd.Flags().StringVar(&vaultPath, "vault", "", "Path to the vault root (default: current directory)")
return cmd
}
// sourceStatus holds per-source metadata for status reporting.
type sourceStatus struct {
ID string `json:"id"`
Type string `json:"type"`
Status string `json:"status"`
PageCount int `json:"pageCount"`
LastFetched string `json:"lastFetched,omitempty"`
Error string `json:"error,omitempty"`
}
// statusData holds all data needed to render the status output.
// Separates data collection from formatting to reduce cyclomatic complexity.
type statusData struct {
PageCount int
BlockCount int
EmbeddingCount int
EmbeddingModel string
EmbeddingAvailable bool
Sources []sourceStatus
IndexPath string
}
// embeddingCoverage computes the percentage of blocks with embeddings.
func (d statusData) embeddingCoverage() float64 {
if d.BlockCount > 0 {
return float64(d.EmbeddingCount) / float64(d.BlockCount) * 100
}
return 0
}
// newStatusCmd creates the `dewey status` subcommand.
// Reports index health: page count, block count, source info.
// Supports --json flag for structured output.
func newStatusCmd() *cobra.Command {
var jsonOutput bool
var vaultPath string
cmd := &cobra.Command{
Use: "status",
Short: "Report index status",
Long: "Show Dewey index health: page count, block count, source info, and index path.",
SilenceUsage: true,
RunE: func(cmd *cobra.Command, args []string) error {
vp, err := resolveVaultPathOrCwd(vaultPath)
if err != nil {
return err
}
deweyDir := filepath.Join(vp, deweyWorkspaceDir)
if _, err := os.Stat(deweyDir); os.IsNotExist(err) {
return fmt.Errorf("not initialized. Run 'dewey init' first")
}
data, err := queryStoreStatus(deweyDir)
if err != nil {
return err
}
w := cmd.OutOrStdout()
if jsonOutput {
return formatStatusJSON(data, w)
}
return formatStatusText(data, w)
},
}
cmd.Flags().BoolVar(&jsonOutput, "json", false, "Output as JSON")
cmd.Flags().StringVar(&vaultPath, "vault", "", "Path to vault (default: OBSIDIAN_VAULT_PATH or current directory)")
return cmd
}
// readEmbeddingModel extracts the embedding model name from config.yaml
// using simple line parsing to avoid a YAML dependency for status display.
func readEmbeddingModel(deweyDir string) string {
configPath := filepath.Join(deweyDir, "config.yaml")
configData, err := os.ReadFile(configPath)
if err != nil {
return ""
}
for _, line := range strings.Split(string(configData), "\n") {
line = strings.TrimSpace(line)
if strings.HasPrefix(line, "model:") {
return strings.TrimSpace(strings.TrimPrefix(line, "model:"))
}
}
return ""
}
// queryStoreStatus opens the store at deweyDir, queries all counts and source
// records, and returns a populated statusData. The store is closed before
// returning. Returns a zero-value statusData (with IndexPath set) if the
// database does not yet exist.
func queryStoreStatus(deweyDir string) (statusData, error) {
data := statusData{
IndexPath: deweyDir,
EmbeddingModel: readEmbeddingModel(deweyDir),
}
dbPath := filepath.Join(deweyDir, "graph.db")
if _, err := os.Stat(dbPath); err != nil {
// Database does not exist yet — return zero counts.
return data, nil
}
s, err := store.New(dbPath)
if err != nil {
return data, fmt.Errorf("open store: %w", err)
}
defer func() { _ = s.Close() }()
pages, err := s.ListPages()
if err != nil {
return data, fmt.Errorf("list pages: %w", err)
}
data.PageCount = len(pages)
if bc, err := s.CountBlocks(); err == nil {
data.BlockCount = bc
}
if ec, err := s.CountEmbeddings(); err == nil {
data.EmbeddingCount = ec
}
storedSources, _ := s.ListSources()
for _, src := range storedSources {
ss := sourceStatus{
ID: src.ID,
Type: src.Type,
Status: src.Status,
Error: src.ErrorMessage,
}
pc, _ := s.CountPagesBySource(src.ID)
ss.PageCount = pc
if src.LastFetchedAt > 0 {
elapsed := time.Since(time.UnixMilli(src.LastFetchedAt))
ss.LastFetched = formatDuration(elapsed)
}
data.Sources = append(data.Sources, ss)
}
return data, nil
}
// formatStatusText writes human-readable status output to w.
func formatStatusText(data statusData, w io.Writer) error {
_, _ = fmt.Fprintln(w, "Dewey Index Status")
_, _ = fmt.Fprintf(w, " Path: %s\n", data.IndexPath)
_, _ = fmt.Fprintf(w, " Pages: %d\n", data.PageCount)
_, _ = fmt.Fprintf(w, " Blocks: %d\n", data.BlockCount)
_, _ = fmt.Fprintf(w, " Embeddings: %d\n", data.EmbeddingCount)
if data.EmbeddingModel != "" {
_, _ = fmt.Fprintf(w, " Model: %s\n", data.EmbeddingModel)
}
_, _ = fmt.Fprintf(w, " Coverage: %.1f%%\n", data.embeddingCoverage())
if len(data.Sources) > 0 {
_, _ = fmt.Fprintln(w, "\nSources")
for _, src := range data.Sources {
lastFetched := "never"
if src.LastFetched != "" {
lastFetched = src.LastFetched + " ago"
}
if src.Error != "" {
_, _ = fmt.Fprintf(w, " %-15s %-8s %3d pages %s error: %s\n",
src.ID, src.Status, src.PageCount, lastFetched, src.Error)
} else {
_, _ = fmt.Fprintf(w, " %-15s %-8s %3d pages %s\n",
src.ID, src.Status, src.PageCount, lastFetched)
}
}
}
return nil
}
// formatStatusJSON writes JSON-formatted status output to w.
func formatStatusJSON(data statusData, w io.Writer) error {
status := map[string]any{
"path": data.IndexPath,
"pages": data.PageCount,
"blocks": data.BlockCount,
"embeddings": data.EmbeddingCount,
"embeddingModel": data.EmbeddingModel,
"embeddingAvailable": data.EmbeddingAvailable,
"embeddingCoverage": data.embeddingCoverage(),
"sources": data.Sources,
}
out, err := json.MarshalIndent(status, "", " ")
if err != nil {
return fmt.Errorf("marshal JSON: %w", err)
}
_, _ = fmt.Fprintln(w, string(out))
return nil
}
// --- Helpers ---
// readContentFromArgs gets content from positional args or stdin (if piped).
func readContentFromArgs(args []string) string {
if len(args) > 0 {
return strings.Join(args, " ")
}
// Only read stdin if it's piped (not a terminal).
stat, err := os.Stdin.Stat()
if err != nil {
return ""
}
if (stat.Mode() & os.ModeCharDevice) != 0 {
return "" // stdin is a terminal, not piped
}
data, err := io.ReadAll(os.Stdin)
if err != nil {
return ""
}
return strings.TrimSpace(string(data))
}
// findJournalPage tries common Logseq journal date formats to find an existing page.
func findJournalPage(ctx context.Context, c *client.Client, t time.Time) string {
names := []string{
ordinalDate(t),
t.Format("2006-01-02"),
t.Format("January 2, 2006"),
}
for _, name := range names {
page, err := c.GetPage(ctx, name)
if err == nil && page != nil {
return name
}
}
return ""
}
// ordinalDate formats a time as "Jan 29th, 2026" (common Logseq journal default).
func ordinalDate(t time.Time) string {
day := t.Day()
suffix := "th"
switch day {
case 1, 21, 31:
suffix = "st"
case 2, 22:
suffix = "nd"
case 3, 23:
suffix = "rd"
}
return fmt.Sprintf("%s %d%s, %d", t.Format("Jan"), day, suffix, t.Year())
}
// printSearchResults recursively prints matching blocks to stdout.
func printSearchResults(blocks []types.BlockEntity, query, pageName string, limit int, found *int) {
for _, b := range blocks {
if *found >= limit {
return
}
if strings.Contains(strings.ToLower(b.Content), query) {
fmt.Printf("%s | %s\n", pageName, b.Content)
*found++
}
if len(b.Children) > 0 {
printSearchResults(b.Children, query, pageName, limit, found)
}
}
}
// formatDuration formats a duration as a human-readable string (e.g., "2m", "4h", "3d").
func formatDuration(d time.Duration) string {
switch {
case d < time.Minute:
return fmt.Sprintf("%ds", int(d.Seconds()))
case d < time.Hour:
return fmt.Sprintf("%dm", int(d.Minutes()))
case d < 24*time.Hour:
return fmt.Sprintf("%dh", int(d.Hours()))
default:
return fmt.Sprintf("%dd", int(d.Hours()/24))
}
}
// --- Index command (T050) ---
// newIndexCmd creates the `dewey index` subcommand.
// Builds or updates the knowledge graph and embedding indexes.
// Per contracts/cli-commands.md.
func newIndexCmd() *cobra.Command {
var sourceName string
var force bool
var noEmbeddings bool
var vaultPath string
cmd := &cobra.Command{
Use: "index",
Short: "Build or update the knowledge graph index",
Long: `Build or update the knowledge graph and embedding indexes.
Fetches content from all configured sources and indexes it.`,
SilenceUsage: true,
RunE: func(cmd *cobra.Command, args []string) error {
vp, err := resolveVaultPathOrCwd(vaultPath)
if err != nil {
return err
}
deweyDir := filepath.Join(vp, deweyWorkspaceDir)
if _, err := os.Stat(deweyDir); os.IsNotExist(err) {
return fmt.Errorf("not initialized. Run 'dewey init' first")
}
// Load sources config.
sourcesPath := filepath.Join(deweyDir, "sources.yaml")
configs, err := source.LoadSourcesConfig(sourcesPath)
if err != nil {
return fmt.Errorf("load sources config: %w", err)
}
// Open store.
dbPath := filepath.Join(deweyDir, "graph.db")
s, err := store.New(dbPath)
if err != nil {
return fmt.Errorf("open store: %w", err)
}
defer func() { _ = s.Close() }()
// Auto-purge orphaned sources (FR-013, T017): compare configured
// source IDs against source IDs in the store. Delete pages for
// any source that no longer appears in sources.yaml.
purgeOrphanedSources(s, configs)
// Create embedder for embedding generation during indexing (R4).
// Hard error: if Ollama is unavailable and --no-embeddings is not set,
// indexing fails with an actionable error message.
embedder, err := createIndexEmbedder(noEmbeddings, deweyDir)
if err != nil {
return err
}
// Build last-fetched times from store.
lastFetchedTimes := make(map[string]time.Time)
storedSources, _ := s.ListSources()
for _, src := range storedSources {
if src.LastFetchedAt > 0 {
lastFetchedTimes[src.ID] = time.UnixMilli(src.LastFetchedAt)
}
}
// Create source manager and fetch.
cacheDir := filepath.Join(deweyDir, "cache")
mgr := source.NewManager(configs, vp, cacheDir)
result, allDocs := mgr.FetchAll(sourceName, force, lastFetchedTimes)
embCfg := embed.ReadEmbeddingConfig(deweyDir)
indexResult, indexErr := vault.IndexDocuments(s, allDocs, configs, embedder, embCfg.MaxChunkChars)
reportSourceErrors(s, result)
if indexErr != nil {
return fmt.Errorf("index failed: %w", indexErr)
}
logger.Info("index complete",
"documents", indexResult.TotalIndexed,
"errors", result.TotalErrs,
"skipped", result.TotalSkip,
)
return nil
},
}
cmd.Flags().StringVar(&sourceName, "source", "", "Index only the specified source")
cmd.Flags().BoolVar(&force, "force", false, "Re-fetch all sources, even if within their refresh interval")
cmd.Flags().BoolVar(&noEmbeddings, "no-embeddings", false, "Skip embedding generation (disables semantic search)")
cmd.Flags().StringVar(&vaultPath, "vault", "", "Path to vault (default: OBSIDIAN_VAULT_PATH or current directory)")
return cmd
}
// newReindexCmd creates the `dewey reindex` subcommand.
// Performs a clean re-index by removing the existing graph.db and all
// associated SQLite files (WAL, SHM) and lock files, then running a
// full index from scratch. This is the recommended way to rebuild the
// index after upgrading Dewey or when the index is corrupted.
func newReindexCmd() *cobra.Command {
var noEmbeddings bool
var vaultPath string
cmd := &cobra.Command{
Use: "reindex",
Short: "Delete and rebuild the index from scratch",
Long: `Remove the existing graph.db and rebuild the index from scratch.
This is equivalent to manually deleting .uf/dewey/graph.db and its WAL/SHM
files, then running 'dewey index --force'. Use this when:
- Upgrading Dewey to a version with schema or indexing changes
- The index is corrupted (UUID collisions, foreign key errors)
- You want a guaranteed clean slate
The command removes: graph.db, graph.db-wal, graph.db-shm, dewey.lock`,
SilenceUsage: true,
RunE: func(cmd *cobra.Command, args []string) error {
reindexStart := time.Now()
vp, err := resolveVaultPathOrCwd(vaultPath)
if err != nil {
return err
}
deweyDir := filepath.Join(vp, deweyWorkspaceDir)
if _, err := os.Stat(deweyDir); os.IsNotExist(err) {
return fmt.Errorf("not initialized. Run 'dewey init' first")
}
logger.Info("reindex starting", "vault", vp, "deweyDir", deweyDir, "pid", os.Getpid())
// Acquire the lock FIRST to prevent TOCTOU race conditions.
// If another dewey process holds the lock, we fail immediately
// instead of checking and then racing to remove files.
lockPath := filepath.Join(deweyDir, "dewey.lock")
lockFile, lockErr := os.OpenFile(lockPath, os.O_CREATE|os.O_RDWR, 0o644)
if lockErr == nil {
if err := syscall.Flock(int(lockFile.Fd()), syscall.LOCK_EX|syscall.LOCK_NB); err != nil {
_ = lockFile.Close()
return fmt.Errorf("database is locked by another dewey process — stop 'dewey serve' first")
}
// We hold the lock now — safe to remove files.
// Release and close before removing the lock file itself.
_ = syscall.Flock(int(lockFile.Fd()), syscall.LOCK_UN)
_ = lockFile.Close()
}
// Remove all database and lock files with per-file logging.
filesToRemove := []string{
filepath.Join(deweyDir, "graph.db"),
filepath.Join(deweyDir, "graph.db-wal"),
filepath.Join(deweyDir, "graph.db-shm"),
lockPath,
}
for _, f := range filesToRemove {
info, statErr := os.Stat(f)
if os.IsNotExist(statErr) {
logger.Debug("file not present, skipping", "file", filepath.Base(f))
continue
}
size := int64(0)
if info != nil {
size = info.Size()
}
if err := os.Remove(f); err != nil {
return fmt.Errorf("remove %s (size=%d): %w — stop 'dewey serve' first", filepath.Base(f), size, err)
}
logger.Info("removed", "file", filepath.Base(f), "size", size)
}
// Open a fresh store (creates new graph.db with schema).
dbPath := filepath.Join(deweyDir, "graph.db")
s, err := store.New(dbPath)
if err != nil {
return fmt.Errorf("open store: %w", err)
}
defer func() { _ = s.Close() }()
logger.Info("store opened", "path", dbPath)
// Load sources config.
sourcesPath := filepath.Join(deweyDir, "sources.yaml")
configs, err := source.LoadSourcesConfig(sourcesPath)
if err != nil {
return fmt.Errorf("load sources config: %w", err)
}
logger.Info("sources loaded", "count", len(configs))
for _, cfg := range configs {
logger.Debug("source config", "id", cfg.ID, "type", cfg.Type, "name", cfg.Name)
}
// Create embedder (hard error if unavailable, unless --no-embeddings).
embedder, err := createIndexEmbedder(noEmbeddings, deweyDir)
if err != nil {
return err
}
// Fetch all sources (force mode — ignore refresh intervals).
cacheDir := filepath.Join(deweyDir, "cache")
mgr := source.NewManager(configs, vp, cacheDir)
lastFetchedTimes := make(map[string]time.Time) // empty — force fetch all
result, allDocs := mgr.FetchAll("", true, lastFetchedTimes)
// Log what was fetched per source.
for srcID, docs := range allDocs {
logger.Info("source fetched for reindex", "source", srcID, "documents", len(docs))
}
embCfg := embed.ReadEmbeddingConfig(deweyDir)
indexResult, indexErr := vault.IndexDocuments(s, allDocs, configs, embedder, embCfg.MaxChunkChars)
reportSourceErrors(s, result)
if indexErr != nil {
return fmt.Errorf("reindex failed: %w", indexErr)
}
// Verify final state.
finalPages, _ := s.ListPages()
elapsed := time.Since(reindexStart)
logger.Info("reindex complete",
"documents", indexResult.TotalIndexed,
"pages_in_db", len(finalPages),
"errors", result.TotalErrs,
"elapsed", elapsed.Round(time.Millisecond),
)
return nil
},
}
cmd.Flags().BoolVar(&noEmbeddings, "no-embeddings", false, "Skip embedding generation (disables semantic search)")
cmd.Flags().StringVar(&vaultPath, "vault", "", "Path to vault (default: OBSIDIAN_VAULT_PATH or current directory)")
return cmd
}
// detectLockHolder checks if the dewey.lock file is held by another process.
// Returns a description of the holder (e.g., "PID 12345 dewey serve --vault /path")
// or empty string if unlocked. When the lock is held but no PID was written
// (pre-T011 lock files), returns a generic "lock held" message.
func detectLockHolder(lockPath string) string {
f, err := os.OpenFile(lockPath, os.O_RDWR, 0o644)
if err != nil {
return "" // file doesn't exist or can't be opened — not locked
}
defer func() { _ = f.Close() }()
// Try to acquire a non-blocking exclusive lock.
err = syscall.Flock(int(f.Fd()), syscall.LOCK_EX|syscall.LOCK_NB)
if err != nil {
// Lock is held by another process. Read PID info written by T011.
data, readErr := os.ReadFile(lockPath)
if readErr == nil {
firstLine := strings.TrimSpace(strings.SplitN(string(data), "\n", 2)[0])
if firstLine != "" {
return fmt.Sprintf("PID %s", firstLine)
}
}
// Lock held but no PID data (old-format lock file).
return "lock held (unknown process)"
}
// We got the lock — release it, no one is holding it.
_ = syscall.Flock(int(f.Fd()), syscall.LOCK_UN)
return ""
}
// createIndexEmbedder creates an Embedder for use during indexing.
// Reads provider configuration from the dewey workspace config.yaml,
// falling back to environment variables and Ollama defaults.
// When noEmbeddings is true, returns nil (no embedder). When the provider
// is Ollama, attempts auto-start. Returns nil on graceful degradation
// and a hard error only when the provider is available but the model is not.
func createIndexEmbedder(noEmbeddings bool, deweyDirs ...string) (embed.Embedder, error) {
if noEmbeddings {
logger.Info("embeddings disabled via --no-embeddings")
return nil, nil
}
// Read config from the dewey workspace directory.
// The config reader handles env var overrides and defaults.
var deweyDir string
if len(deweyDirs) > 0 {
deweyDir = deweyDirs[0]
}
cfg := embed.ReadEmbeddingConfig(deweyDir)
// For Ollama provider, ensure Ollama is running (auto-start if needed).
if cfg.Provider == "ollama" || cfg.Provider == "" {
ollamaState, err := ensureOllama(cfg.Endpoint, true, &execOllamaStarter{})
if err != nil {
logger.Warn("ollama auto-start failed, continuing without embeddings", "err", err)
}
logger.Info("ollama state", "state", ollamaState, "endpoint", cfg.Endpoint)
if ollamaState == OllamaUnavailable {
logger.Info("semantic search unavailable — ollama not installed",
"install", "brew install ollama")
return nil, nil
}
}
embedder, err := embed.NewEmbedderFromConfig(cfg)
if err != nil {
return nil, fmt.Errorf("embedding provider error: %w", err)
}
if !embedder.Available() {
// Graceful degradation: keyword-only mode when model is missing.
// Matches the OllamaUnavailable path above — warn and continue
// without embeddings instead of hard-exiting (design decision D3).
logger.Warn("embedding model not available, continuing without embeddings",
"model", cfg.Model,
"endpoint", cfg.Endpoint,
"pull", fmt.Sprintf("ollama pull %s", cfg.Model),
"note", "semantic search is unavailable")
return nil, nil
}
logger.Info("embedding model available for indexing", "provider", cfg.Provider, "model", cfg.Model)
return embedder, nil
}
// purgeOrphanedSources compares configured source IDs against source IDs
// stored in the database. Any source in the store that is not in the config
// has its pages deleted (FR-013 auto-purge).
func purgeOrphanedSources(s *store.Store, configs []source.SourceConfig) {
configIDs := make(map[string]bool, len(configs))
for _, cfg := range configs {
configIDs[cfg.ID] = true
}
storedSources, err := s.ListSources()
if err != nil {
logger.Warn("failed to list stored sources for purge check", "err", err)
return
}
for _, src := range storedSources {
logger.Debug("checking source for orphan purge", "source", src.ID, "inConfig", configIDs[src.ID])
if !configIDs[src.ID] {
deleted, err := s.DeletePagesBySource(src.ID)
if err != nil {
logger.Warn("failed to purge orphaned source pages",
"source", src.ID, "err", err)
continue
}
if deleted > 0 {
logger.Info("purged orphaned source",
"source", src.ID, "pages_deleted", deleted)
}
}
}
}
// reportSourceErrors updates source status for any sources that failed