Skip to content

Commit 0a05b6c

Browse files
committed
feat: add cgo universal accumulator
1 parent 930c590 commit 0a05b6c

File tree

17 files changed

+5322
-1
lines changed

17 files changed

+5322
-1
lines changed

go.mod

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
module github.com/sei-protocol/sei-db
22

3-
go 1.19
3+
go 1.22
44

55
require (
66
github.com/alitto/pond v1.8.3

go.sum

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -202,6 +202,7 @@ github.com/cncf/xds/go v0.0.0-20211130200136-a8f946100490/go.mod h1:eXthEFrGJvWH
202202
github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8=
203203
github.com/cockroachdb/datadriven v1.0.0/go.mod h1:5Ib8Meh+jk1RlHIXej6Pzevx/NLlNvQB9pmSBZErGA4=
204204
github.com/cockroachdb/datadriven v1.0.3-0.20230413201302-be42291fc80f h1:otljaYPt5hWxV3MUfO5dFPFiOXg9CyG5/kCfayTqsJ4=
205+
github.com/cockroachdb/datadriven v1.0.3-0.20230413201302-be42291fc80f/go.mod h1:a9RdTaap04u637JoCzcUoIcDmvwSUtcUFtT/C3kJlTU=
205206
github.com/cockroachdb/errors v1.6.1/go.mod h1:tm6FTP5G81vwJ5lC0SizQo374JNCOPrHyXGitRJoDqM=
206207
github.com/cockroachdb/errors v1.8.1 h1:A5+txlVZfOqFBDa4mGz2bUWSp0aHElvHX2bKkdbQu+Y=
207208
github.com/cockroachdb/errors v1.8.1/go.mod h1:qGwQn6JmZ+oMjuLwjWzUNqblqk0xl4CVV3SQbGwK7Ac=
@@ -459,6 +460,7 @@ github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/
459460
github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE=
460461
github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
461462
github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
463+
github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
462464
github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck=
463465
github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
464466
github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
@@ -484,6 +486,7 @@ github.com/google/pprof v0.0.0-20210601050228-01bbb1931b22/go.mod h1:kpwsk12EmLe
484486
github.com/google/pprof v0.0.0-20210609004039-a478d1d731e9/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
485487
github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
486488
github.com/google/pprof v0.0.0-20221118152302-e6195bd50e26 h1:Xim43kblpZXfIBQsbuBVKCudVG457BR2GZFIz3uw3hQ=
489+
github.com/google/pprof v0.0.0-20221118152302-e6195bd50e26/go.mod h1:dDKJzRmX4S37WGHujM7tX//fmj1uioxKzKxz3lo4HJo=
487490
github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
488491
github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ=
489492
github.com/google/trillian v1.3.11/go.mod h1:0tPraVHrSDkA3BO6vKX67zgLXs6SsOAbHEivX+9mPgw=
@@ -734,6 +737,7 @@ github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m
734737
github.com/mattn/go-sqlite3 v1.9.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc=
735738
github.com/mattn/go-sqlite3 v1.14.9/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU=
736739
github.com/mattn/go-sqlite3 v1.14.16 h1:yOQRA0RpS5PFz/oikGwBEqvAWhWg5ufRz4ETLjwpU1Y=
740+
github.com/mattn/go-sqlite3 v1.14.16/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg=
737741
github.com/mattn/goveralls v0.0.2/go.mod h1:8d1ZMHsd7fW6IRPKQh46F2WRpyib5/X4FOpevwGNQEw=
738742
github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
739743
github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo=
@@ -1499,6 +1503,7 @@ golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
14991503
golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
15001504
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
15011505
golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k=
1506+
golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
15021507
golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
15031508
golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
15041509
golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
@@ -1822,6 +1827,7 @@ gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8
18221827
gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
18231828
gopkg.in/check.v1 v1.0.0-20200902074654-038fdea0a05b/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
18241829
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
1830+
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
18251831
gopkg.in/cheggaaa/pb.v1 v1.0.25/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw=
18261832
gopkg.in/cheggaaa/pb.v1 v1.0.28/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw=
18271833
gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
@@ -1870,7 +1876,9 @@ modernc.org/cc/v3 v3.40.0/go.mod h1:/bTg4dnWkSXowUO6ssQKnOV0yMVxDYNIsIrzqTFDGH0=
18701876
modernc.org/ccgo/v3 v3.16.13 h1:Mkgdzl46i5F/CNR/Kj80Ri59hC8TKAhZrYSaqvkwzUw=
18711877
modernc.org/ccgo/v3 v3.16.13/go.mod h1:2Quk+5YgpImhPjv2Qsob1DnZ/4som1lJTodubIcoUkY=
18721878
modernc.org/ccorpus v1.11.6 h1:J16RXiiqiCgua6+ZvQot4yUuUy8zxgqbqEEUuGPlISk=
1879+
modernc.org/ccorpus v1.11.6/go.mod h1:2gEUTrWqdpH2pXsmTM1ZkjeSrUWDpjMu2T6m29L/ErQ=
18731880
modernc.org/httpfs v1.0.6 h1:AAgIpFZRXuYnkjftxTAZwMIiwEqAfk8aVB2/oA6nAeM=
1881+
modernc.org/httpfs v1.0.6/go.mod h1:7dosgurJGp0sPaRanU53W4xZYKh14wfzX420oZADeHM=
18741882
modernc.org/libc v1.24.1 h1:uvJSeCKL/AgzBo2yYIPPTy82v21KgGnizcGYfBHaNuM=
18751883
modernc.org/libc v1.24.1/go.mod h1:FmfO1RLrU3MHJfyi9eYYmZBfi/R+tqZ6+hQ3yQQUkak=
18761884
modernc.org/mathutil v1.5.0 h1:rV0Ko/6SfM+8G+yKiyI830l3Wuz1zRutdslNoQ0kfiQ=
@@ -1884,9 +1892,11 @@ modernc.org/sqlite v1.26.0/go.mod h1:FL3pVXie73rg3Rii6V/u5BoHlSoyeZeIgKZEgHARyCU
18841892
modernc.org/strutil v1.1.3 h1:fNMm+oJklMGYfU9Ylcywl0CO5O6nTfaowNsh2wpPjzY=
18851893
modernc.org/strutil v1.1.3/go.mod h1:MEHNA7PdEnEwLvspRMtWTNnp2nnyvMfkimT1NKNAGbw=
18861894
modernc.org/tcl v1.15.2 h1:C4ybAYCGJw968e+Me18oW55kD/FexcHbqH2xak1ROSY=
1895+
modernc.org/tcl v1.15.2/go.mod h1:3+k/ZaEbKrC8ePv8zJWPtBSW0V7Gg9g8rkmhI1Kfs3c=
18871896
modernc.org/token v1.0.1 h1:A3qvTqOwexpfZZeyI0FeGPDlSWX5pjZu9hF4lU+EKWg=
18881897
modernc.org/token v1.0.1/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM=
18891898
modernc.org/z v1.7.3 h1:zDJf6iHjrnB+WRD88stbXokugjyc0/pB91ri1gO6LZY=
1899+
modernc.org/z v1.7.3/go.mod h1:Ipv4tsdxZRbQyLq9Q1M6gdbkxYzdlrciF2Hi/lS7nWE=
18901900
mvdan.cc/gofumpt v0.3.1/go.mod h1:w3ymliuxvzVx8DAutBnVyDqYb1Niy/yCJt/lk821YCE=
18911901
mvdan.cc/interfacer v0.0.0-20180901003855-c20040233aed/go.mod h1:Xkxe497xwlCKkIaQYRfC7CSLworTXY9RMqwhhCm+8Nc=
18921902
mvdan.cc/lint v0.0.0-20170908181259-adc824a0674b/go.mod h1:2odslEg/xrtNQqCYg2/jCoyKnw3vv5biOc3JnIcYfL4=

sc/universal_accumulator/api.go

Lines changed: 240 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,240 @@
1+
package universalaccumulator
2+
3+
import (
4+
"context"
5+
"errors"
6+
"fmt"
7+
"sync"
8+
)
9+
10+
// UniversalAccumulator provides the main interface for the Universal Accumulator.
11+
// This follows the same patterns as other storage APIs in the repo.
12+
type UniversalAccumulator struct {
13+
engine *AccumulatorEngine
14+
mu sync.RWMutex
15+
}
16+
17+
// NewUniversalAccumulator creates a new universal accumulator instance.
18+
func NewUniversalAccumulator(snapshotInterval uint64) (*UniversalAccumulator, error) {
19+
engine, err := NewAccumulatorEngine(snapshotInterval)
20+
if err != nil {
21+
return nil, fmt.Errorf("failed to create accumulator engine: %w", err)
22+
}
23+
24+
return &UniversalAccumulator{
25+
engine: engine,
26+
}, nil
27+
}
28+
29+
// AddEntries adds multiple entries to the accumulator.
30+
func (acc *UniversalAccumulator) AddEntries(entries []AccumulatorKVPair) error {
31+
acc.mu.Lock()
32+
defer acc.mu.Unlock()
33+
34+
if acc.engine == nil {
35+
return errors.New("accumulator not initialized")
36+
}
37+
38+
// Fast path: direct processing without changeset overhead.
39+
return acc.engine.processEntriesDirect(entries)
40+
}
41+
42+
// AddEntriesStream adds multiple entries to the accumulator via a channel.
43+
func (acc *UniversalAccumulator) AddEntriesStream(
44+
ctx context.Context,
45+
entries <-chan AccumulatorKVPair,
46+
bufferSize int,
47+
) error {
48+
if acc.engine == nil {
49+
return errors.New("accumulator not initialized")
50+
}
51+
52+
buffer := make([]AccumulatorKVPair, 0, bufferSize)
53+
54+
for {
55+
select {
56+
case entry, ok := <-entries:
57+
if !ok {
58+
// Channel closed, process remaining buffer.
59+
if len(buffer) > 0 {
60+
if err := acc.AddEntries(buffer); err != nil {
61+
return fmt.Errorf("failed to add final buffer: %w", err)
62+
}
63+
}
64+
return nil
65+
}
66+
67+
buffer = append(buffer, entry)
68+
69+
// Process buffer when it's full
70+
if len(buffer) >= bufferSize {
71+
if err := acc.AddEntries(buffer); err != nil {
72+
return fmt.Errorf("failed to add buffer: %w", err)
73+
}
74+
buffer = buffer[:0] // Reset buffer
75+
}
76+
77+
case <-ctx.Done():
78+
return ctx.Err()
79+
}
80+
}
81+
}
82+
83+
// DeleteEntries removes multiple entries from the accumulator.
84+
func (acc *UniversalAccumulator) DeleteEntries(entries []AccumulatorKVPair) error {
85+
acc.mu.Lock()
86+
defer acc.mu.Unlock()
87+
88+
if acc.engine == nil {
89+
return errors.New("accumulator not initialized")
90+
}
91+
92+
// Mark all entries as deleted
93+
for i := range entries {
94+
entries[i].Deleted = true
95+
}
96+
97+
// Create a changeset for the deletions
98+
changeset := AccumulatorChangeset{
99+
Version: acc.engine.currentVersion + 1,
100+
Entries: entries,
101+
Name: "api_batch",
102+
}
103+
104+
return acc.engine.ApplyChangeset(changeset)
105+
}
106+
107+
// CalculateRoot calculates and returns the current root hash.
108+
func (acc *UniversalAccumulator) CalculateRoot() ([]byte, error) {
109+
acc.mu.RLock()
110+
defer acc.mu.RUnlock()
111+
112+
if acc.engine == nil {
113+
return nil, errors.New("accumulator not initialized")
114+
}
115+
116+
stateHash := acc.engine.CalculateStateHash()
117+
return stateHash.Hash, nil
118+
}
119+
120+
// GetTotalElements returns the total number of elements in the accumulator.
121+
func (acc *UniversalAccumulator) GetTotalElements() int {
122+
acc.mu.RLock()
123+
defer acc.mu.RUnlock()
124+
125+
if acc.engine == nil {
126+
return 0
127+
}
128+
129+
return acc.engine.totalElements
130+
}
131+
132+
// GetCurrentVersion returns the current version of the accumulator.
133+
func (acc *UniversalAccumulator) GetCurrentVersion() (uint64, error) {
134+
acc.mu.RLock()
135+
defer acc.mu.RUnlock()
136+
137+
if acc.engine == nil {
138+
return 0, errors.New("accumulator not initialized")
139+
}
140+
141+
return acc.engine.GetCurrentVersion()
142+
}
143+
144+
// GetStateHash returns the current state hash with version.
145+
func (acc *UniversalAccumulator) GetStateHash() (AccumulatorStateHash, error) {
146+
acc.mu.RLock()
147+
defer acc.mu.RUnlock()
148+
149+
if acc.engine == nil {
150+
return AccumulatorStateHash{}, errors.New("accumulator not initialized")
151+
}
152+
153+
return acc.engine.GetStateHash(), nil
154+
}
155+
156+
// ExportPerHeightState returns (version, root, factor) for external persistence.
157+
func (acc *UniversalAccumulator) ExportPerHeightState() (uint64, []byte, Factor, error) {
158+
acc.mu.RLock()
159+
defer acc.mu.RUnlock()
160+
if acc.engine == nil {
161+
return 0, nil, nil, errors.New("accumulator not initialized")
162+
}
163+
ver := acc.engine.currentVersion
164+
state := acc.engine.CalculateStateHash()
165+
factor, err := acc.engine.Factor()
166+
if err != nil {
167+
return 0, nil, nil, err
168+
}
169+
return ver, state.Hash, factor, nil
170+
}
171+
172+
// Factor exposes the current fVa for persistence at a given height.
173+
func (acc *UniversalAccumulator) Factor() (Factor, error) {
174+
acc.mu.RLock()
175+
defer acc.mu.RUnlock()
176+
if acc.engine == nil {
177+
return nil, errors.New("accumulator not initialized")
178+
}
179+
return acc.engine.Factor()
180+
}
181+
182+
// SetStateFromFactor restores state fast from a stored factor.
183+
func (acc *UniversalAccumulator) SetStateFromFactor(f Factor) error {
184+
acc.mu.Lock()
185+
defer acc.mu.Unlock()
186+
if acc.engine == nil {
187+
return errors.New("accumulator not initialized")
188+
}
189+
return acc.engine.SetStateFromFactor(f)
190+
}
191+
192+
// ApplyChangeset applies a changeset to the accumulator.
193+
func (acc *UniversalAccumulator) ApplyChangeset(changeset AccumulatorChangeset) error {
194+
acc.mu.Lock()
195+
defer acc.mu.Unlock()
196+
197+
if acc.engine == nil {
198+
return errors.New("accumulator not initialized")
199+
}
200+
201+
return acc.engine.ApplyChangeset(changeset)
202+
}
203+
204+
// ApplyChangesetAsync applies a changeset asynchronously.
205+
func (acc *UniversalAccumulator) ApplyChangesetAsync(changeset AccumulatorChangeset) {
206+
acc.mu.Lock()
207+
defer acc.mu.Unlock()
208+
209+
if acc.engine == nil {
210+
return
211+
}
212+
213+
acc.engine.ApplyChangesetAsync(changeset)
214+
}
215+
216+
// Reset resets the accumulator to a clean state.
217+
func (acc *UniversalAccumulator) Reset() error {
218+
acc.mu.Lock()
219+
defer acc.mu.Unlock()
220+
221+
if acc.engine == nil {
222+
return errors.New("accumulator not initialized")
223+
}
224+
225+
return acc.engine.Reset()
226+
}
227+
228+
// Close closes the accumulator and frees resources.
229+
func (acc *UniversalAccumulator) Close() error {
230+
acc.mu.Lock()
231+
defer acc.mu.Unlock()
232+
233+
if acc.engine == nil {
234+
return nil
235+
}
236+
237+
err := acc.engine.Close()
238+
acc.engine = nil
239+
return err
240+
}

0 commit comments

Comments
 (0)