Skip to content

Commit bfc594c

Browse files
jackzampolinjackzampolin
committed
push
1 parent 9b4c033 commit bfc594c

File tree

4 files changed

+67
-56
lines changed

4 files changed

+67
-56
lines changed

‎gturbine/shredding/processor.go

+16-21
Original file line numberDiff line numberDiff line change
@@ -15,15 +15,6 @@ const (
1515
maxBlockSize = 128 * 1024 * 1024 // 128MB maximum block size (matches Solana)
1616
)
1717

18-
type ShredGroup struct {
19-
DataShreds []*gturbine.Shred
20-
RecoveryShreds []*gturbine.Shred
21-
GroupID string // Changed to string for UUID
22-
BlockHash []byte
23-
Height uint64 // Added to struct level
24-
OriginalSize int
25-
}
26-
2718
type Processor struct {
2819
encoder *erasure.Encoder
2920
dataShreds int
@@ -77,12 +68,14 @@ func (p *Processor) CollectDataShred(shred *gturbine.Shred) error {
7768
value, ok := p.groups.Load(shred.GroupID)
7869
if !ok {
7970
group := &ShredGroup{
80-
DataShreds: make([]*gturbine.Shred, shred.Total),
81-
RecoveryShreds: make([]*gturbine.Shred, shred.Total),
82-
GroupID: shred.GroupID,
83-
BlockHash: shred.BlockHash,
84-
Height: shred.Height,
85-
OriginalSize: shred.FullDataSize,
71+
DataShreds: make([]*gturbine.Shred, shred.TotalDataShreds),
72+
RecoveryShreds: make([]*gturbine.Shred, shred.TotalRecoveryShreds),
73+
TotalDataShreds: shred.TotalDataShreds,
74+
TotalRecoveryShreds: shred.TotalRecoveryShreds,
75+
GroupID: shred.GroupID,
76+
BlockHash: shred.BlockHash,
77+
Height: shred.Height,
78+
OriginalSize: shred.FullDataSize,
8679
}
8780
group.DataShreds[shred.Index] = shred
8881
p.groups.Store(shred.GroupID, group)
@@ -118,12 +111,14 @@ func (p *Processor) CollectRecoveryShred(shred *gturbine.Shred) error {
118111
value, ok := p.groups.Load(shred.GroupID)
119112
if !ok {
120113
group := &ShredGroup{
121-
DataShreds: make([]*gturbine.Shred, shred.Total),
122-
RecoveryShreds: make([]*gturbine.Shred, shred.Total),
123-
GroupID: shred.GroupID,
124-
BlockHash: shred.BlockHash,
125-
Height: shred.Height,
126-
OriginalSize: shred.FullDataSize,
114+
DataShreds: make([]*gturbine.Shred, shred.TotalDataShreds),
115+
RecoveryShreds: make([]*gturbine.Shred, shred.TotalRecoveryShreds),
116+
TotalDataShreds: shred.TotalDataShreds,
117+
TotalRecoveryShreds: shred.TotalRecoveryShreds,
118+
GroupID: shred.GroupID,
119+
BlockHash: shred.BlockHash,
120+
Height: shred.Height,
121+
OriginalSize: shred.FullDataSize,
127122
}
128123
group.RecoveryShreds[shred.Index] = shred
129124
p.groups.Store(shred.GroupID, group)

‎gturbine/shredding/shred_group.go

+39-24
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,17 @@ import (
99
"github.com/gordian-engine/gordian/gturbine/erasure"
1010
)
1111

12+
type ShredGroup struct {
13+
DataShreds []*gturbine.Shred
14+
RecoveryShreds []*gturbine.Shred
15+
TotalDataShreds int
16+
TotalRecoveryShreds int
17+
GroupID string // Changed to string for UUID
18+
BlockHash []byte
19+
Height uint64 // Added to struct level
20+
OriginalSize int
21+
}
22+
1223
// FromBlock creates a new ShredGroup from a block of data
1324
func NewShredGroup(block []byte, height uint64, dataShreds, recoveryShreds int, chunkSize uint32) (*ShredGroup, error) {
1425
if len(block) == 0 {
@@ -32,12 +43,14 @@ func NewShredGroup(block []byte, height uint64, dataShreds, recoveryShreds int,
3243

3344
// Create new shred group
3445
group := &ShredGroup{
35-
DataShreds: make([]*gturbine.Shred, dataShreds),
36-
RecoveryShreds: make([]*gturbine.Shred, recoveryShreds),
37-
GroupID: uuid.New().String(),
38-
BlockHash: blockHash[:],
39-
Height: height,
40-
OriginalSize: len(block),
46+
DataShreds: make([]*gturbine.Shred, dataShreds),
47+
RecoveryShreds: make([]*gturbine.Shred, recoveryShreds),
48+
TotalDataShreds: dataShreds,
49+
TotalRecoveryShreds: recoveryShreds,
50+
GroupID: uuid.New().String(),
51+
BlockHash: blockHash[:],
52+
Height: height,
53+
OriginalSize: len(block),
4154
}
4255

4356
// Create fixed-size data chunks
@@ -71,35 +84,37 @@ func NewShredGroup(block []byte, height uint64, dataShreds, recoveryShreds int,
7184
// Create data shreds
7285
for i := range dataBytes {
7386
group.DataShreds[i] = &gturbine.Shred{
74-
Index: uint32(i),
75-
Total: uint32(dataShreds),
76-
Data: dataBytes[i],
77-
BlockHash: blockHash[:],
78-
GroupID: group.GroupID,
79-
Height: height,
80-
FullDataSize: group.OriginalSize,
87+
Index: i,
88+
TotalDataShreds: dataShreds,
89+
TotalRecoveryShreds: recoveryShreds,
90+
Data: dataBytes[i],
91+
BlockHash: blockHash[:],
92+
GroupID: group.GroupID,
93+
Height: height,
94+
FullDataSize: group.OriginalSize,
8195
}
8296
}
8397

8498
// Create recovery shreds
8599
for i := range recoveryBytes {
86100
group.RecoveryShreds[i] = &gturbine.Shred{
87-
Index: uint32(i),
88-
Total: uint32(len(recoveryBytes)),
89-
Data: recoveryBytes[i],
90-
BlockHash: blockHash[:],
91-
GroupID: group.GroupID,
92-
Height: height,
93-
FullDataSize: group.OriginalSize,
101+
Index: i,
102+
TotalDataShreds: dataShreds,
103+
TotalRecoveryShreds: recoveryShreds,
104+
Data: recoveryBytes[i],
105+
BlockHash: blockHash[:],
106+
GroupID: group.GroupID,
107+
Height: height,
108+
FullDataSize: group.OriginalSize,
94109
}
95110
}
96111

97112
return group, nil
98113
}
99114

100-
// IsComplete checks if enough shreds are available for reconstruction
115+
// IsFull checks if enough shreds are available for reconstruction
101116
// NOTE: we'd like shredgroup to know the data threshold as a property on the shredgroup
102-
func (g *ShredGroup) IsComplete(dataThreshold int) bool {
117+
func (g *ShredGroup) IsFull(dataThreshold int) bool {
103118

104119
// TODO: ensure that we've met the threshold by quorum of both data and recovery using the
105120
valid := 0
@@ -191,7 +206,7 @@ func (g *ShredGroup) CollectDataShred(shred *gturbine.Shred) (bool, error) {
191206
}
192207

193208
g.DataShreds[shred.Index] = shred
194-
return g.IsComplete(len(g.DataShreds)), nil
209+
return g.IsFull(len(g.DataShreds)), nil
195210
}
196211

197212
// CollectRecoveryShred adds a recovery shred to the group
@@ -217,5 +232,5 @@ func (g *ShredGroup) CollectRecoveryShred(shred *gturbine.Shred) (bool, error) {
217232
}
218233

219234
g.RecoveryShreds[shred.Index] = shred
220-
return g.IsComplete(len(g.DataShreds)), nil
235+
return g.IsFull(len(g.DataShreds)), nil
221236
}

‎gturbine/shredding/types.go

-9
This file was deleted.

‎gturbine/turbine.go

+12-2
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,14 @@ import (
44
"crypto/ed25519"
55
)
66

7+
// ShredType indicates whether a shred contains data or recovery information
8+
type ShredType int32
9+
10+
const (
11+
ShredTypeData ShredType = iota
12+
ShredTypeRecovery
13+
)
14+
715
// Config holds Turbine configuration
816
type Config struct {
917
DataPlaneFanout uint32
@@ -21,8 +29,10 @@ type Shred struct {
2129
Height uint64 // Block height for chain reference
2230

2331
// Shred-specific metadata
24-
Index uint32 // Index of this shred within the block
25-
Total uint32 // Total number of shreds for this block
32+
Index int // Index of this shred within the block
33+
TotalDataShreds int // Total number of shreds for this block
34+
TotalRecoveryShreds int // Total number of shreds for this block
35+
ShredType ShredType // The type of the shred
2636

2737
Data []byte // The actual shred data
2838
}

0 commit comments

Comments
 (0)