Skip to content

Commit 381fe9b

Browse files
turbolentturbolent
authored
Merge pull request #5916 from onflow/fxamacker/optimize-commit-and-preload-for-cadence-migration
Use faster atree funcs to speedup Cadence 1.0 migration
2 parents 5d35b0b + 4f5a999 commit 381fe9b

17 files changed

+109
-58
lines changed

‎cmd/util/ledger/migrations/account_storage_migration.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -41,7 +41,7 @@ func NewAccountStorageMigration(
4141
}
4242

4343
// Commit the changes
44-
err = storage.Commit(inter, false)
44+
err = storage.NondeterministicCommit(inter, false)
4545
if err != nil {
4646
return fmt.Errorf("failed to commit changes: %w", err)
4747
}

‎cmd/util/ledger/migrations/cadence_value_diff.go

+35-4
Original file line numberDiff line numberDiff line change
@@ -77,44 +77,53 @@ type CadenceValueDiffReporter struct {
7777
chainID flow.ChainID
7878
reportWriter reporters.ReportWriter
7979
verboseLogging bool
80+
nWorkers int
8081
}
8182

8283
func NewCadenceValueDiffReporter(
8384
address common.Address,
8485
chainID flow.ChainID,
8586
rw reporters.ReportWriter,
8687
verboseLogging bool,
88+
nWorkers int,
8789
) *CadenceValueDiffReporter {
8890
return &CadenceValueDiffReporter{
8991
address: address,
9092
chainID: chainID,
9193
reportWriter: rw,
9294
verboseLogging: verboseLogging,
95+
nWorkers: nWorkers,
9396
}
9497
}
9598

9699
func (dr *CadenceValueDiffReporter) newStorageRuntime(
97100
payloads []*ledger.Payload,
98101
) (
99102
*InterpreterMigrationRuntime,
103+
registers.Registers,
100104
error,
101105
) {
102106
registersByAccount, err := registers.NewByAccountFromPayloads(payloads)
103107
if err != nil {
104-
return nil, err
108+
return nil, nil, err
105109
}
106110

107111
// TODO: maybe make read-only again
108-
return NewInterpreterMigrationRuntime(
112+
runtimeInterface, err := NewInterpreterMigrationRuntime(
109113
registersByAccount,
110114
dr.chainID,
111115
InterpreterMigrationRuntimeConfig{},
112116
)
117+
if err != nil {
118+
return nil, nil, err
119+
}
120+
121+
return runtimeInterface, registersByAccount, nil
113122
}
114123

115124
func (dr *CadenceValueDiffReporter) DiffStates(oldPayloads, newPayloads []*ledger.Payload, domains []string) {
116125
// Create all the runtime components we need for comparing Cadence values.
117-
oldRuntime, err := dr.newStorageRuntime(oldPayloads)
126+
oldRuntime, oldRegs, err := dr.newStorageRuntime(oldPayloads)
118127
if err != nil {
119128
dr.reportWriter.Write(
120129
diffError{
@@ -125,7 +134,18 @@ func (dr *CadenceValueDiffReporter) DiffStates(oldPayloads, newPayloads []*ledge
125134
return
126135
}
127136

128-
newRuntime, err := dr.newStorageRuntime(newPayloads)
137+
err = loadAtreeSlabsInStorage(oldRuntime.Storage, oldRegs, dr.nWorkers)
138+
if err != nil {
139+
dr.reportWriter.Write(
140+
diffError{
141+
Address: dr.address.Hex(),
142+
Kind: diffErrorKindString[abortErrorKind],
143+
Msg: fmt.Sprintf("failed to preload old state atree payloads: %s", err),
144+
})
145+
return
146+
}
147+
148+
newRuntime, newRegs, err := dr.newStorageRuntime(newPayloads)
129149
if err != nil {
130150
dr.reportWriter.Write(
131151
diffError{
@@ -136,6 +156,17 @@ func (dr *CadenceValueDiffReporter) DiffStates(oldPayloads, newPayloads []*ledge
136156
return
137157
}
138158

159+
err = loadAtreeSlabsInStorage(newRuntime.Storage, newRegs, dr.nWorkers)
160+
if err != nil {
161+
dr.reportWriter.Write(
162+
diffError{
163+
Address: dr.address.Hex(),
164+
Kind: diffErrorKindString[abortErrorKind],
165+
Msg: fmt.Sprintf("failed to preload new state atree payloads: %s", err),
166+
})
167+
return
168+
}
169+
139170
// Iterate through all domains and compare cadence values.
140171
for _, domain := range domains {
141172
dr.diffStorageDomain(oldRuntime, newRuntime, domain)

‎cmd/util/ledger/migrations/cadence_value_diff_test.go

+17-16
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@ package migrations
22

33
import (
44
"fmt"
5+
"runtime"
56
"strconv"
67
"testing"
78

@@ -30,7 +31,7 @@ func TestDiffCadenceValues(t *testing.T) {
3031

3132
writer := &testReportWriter{}
3233

33-
diffReporter := NewCadenceValueDiffReporter(address, flow.Emulator, writer, true)
34+
diffReporter := NewCadenceValueDiffReporter(address, flow.Emulator, writer, true, runtime.NumCPU())
3435

3536
diffReporter.DiffStates(
3637
createTestPayloads(t, address, domain),
@@ -46,7 +47,7 @@ func TestDiffCadenceValues(t *testing.T) {
4647

4748
writer := &testReportWriter{}
4849

49-
diffReporter := NewCadenceValueDiffReporter(address, flow.Emulator, writer, true)
50+
diffReporter := NewCadenceValueDiffReporter(address, flow.Emulator, writer, true, runtime.NumCPU())
5051

5152
diffReporter.DiffStates(
5253
createTestPayloads(t, address, domain),
@@ -67,7 +68,7 @@ func TestDiffCadenceValues(t *testing.T) {
6768

6869
writer := &testReportWriter{}
6970

70-
diffReporter := NewCadenceValueDiffReporter(address, flow.Emulator, writer, true)
71+
diffReporter := NewCadenceValueDiffReporter(address, flow.Emulator, writer, true, runtime.NumCPU())
7172

7273
diffReporter.DiffStates(
7374
createTestPayloads(t, address, domain),
@@ -94,7 +95,7 @@ func TestDiffCadenceValues(t *testing.T) {
9495

9596
writer := &testReportWriter{}
9697

97-
diffReporter := NewCadenceValueDiffReporter(address, flow.Emulator, writer, true)
98+
diffReporter := NewCadenceValueDiffReporter(address, flow.Emulator, writer, true, runtime.NumCPU())
9899

99100
diffReporter.DiffStates(
100101
createStorageMapPayloads(t, address, domain, []string{"0", "1"}, []interpreter.Value{interpreter.UInt64Value(0), interpreter.UInt64Value(0)}),
@@ -121,7 +122,7 @@ func TestDiffCadenceValues(t *testing.T) {
121122

122123
writer := &testReportWriter{}
123124

124-
diffReporter := NewCadenceValueDiffReporter(address, flow.Emulator, writer, false)
125+
diffReporter := NewCadenceValueDiffReporter(address, flow.Emulator, writer, false, runtime.NumCPU())
125126

126127
diffReporter.DiffStates(
127128
createStorageMapPayloads(t, address, domain, []string{"0", "1"}, []interpreter.Value{interpreter.UInt64Value(100), interpreter.UInt64Value(101)}),
@@ -148,7 +149,7 @@ func TestDiffCadenceValues(t *testing.T) {
148149

149150
writer := &testReportWriter{}
150151

151-
diffReporter := NewCadenceValueDiffReporter(address, flow.Emulator, writer, false)
152+
diffReporter := NewCadenceValueDiffReporter(address, flow.Emulator, writer, false, runtime.NumCPU())
152153

153154
diffReporter.DiffStates(
154155
createStorageMapPayloads(t, address, domain, []string{"0", "1"}, []interpreter.Value{interpreter.UInt64Value(100), interpreter.UInt64Value(101)}),
@@ -174,7 +175,7 @@ func TestDiffCadenceValues(t *testing.T) {
174175

175176
writer := &testReportWriter{}
176177

177-
diffReporter := NewCadenceValueDiffReporter(address, flow.Emulator, writer, false)
178+
diffReporter := NewCadenceValueDiffReporter(address, flow.Emulator, writer, false, runtime.NumCPU())
178179

179180
createPayloads := func(arrayValues []interpreter.Value) []*ledger.Payload {
180181

@@ -229,7 +230,7 @@ func TestDiffCadenceValues(t *testing.T) {
229230
),
230231
)
231232

232-
err = mr.Storage.Commit(mr.Interpreter, false)
233+
err = mr.Storage.NondeterministicCommit(mr.Interpreter, false)
233234
require.NoError(t, err)
234235

235236
// finalize the transaction
@@ -284,7 +285,7 @@ func TestDiffCadenceValues(t *testing.T) {
284285

285286
writer := &testReportWriter{}
286287

287-
diffReporter := NewCadenceValueDiffReporter(address, flow.Emulator, writer, false)
288+
diffReporter := NewCadenceValueDiffReporter(address, flow.Emulator, writer, false, runtime.NumCPU())
288289

289290
createPayloads := func(dictValues []interpreter.Value) []*ledger.Payload {
290291

@@ -340,7 +341,7 @@ func TestDiffCadenceValues(t *testing.T) {
340341
),
341342
)
342343

343-
err = mr.Storage.Commit(mr.Interpreter, false)
344+
err = mr.Storage.NondeterministicCommit(mr.Interpreter, false)
344345
require.NoError(t, err)
345346

346347
// finalize the transaction
@@ -401,7 +402,7 @@ func TestDiffCadenceValues(t *testing.T) {
401402

402403
writer := &testReportWriter{}
403404

404-
diffReporter := NewCadenceValueDiffReporter(address, flow.Emulator, writer, false)
405+
diffReporter := NewCadenceValueDiffReporter(address, flow.Emulator, writer, false, runtime.NumCPU())
405406

406407
createPayloads := func(compositeFields []string, compositeValues []interpreter.Value) []*ledger.Payload {
407408

@@ -462,7 +463,7 @@ func TestDiffCadenceValues(t *testing.T) {
462463
),
463464
)
464465

465-
err = mr.Storage.Commit(mr.Interpreter, false)
466+
err = mr.Storage.NondeterministicCommit(mr.Interpreter, false)
466467
require.NoError(t, err)
467468

468469
// finalize the transaction
@@ -529,7 +530,7 @@ func TestDiffCadenceValues(t *testing.T) {
529530

530531
writer := &testReportWriter{}
531532

532-
diffReporter := NewCadenceValueDiffReporter(address, flow.Emulator, writer, true)
533+
diffReporter := NewCadenceValueDiffReporter(address, flow.Emulator, writer, true, runtime.NumCPU())
533534

534535
createPayloads := func(compositeFields []string, compositeValues []interpreter.Value) []*ledger.Payload {
535536

@@ -590,7 +591,7 @@ func TestDiffCadenceValues(t *testing.T) {
590591
),
591592
)
592593

593-
err = mr.Storage.Commit(mr.Interpreter, false)
594+
err = mr.Storage.NondeterministicCommit(mr.Interpreter, false)
594595
require.NoError(t, err)
595596

596597
// finalize the transaction
@@ -701,7 +702,7 @@ func createStorageMapPayloads(t *testing.T, address common.Address, domain strin
701702
)
702703
}
703704

704-
err = mr.Storage.Commit(mr.Interpreter, false)
705+
err = mr.Storage.NondeterministicCommit(mr.Interpreter, false)
705706
require.NoError(t, err)
706707

707708
// finalize the transaction
@@ -850,7 +851,7 @@ func createTestPayloads(t *testing.T, address common.Address, domain string) []*
850851
),
851852
)
852853

853-
err = mr.Storage.Commit(mr.Interpreter, false)
854+
err = mr.Storage.NondeterministicCommit(mr.Interpreter, false)
854855
require.NoError(t, err)
855856

856857
// finalize the transaction

‎cmd/util/ledger/migrations/cadence_values_migration.go

+5-2
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,7 @@ type CadenceBaseMigration struct {
4646
errorMessageHandler *errorMessageHandler
4747
programs map[runtime.Location]*interpreter.Program
4848
chainID flow.ChainID
49+
nWorkers int
4950
}
5051

5152
var _ AccountBasedMigration = (*CadenceBaseMigration)(nil)
@@ -65,9 +66,10 @@ func (m *CadenceBaseMigration) Close() error {
6566
func (m *CadenceBaseMigration) InitMigration(
6667
log zerolog.Logger,
6768
_ *registers.ByAccount,
68-
_ int,
69+
nWorkers int,
6970
) error {
7071
m.log = log.With().Str("migration", m.name).Logger()
72+
m.nWorkers = nWorkers
7173

7274
// During the migration, we only provide already checked programs,
7375
// no parsing/checking of contracts is expected.
@@ -120,7 +122,7 @@ func (m *CadenceBaseMigration) MigrateAccount(
120122
var storageHealthErrorBefore error
121123
if m.checkStorageHealthBeforeMigration {
122124

123-
storageHealthErrorBefore = checkStorageHealth(address, storage, accountRegisters)
125+
storageHealthErrorBefore = checkStorageHealth(address, storage, accountRegisters, m.nWorkers)
124126
if storageHealthErrorBefore != nil {
125127
m.log.Warn().
126128
Err(storageHealthErrorBefore).
@@ -204,6 +206,7 @@ func (m *CadenceBaseMigration) MigrateAccount(
204206
m.chainID,
205207
m.diffReporter,
206208
m.logVerboseDiff,
209+
m.nWorkers,
207210
)
208211

209212
accountDiffReporter.DiffStates(

‎cmd/util/ledger/migrations/cadence_values_migration_test.go

+2-2
Original file line numberDiff line numberDiff line change
@@ -791,7 +791,7 @@ func TestProgramParsingError(t *testing.T) {
791791
capabilityValue,
792792
)
793793

794-
err = storage.Commit(runtime.Interpreter, false)
794+
err = storage.NondeterministicCommit(runtime.Interpreter, false)
795795
require.NoError(t, err)
796796

797797
// finalize the transaction
@@ -930,7 +930,7 @@ func TestCoreContractUsage(t *testing.T) {
930930
capabilityValue,
931931
)
932932

933-
err = storage.Commit(runtime.Interpreter, false)
933+
err = storage.NondeterministicCommit(runtime.Interpreter, false)
934934
require.NoError(t, err)
935935

936936
// finalize the transaction

‎cmd/util/ledger/migrations/filter_unreferenced_slabs_migration.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -86,7 +86,7 @@ func (m *FilterUnreferencedSlabsMigration) MigrateAccount(
8686
nil,
8787
)
8888

89-
err := checkStorageHealth(address, storage, accountRegisters)
89+
err := checkStorageHealth(address, storage, accountRegisters, m.nWorkers)
9090
if err == nil {
9191
return nil
9292
}

‎cmd/util/ledger/migrations/filter_unreferenced_slabs_migration_test.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -123,7 +123,7 @@ func TestFilterUnreferencedSlabs(t *testing.T) {
123123
dict1,
124124
)
125125

126-
err = storage.Commit(inter, false)
126+
err = storage.NondeterministicCommit(inter, false)
127127
require.NoError(t, err)
128128

129129
oldPayloads := make([]*ledger.Payload, 0, len(payloads))

‎cmd/util/ledger/migrations/fix_broken_data_migration.go

+2-2
Original file line numberDiff line numberDiff line change
@@ -78,7 +78,7 @@ func (m *FixSlabsWithBrokenReferencesMigration) MigrateAccount(
7878
storage := migrationRuntime.Storage
7979

8080
// Load all atree registers in storage
81-
err := loadAtreeSlabsInStorage(storage, accountRegisters)
81+
err := loadAtreeSlabsInStorage(storage, accountRegisters, m.nWorkers)
8282
if err != nil {
8383
return err
8484
}
@@ -116,7 +116,7 @@ func (m *FixSlabsWithBrokenReferencesMigration) MigrateAccount(
116116

117117
m.mergeBrokenPayloads(brokenPayloads)
118118

119-
err = storage.FastCommit(m.nWorkers)
119+
err = storage.NondeterministicFastCommit(m.nWorkers)
120120
if err != nil {
121121
return fmt.Errorf("failed to commit storage: %w", err)
122122
}

‎cmd/util/ledger/migrations/staged_contracts_migration_test.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -707,7 +707,7 @@ func TestStagedContractsMigration(t *testing.T) {
707707
interpreter.NewUnmeteredStringValue("Also in the same storage path prefix"),
708708
)
709709

710-
err = mr.Storage.Commit(mr.Interpreter, false)
710+
err = mr.Storage.NondeterministicCommit(mr.Interpreter, false)
711711
require.NoError(t, err)
712712

713713
result, err := mr.TransactionState.FinalizeMainTransaction()

0 commit comments

Comments
 (0)