From c3dc68625d7c79cf7a3b671f5b36ba5da157ff69 Mon Sep 17 00:00:00 2001 From: Faye Amacker <33205765+fxamacker@users.noreply.github.com> Date: Fri, 15 Mar 2024 17:50:56 -0500 Subject: [PATCH 01/60] Integrate Atree inlining & Cadence v1.0 to flow-go --- .../migrations/atree_register_migration.go | 2 +- .../migrations/cadence_value_validation.go | 4 +- cmd/util/ledger/migrations/utils.go | 8 +- .../util/migration_runtime_interface.go | 4 +- cmd/util/ledger/util/util.go | 14 +-- fvm/environment/account_key_updater_test.go | 4 +- fvm/environment/accounts.go | 12 +-- fvm/environment/accounts_status.go | 6 +- fvm/environment/accounts_status_test.go | 4 +- fvm/environment/accounts_test.go | 12 +-- fvm/environment/mock/accounts.go | 12 +-- fvm/environment/mock/environment.go | 12 +-- fvm/environment/mock/value_store.go | 12 +-- fvm/environment/value_store.go | 20 ++-- fvm/evm/backends/wrappedEnv.go | 4 +- fvm/evm/emulator/state/collection.go | 97 +++++++++++-------- fvm/evm/emulator/state/stateDB_test.go | 8 +- fvm/evm/testutils/backend.go | 8 +- fvm/evm/testutils/cadence.go | 10 +- go.mod | 6 +- go.sum | 12 +-- insecure/go.mod | 6 +- insecure/go.sum | 12 +-- integration/go.mod | 6 +- integration/go.sum | 12 +-- model/flow/ledger_test.go | 2 +- 26 files changed, 164 insertions(+), 145 deletions(-) diff --git a/cmd/util/ledger/migrations/atree_register_migration.go b/cmd/util/ledger/migrations/atree_register_migration.go index 4de94d0fae4..e3f5bc42bcb 100644 --- a/cmd/util/ledger/migrations/atree_register_migration.go +++ b/cmd/util/ledger/migrations/atree_register_migration.go @@ -176,7 +176,7 @@ func (m *AtreeRegisterMigrator) convertStorageDomain( // no storage for this domain return nil } - storageMapIds[string(atree.SlabIndexToLedgerKey(storageMap.StorageID().Index))] = struct{}{} + storageMapIds[string(atree.SlabIndexToLedgerKey(storageMap.SlabID().Index()))] = struct{}{} iterator := storageMap.Iterator(util.NopMemoryGauge{}) keys := make([]interpreter.StringStorageMapKey, 0, storageMap.Count()) diff --git a/cmd/util/ledger/migrations/cadence_value_validation.go b/cmd/util/ledger/migrations/cadence_value_validation.go index e21183683dd..f830cbd778c 100644 --- a/cmd/util/ledger/migrations/cadence_value_validation.go +++ b/cmd/util/ledger/migrations/cadence_value_validation.go @@ -543,8 +543,8 @@ func (NoopRuntimeInterface) GetAccountContractNames(_ runtime.Address) ([]string panic("unexpected GetAccountContractNames call") } -func (NoopRuntimeInterface) AllocateStorageIndex(_ []byte) (atree.StorageIndex, error) { - panic("unexpected AllocateStorageIndex call") +func (NoopRuntimeInterface) AllocateSlabIndex(_ []byte) (atree.SlabIndex, error) { + panic("unexpected AllocateSlabIndex call") } func (NoopRuntimeInterface) ComputationUsed() (uint64, error) { diff --git a/cmd/util/ledger/migrations/utils.go b/cmd/util/ledger/migrations/utils.go index e747b3dc508..4ef30eb315c 100644 --- a/cmd/util/ledger/migrations/utils.go +++ b/cmd/util/ledger/migrations/utils.go @@ -51,11 +51,11 @@ func (a *AccountsAtreeLedger) ValueExists(owner, key []byte) (exists bool, err e return len(v) > 0, nil } -// AllocateStorageIndex allocates new storage index under the owner accounts to store a new register -func (a *AccountsAtreeLedger) AllocateStorageIndex(owner []byte) (atree.StorageIndex, error) { - v, err := a.Accounts.AllocateStorageIndex(flow.BytesToAddress(owner)) +// AllocateSlabIndex allocates new storage index under the owner accounts to store a new register +func (a *AccountsAtreeLedger) AllocateSlabIndex(owner []byte) (atree.SlabIndex, error) { + v, err := a.Accounts.AllocateSlabIndex(flow.BytesToAddress(owner)) if err != nil { - return atree.StorageIndex{}, fmt.Errorf("storage address allocation failed: %w", err) + return atree.SlabIndex{}, fmt.Errorf("storage address allocation failed: %w", err) } return v, nil } diff --git a/cmd/util/ledger/util/migration_runtime_interface.go b/cmd/util/ledger/util/migration_runtime_interface.go index e7850494378..81e8bcb5b6d 100644 --- a/cmd/util/ledger/util/migration_runtime_interface.go +++ b/cmd/util/ledger/util/migration_runtime_interface.go @@ -291,8 +291,8 @@ func (m *MigrationRuntimeInterface) GetAccountContractNames(_ runtime.Address) ( panic("unexpected GetAccountContractNames call") } -func (m *MigrationRuntimeInterface) AllocateStorageIndex(_ []byte) (atree.StorageIndex, error) { - panic("unexpected AllocateStorageIndex call") +func (m *MigrationRuntimeInterface) AllocateSlabIndex(_ []byte) (atree.SlabIndex, error) { + panic("unexpected AllocateSlabIndex call") } func (m *MigrationRuntimeInterface) ComputationUsed() (uint64, error) { diff --git a/cmd/util/ledger/util/util.go b/cmd/util/ledger/util/util.go index b2248efb513..e26471c78a0 100644 --- a/cmd/util/ledger/util/util.go +++ b/cmd/util/ledger/util/util.go @@ -60,11 +60,11 @@ func (a *AccountsAtreeLedger) ValueExists(owner, key []byte) (exists bool, err e return len(v) > 0, nil } -// AllocateStorageIndex allocates new storage index under the owner accounts to store a new register -func (a *AccountsAtreeLedger) AllocateStorageIndex(owner []byte) (atree.StorageIndex, error) { - v, err := a.Accounts.AllocateStorageIndex(flow.BytesToAddress(owner)) +// AllocateSlabIndex allocates new storage index under the owner accounts to store a new register +func (a *AccountsAtreeLedger) AllocateSlabIndex(owner []byte) (atree.SlabIndex, error) { + v, err := a.Accounts.AllocateSlabIndex(flow.BytesToAddress(owner)) if err != nil { - return atree.StorageIndex{}, fmt.Errorf("storage index allocation failed: %w", err) + return atree.SlabIndex{}, fmt.Errorf("storage index allocation failed: %w", err) } return v, nil } @@ -113,7 +113,7 @@ var _ common.MemoryGauge = (*NopMemoryGauge)(nil) type PayloadsReadonlyLedger struct { Snapshot *PayloadSnapshot - AllocateStorageIndexFunc func(owner []byte) (atree.StorageIndex, error) + AllocateStorageIndexFunc func(owner []byte) (atree.SlabIndex, error) SetValueFunc func(owner, key, value []byte) (err error) } @@ -138,12 +138,12 @@ func (p *PayloadsReadonlyLedger) ValueExists(owner, key []byte) (exists bool, er return ok, nil } -func (p *PayloadsReadonlyLedger) AllocateStorageIndex(owner []byte) (atree.StorageIndex, error) { +func (p *PayloadsReadonlyLedger) AllocateSlabIndex(owner []byte) (atree.SlabIndex, error) { if p.AllocateStorageIndexFunc != nil { return p.AllocateStorageIndexFunc(owner) } - panic("AllocateStorageIndex not expected to be called") + panic("AllocateSlabIndex not expected to be called") } func NewPayloadsReadonlyLedger(snapshot *PayloadSnapshot) *PayloadsReadonlyLedger { diff --git a/fvm/environment/account_key_updater_test.go b/fvm/environment/account_key_updater_test.go index bfb2fa9d2c7..2979d872d7f 100644 --- a/fvm/environment/account_key_updater_test.go +++ b/fvm/environment/account_key_updater_test.go @@ -174,8 +174,8 @@ func (f FakeAccounts) Create(_ []flow.AccountPublicKey, _ flow.Address) error { func (f FakeAccounts) GetValue(_ flow.RegisterID) (flow.RegisterValue, error) { return nil, nil } func (f FakeAccounts) GetStorageUsed(_ flow.Address) (uint64, error) { return 0, nil } func (f FakeAccounts) SetValue(_ flow.RegisterID, _ []byte) error { return nil } -func (f FakeAccounts) AllocateStorageIndex(_ flow.Address) (atree.StorageIndex, error) { - return atree.StorageIndex{}, nil +func (f FakeAccounts) AllocateSlabIndex(_ flow.Address) (atree.SlabIndex, error) { + return atree.SlabIndex{}, nil } func (f FakeAccounts) GenerateAccountLocalID(address flow.Address) (uint64, error) { return 0, nil diff --git a/fvm/environment/accounts.go b/fvm/environment/accounts.go index 01041f19a3f..2cf63b9f582 100644 --- a/fvm/environment/accounts.go +++ b/fvm/environment/accounts.go @@ -36,7 +36,7 @@ type Accounts interface { GetValue(id flow.RegisterID) (flow.RegisterValue, error) GetStorageUsed(address flow.Address) (uint64, error) SetValue(id flow.RegisterID, value flow.RegisterValue) error - AllocateStorageIndex(address flow.Address) (atree.StorageIndex, error) + AllocateSlabIndex(address flow.Address) (atree.SlabIndex, error) GenerateAccountLocalID(address flow.Address) (uint64, error) } @@ -52,16 +52,16 @@ func NewAccounts(txnState state.NestedTransactionPreparer) *StatefulAccounts { } } -func (a *StatefulAccounts) AllocateStorageIndex( +func (a *StatefulAccounts) AllocateSlabIndex( address flow.Address, ) ( - atree.StorageIndex, + atree.SlabIndex, error, ) { // get status status, err := a.getAccountStatus(address) if err != nil { - return atree.StorageIndex{}, err + return atree.SlabIndex{}, err } // get and increment the index @@ -79,7 +79,7 @@ func (a *StatefulAccounts) AllocateStorageIndex( []byte{}) }) if err != nil { - return atree.StorageIndex{}, fmt.Errorf( + return atree.SlabIndex{}, fmt.Errorf( "failed to allocate an storage index: %w", err) } @@ -88,7 +88,7 @@ func (a *StatefulAccounts) AllocateStorageIndex( status.SetStorageIndex(newIndexBytes) err = a.setAccountStatus(address, status) if err != nil { - return atree.StorageIndex{}, fmt.Errorf( + return atree.SlabIndex{}, fmt.Errorf( "failed to allocate an storage index: %w", err) } diff --git a/fvm/environment/accounts_status.go b/fvm/environment/accounts_status.go index a420051550f..93c9a81db6e 100644 --- a/fvm/environment/accounts_status.go +++ b/fvm/environment/accounts_status.go @@ -105,13 +105,13 @@ func (a *AccountStatus) StorageUsed() uint64 { } // SetStorageIndex updates the storage index of the account -func (a *AccountStatus) SetStorageIndex(index atree.StorageIndex) { +func (a *AccountStatus) SetStorageIndex(index atree.SlabIndex) { copy(a[storageIndexStartIndex:storageIndexStartIndex+storageIndexSize], index[:storageIndexSize]) } // StorageIndex returns the storage index of the account -func (a *AccountStatus) StorageIndex() atree.StorageIndex { - var index atree.StorageIndex +func (a *AccountStatus) StorageIndex() atree.SlabIndex { + var index atree.SlabIndex copy(index[:], a[storageIndexStartIndex:storageIndexStartIndex+storageIndexSize]) return index } diff --git a/fvm/environment/accounts_status_test.go b/fvm/environment/accounts_status_test.go index 543ee2b05f1..4ab3c9c1ee5 100644 --- a/fvm/environment/accounts_status_test.go +++ b/fvm/environment/accounts_status_test.go @@ -15,7 +15,7 @@ func TestAccountStatus(t *testing.T) { s := environment.NewAccountStatus() t.Run("test setting values", func(t *testing.T) { - index := atree.StorageIndex{1, 2, 3, 4, 5, 6, 7, 8} + index := atree.SlabIndex{1, 2, 3, 4, 5, 6, 7, 8} s.SetStorageIndex(index) s.SetPublicKeyCount(34) s.SetStorageUsed(56) @@ -58,7 +58,7 @@ func TestAccountStatus(t *testing.T) { migrated, err := environment.AccountStatusFromBytes(oldBytes) require.NoError(t, err) - require.Equal(t, atree.StorageIndex{0, 0, 0, 0, 0, 0, 0, 6}, migrated.StorageIndex()) + require.Equal(t, atree.SlabIndex{0, 0, 0, 0, 0, 0, 0, 6}, migrated.StorageIndex()) require.Equal(t, uint64(5), migrated.PublicKeyCount()) require.Equal(t, uint64(7)+increaseInSize, migrated.StorageUsed()) require.Equal(t, uint64(0), migrated.AccountIdCounter()) diff --git a/fvm/environment/accounts_test.go b/fvm/environment/accounts_test.go index c6ef3cce467..aa44f0b9c2c 100644 --- a/fvm/environment/accounts_test.go +++ b/fvm/environment/accounts_test.go @@ -422,17 +422,17 @@ func TestAccounts_AllocateStorageIndex(t *testing.T) { require.NoError(t, err) // no register set case - i, err := accounts.AllocateStorageIndex(address) + i, err := accounts.AllocateSlabIndex(address) require.NoError(t, err) - require.Equal(t, i, atree.StorageIndex([8]byte{0, 0, 0, 0, 0, 0, 0, 1})) + require.Equal(t, i, atree.SlabIndex([8]byte{0, 0, 0, 0, 0, 0, 0, 1})) // register already set case - i, err = accounts.AllocateStorageIndex(address) + i, err = accounts.AllocateSlabIndex(address) require.NoError(t, err) - require.Equal(t, i, atree.StorageIndex([8]byte{0, 0, 0, 0, 0, 0, 0, 2})) + require.Equal(t, i, atree.SlabIndex([8]byte{0, 0, 0, 0, 0, 0, 0, 2})) // register update successful - i, err = accounts.AllocateStorageIndex(address) + i, err = accounts.AllocateSlabIndex(address) require.NoError(t, err) - require.Equal(t, i, atree.StorageIndex([8]byte{0, 0, 0, 0, 0, 0, 0, 3})) + require.Equal(t, i, atree.SlabIndex([8]byte{0, 0, 0, 0, 0, 0, 0, 3})) } diff --git a/fvm/environment/mock/accounts.go b/fvm/environment/mock/accounts.go index ee4656a4be8..413a25b9b4e 100644 --- a/fvm/environment/mock/accounts.go +++ b/fvm/environment/mock/accounts.go @@ -15,20 +15,20 @@ type Accounts struct { mock.Mock } -// AllocateStorageIndex provides a mock function with given fields: address -func (_m *Accounts) AllocateStorageIndex(address flow.Address) (atree.StorageIndex, error) { +// AllocateSlabIndex provides a mock function with given fields: address +func (_m *Accounts) AllocateSlabIndex(address flow.Address) (atree.SlabIndex, error) { ret := _m.Called(address) - var r0 atree.StorageIndex + var r0 atree.SlabIndex var r1 error - if rf, ok := ret.Get(0).(func(flow.Address) (atree.StorageIndex, error)); ok { + if rf, ok := ret.Get(0).(func(flow.Address) (atree.SlabIndex, error)); ok { return rf(address) } - if rf, ok := ret.Get(0).(func(flow.Address) atree.StorageIndex); ok { + if rf, ok := ret.Get(0).(func(flow.Address) atree.SlabIndex); ok { r0 = rf(address) } else { if ret.Get(0) != nil { - r0 = ret.Get(0).(atree.StorageIndex) + r0 = ret.Get(0).(atree.SlabIndex) } } diff --git a/fvm/environment/mock/environment.go b/fvm/environment/mock/environment.go index 07dec698cb1..53fd5b74968 100644 --- a/fvm/environment/mock/environment.go +++ b/fvm/environment/mock/environment.go @@ -120,20 +120,20 @@ func (_m *Environment) AddAccountKey(address common.Address, publicKey *stdlib.P return r0, r1 } -// AllocateStorageIndex provides a mock function with given fields: owner -func (_m *Environment) AllocateStorageIndex(owner []byte) (atree.StorageIndex, error) { +// AllocateSlabIndex provides a mock function with given fields: owner +func (_m *Environment) AllocateSlabIndex(owner []byte) (atree.SlabIndex, error) { ret := _m.Called(owner) - var r0 atree.StorageIndex + var r0 atree.SlabIndex var r1 error - if rf, ok := ret.Get(0).(func([]byte) (atree.StorageIndex, error)); ok { + if rf, ok := ret.Get(0).(func([]byte) (atree.SlabIndex, error)); ok { return rf(owner) } - if rf, ok := ret.Get(0).(func([]byte) atree.StorageIndex); ok { + if rf, ok := ret.Get(0).(func([]byte) atree.SlabIndex); ok { r0 = rf(owner) } else { if ret.Get(0) != nil { - r0 = ret.Get(0).(atree.StorageIndex) + r0 = ret.Get(0).(atree.SlabIndex) } } diff --git a/fvm/environment/mock/value_store.go b/fvm/environment/mock/value_store.go index acfc3918545..59c54d7dac2 100644 --- a/fvm/environment/mock/value_store.go +++ b/fvm/environment/mock/value_store.go @@ -13,20 +13,20 @@ type ValueStore struct { mock.Mock } -// AllocateStorageIndex provides a mock function with given fields: owner -func (_m *ValueStore) AllocateStorageIndex(owner []byte) (atree.StorageIndex, error) { +// AllocateSlabIndex provides a mock function with given fields: owner +func (_m *ValueStore) AllocateSlabIndex(owner []byte) (atree.SlabIndex, error) { ret := _m.Called(owner) - var r0 atree.StorageIndex + var r0 atree.SlabIndex var r1 error - if rf, ok := ret.Get(0).(func([]byte) (atree.StorageIndex, error)); ok { + if rf, ok := ret.Get(0).(func([]byte) (atree.SlabIndex, error)); ok { return rf(owner) } - if rf, ok := ret.Get(0).(func([]byte) atree.StorageIndex); ok { + if rf, ok := ret.Get(0).(func([]byte) atree.SlabIndex); ok { r0 = rf(owner) } else { if ret.Get(0) != nil { - r0 = ret.Get(0).(atree.StorageIndex) + r0 = ret.Get(0).(atree.SlabIndex) } } diff --git a/fvm/environment/value_store.go b/fvm/environment/value_store.go index 8113de6762c..4f768378500 100644 --- a/fvm/environment/value_store.go +++ b/fvm/environment/value_store.go @@ -20,7 +20,7 @@ type ValueStore interface { ValueExists(owner []byte, key []byte) (bool, error) - AllocateStorageIndex(owner []byte) (atree.StorageIndex, error) + AllocateSlabIndex(owner []byte) (atree.SlabIndex, error) } type ParseRestrictedValueStore struct { @@ -82,16 +82,16 @@ func (store ParseRestrictedValueStore) ValueExists( key) } -func (store ParseRestrictedValueStore) AllocateStorageIndex( +func (store ParseRestrictedValueStore) AllocateSlabIndex( owner []byte, ) ( - atree.StorageIndex, + atree.SlabIndex, error, ) { return parseRestrict1Arg1Ret( store.txnState, trace.FVMEnvAllocateStorageIndex, - store.impl.AllocateStorageIndex, + store.impl.AllocateSlabIndex, owner) } @@ -189,26 +189,26 @@ func (store *valueStore) ValueExists( return len(v) > 0, nil } -// AllocateStorageIndex allocates new storage index under the owner accounts +// AllocateSlabIndex allocates new storage index under the owner accounts // to store a new register. -func (store *valueStore) AllocateStorageIndex( +func (store *valueStore) AllocateSlabIndex( owner []byte, ) ( - atree.StorageIndex, + atree.SlabIndex, error, ) { defer store.tracer.StartChildSpan(trace.FVMEnvAllocateStorageIndex).End() err := store.meter.MeterComputation(ComputationKindAllocateStorageIndex, 1) if err != nil { - return atree.StorageIndex{}, fmt.Errorf( + return atree.SlabIndex{}, fmt.Errorf( "allocate storage index failed: %w", err) } - v, err := store.accounts.AllocateStorageIndex(flow.BytesToAddress(owner)) + v, err := store.accounts.AllocateSlabIndex(flow.BytesToAddress(owner)) if err != nil { - return atree.StorageIndex{}, fmt.Errorf( + return atree.SlabIndex{}, fmt.Errorf( "storage address allocation failed: %w", err) } diff --git a/fvm/evm/backends/wrappedEnv.go b/fvm/evm/backends/wrappedEnv.go index d22aabc191c..152d54236ab 100644 --- a/fvm/evm/backends/wrappedEnv.go +++ b/fvm/evm/backends/wrappedEnv.go @@ -40,8 +40,8 @@ func (we *WrappedEnvironment) ValueExists(owner, key []byte) (bool, error) { return b, handleEnvironmentError(err) } -func (we *WrappedEnvironment) AllocateStorageIndex(owner []byte) (atree.StorageIndex, error) { - index, err := we.env.AllocateStorageIndex(owner) +func (we *WrappedEnvironment) AllocateSlabIndex(owner []byte) (atree.SlabIndex, error) { + index, err := we.env.AllocateSlabIndex(owner) return index, handleEnvironmentError(err) } diff --git a/fvm/evm/emulator/state/collection.go b/fvm/evm/emulator/state/collection.go index 780d81652df..482960370c5 100644 --- a/fvm/evm/emulator/state/collection.go +++ b/fvm/evm/emulator/state/collection.go @@ -47,16 +47,22 @@ func NewCollectionProvider( // calling twice for the same collection might result in odd-behaviours // currently collection provider doesn't do any internal caching to protect aginast these cases func (cp *CollectionProvider) CollectionByID(collectionID []byte) (*Collection, error) { - storageID, err := atree.NewStorageIDFromRawBytes(collectionID) + slabID, err := atree.NewSlabIDFromRawBytes(collectionID) if err != nil { return nil, err } + + // TODO: expose SlabID.Address() in atree + + var address atree.Address + binary.BigEndian.PutUint64(address[:], slabID.AddressAsUint64()) + // sanity check the storage ID address - if storageID.Address != cp.rootAddr { - return nil, fmt.Errorf("root address mismatch %x != %x", storageID.Address, cp.rootAddr) + if address != cp.rootAddr { + return nil, fmt.Errorf("root address mismatch %x != %x", address, cp.rootAddr) } - omap, err := atree.NewMapWithRootID(cp.storage, storageID, atree.NewDefaultDigesterBuilder()) + omap, err := atree.NewMapWithRootID(cp.storage, slabID, atree.NewDefaultDigesterBuilder()) if err != nil { return nil, err } @@ -74,7 +80,7 @@ func (cp *CollectionProvider) NewCollection() (*Collection, error) { return nil, err } storageIDBytes := make([]byte, storageIDSize) - _, err = omap.StorageID().ToRawBytes(storageIDBytes) + _, err = omap.SlabID().ToRawBytes(storageIDBytes) if err != nil { return nil, err } @@ -110,7 +116,7 @@ func (c *Collection) CollectionID() []byte { // // if key doesn't exist it returns nil (no error) func (c *Collection) Get(key []byte) ([]byte, error) { - data, err := c.omap.Get(compare, hashInputProvider, NewByteStringValue(key)) + value, err := c.omap.Get(compare, hashInputProvider, NewByteStringValue(key)) if err != nil { var keyNotFoundError *atree.KeyNotFoundError if errors.As(err, &keyNotFoundError) { @@ -119,11 +125,6 @@ func (c *Collection) Get(key []byte) ([]byte, error) { return nil, err } - value, err := data.StoredValue(c.omap.Storage) - if err != nil { - return nil, err - } - return value.(ByteStringValue).Bytes(), nil } @@ -136,9 +137,9 @@ func (c *Collection) Set(key, value []byte) error { return err } - if id, ok := existingValueStorable.(atree.StorageIDStorable); ok { + if id, ok := existingValueStorable.(atree.SlabIDStorable); ok { // NOTE: deep remove isn't necessary because value is ByteStringValue (not container) - err := c.storage.Remove(atree.StorageID(id)) + err := c.storage.Remove(atree.SlabID(id)) if err != nil { return err } @@ -159,9 +160,9 @@ func (c *Collection) Remove(key []byte) error { return err } - if id, ok := existingValueStorable.(atree.StorageIDStorable); ok { + if id, ok := existingValueStorable.(atree.SlabIDStorable); ok { // NOTE: deep remove isn't necessary because value is ByteStringValue (not container) - err := c.storage.Remove(atree.StorageID(id)) + err := c.storage.Remove(atree.SlabID(id)) if err != nil { return err } @@ -175,8 +176,8 @@ func (c *Collection) Destroy() ([][]byte, error) { keys := make([][]byte, c.omap.Count()) i := 0 err := c.omap.PopIterate(func(keyStorable atree.Storable, valueStorable atree.Storable) { - if id, ok := valueStorable.(atree.StorageIDStorable); ok { - err := c.storage.Remove(atree.StorageID(id)) + if id, ok := valueStorable.(atree.SlabIDStorable); ok { + err := c.storage.Remove(atree.SlabID(id)) if err != nil && cachedErr == nil { cachedErr = err } @@ -194,7 +195,7 @@ func (c *Collection) Destroy() ([][]byte, error) { if err != nil { return keys, err } - return keys, c.storage.Remove(c.omap.StorageID()) + return keys, c.storage.Remove(c.omap.SlabID()) } // Size returns the number of items in the collection @@ -229,24 +230,7 @@ func (v ByteStringValue) Storable(storage atree.SlabStorage, address atree.Addre } // Create StorableSlab - id, err := storage.GenerateStorageID(address) - if err != nil { - return nil, err - } - - slab := &atree.StorableSlab{ - StorageID: id, - Storable: v, - } - - // Store StorableSlab in storage - err = storage.Store(id, slab) - if err != nil { - return nil, err - } - - // Return storage id as storable - return atree.StorageIDStorable(id), nil + return atree.NewStorableSlab(storage, address, v) } func (v ByteStringValue) Encode(enc *atree.Encoder) error { @@ -311,7 +295,7 @@ func (v ByteStringValue) Bytes() []byte { return v.data } -func decodeStorable(dec *cbor.StreamDecoder, _ atree.StorageID) (atree.Storable, error) { +func decodeStorable(dec *cbor.StreamDecoder, slabID atree.SlabID, inlinedExtraData []atree.ExtraData) (atree.Storable, error) { t, err := dec.NextType() if err != nil { return nil, err @@ -333,8 +317,31 @@ func decodeStorable(dec *cbor.StreamDecoder, _ atree.StorageID) (atree.Storable, switch tagNumber { - case atree.CBORTagStorageID: - return atree.DecodeStorageIDStorable(dec) + case atree.CBORTagSlabID: + return atree.DecodeSlabIDStorable(dec) + + case atree.CBORTagInlinedArray: + return atree.DecodeInlinedArrayStorable( + dec, + decodeStorable, + slabID, + inlinedExtraData) + + case atree.CBORTagInlinedMap: + return atree.DecodeInlinedMapStorable( + dec, + decodeStorable, + slabID, + inlinedExtraData, + ) + + case atree.CBORTagInlinedCompactMap: + return atree.DecodeInlinedCompactMapStorable( + dec, + decodeStorable, + slabID, + inlinedExtraData, + ) default: return nil, fmt.Errorf("invalid tag number %d", tagNumber) @@ -403,6 +410,18 @@ type emptyTypeInfo struct{} var _ atree.TypeInfo = emptyTypeInfo{} +func (emptyTypeInfo) IsComposite() bool { + return false +} + +func (emptyTypeInfo) Identifier() string { + return "" +} + +func (e emptyTypeInfo) Copy() atree.TypeInfo { + return e +} + func (emptyTypeInfo) Encode(e *cbor.StreamEncoder) error { return e.EncodeNil() } diff --git a/fvm/evm/emulator/state/stateDB_test.go b/fvm/evm/emulator/state/stateDB_test.go index f7ff6bbcbd9..bbf402973fc 100644 --- a/fvm/evm/emulator/state/stateDB_test.go +++ b/fvm/evm/emulator/state/stateDB_test.go @@ -247,8 +247,8 @@ func TestStateDB(t *testing.T) { SetValueFunc: func(owner, key, value []byte) error { return atree.NewUserError(fmt.Errorf("key not found")) }, - AllocateStorageIndexFunc: func(owner []byte) (atree.StorageIndex, error) { - return atree.StorageIndex{}, nil + AllocateStorageIndexFunc: func(owner []byte) (atree.SlabIndex, error) { + return atree.SlabIndex{}, nil }, } db, err := state.NewStateDB(ledger, rootAddr) @@ -271,8 +271,8 @@ func TestStateDB(t *testing.T) { SetValueFunc: func(owner, key, value []byte) error { return atree.NewFatalError(fmt.Errorf("key not found")) }, - AllocateStorageIndexFunc: func(owner []byte) (atree.StorageIndex, error) { - return atree.StorageIndex{}, nil + AllocateStorageIndexFunc: func(owner []byte) (atree.SlabIndex, error) { + return atree.SlabIndex{}, nil }, } db, err := state.NewStateDB(ledger, rootAddr) diff --git a/fvm/evm/testutils/backend.go b/fvm/evm/testutils/backend.go index a8c831ca7d4..077b5e38955 100644 --- a/fvm/evm/testutils/backend.go +++ b/fvm/evm/testutils/backend.go @@ -78,7 +78,7 @@ func GetSimpleValueStore() *TestValueStore { bytesRead += len(fk) + len(value) return len(value) > 0, nil }, - AllocateStorageIndexFunc: func(owner []byte) (atree.StorageIndex, error) { + AllocateStorageIndexFunc: func(owner []byte) (atree.SlabIndex, error) { index := allocator[string(owner)] // TODO: figure out why it result in a collision if index == 0 { @@ -89,7 +89,7 @@ func GetSimpleValueStore() *TestValueStore { binary.BigEndian.PutUint64(data[:], index) bytesRead += len(owner) + 8 bytesWritten += len(owner) + 8 - return atree.StorageIndex(data), nil + return atree.SlabIndex(data), nil }, TotalStorageSizeFunc: func() int { size := 0 @@ -191,7 +191,7 @@ type TestValueStore struct { GetValueFunc func(owner, key []byte) ([]byte, error) SetValueFunc func(owner, key, value []byte) error ValueExistsFunc func(owner, key []byte) (bool, error) - AllocateStorageIndexFunc func(owner []byte) (atree.StorageIndex, error) + AllocateStorageIndexFunc func(owner []byte) (atree.SlabIndex, error) TotalStorageSizeFunc func() int TotalBytesReadFunc func() int TotalBytesWrittenFunc func() int @@ -222,7 +222,7 @@ func (vs *TestValueStore) ValueExists(owner, key []byte) (bool, error) { return vs.ValueExistsFunc(owner, key) } -func (vs *TestValueStore) AllocateStorageIndex(owner []byte) (atree.StorageIndex, error) { +func (vs *TestValueStore) AllocateSlabIndex(owner []byte) (atree.SlabIndex, error) { if vs.AllocateStorageIndexFunc == nil { panic("method not set") } diff --git a/fvm/evm/testutils/cadence.go b/fvm/evm/testutils/cadence.go index 577ab740c59..b6dfb7370ab 100644 --- a/fvm/evm/testutils/cadence.go +++ b/fvm/evm/testutils/cadence.go @@ -110,7 +110,7 @@ type TestLedger struct { OnValueExists func(owner, key []byte) (exists bool, err error) OnGetValue func(owner, key []byte) (value []byte, err error) OnSetValue func(owner, key, value []byte) (err error) - OnAllocateStorageIndex func(owner []byte) (atree.StorageIndex, error) + OnAllocateStorageIndex func(owner []byte) (atree.SlabIndex, error) } var _ atree.Ledger = TestLedger{} @@ -127,7 +127,7 @@ func (s TestLedger) ValueExists(owner, key []byte) (exists bool, err error) { return s.OnValueExists(owner, key) } -func (s TestLedger) AllocateStorageIndex(owner []byte) (atree.StorageIndex, error) { +func (s TestLedger) AllocateSlabIndex(owner []byte) (atree.SlabIndex, error) { return s.OnAllocateStorageIndex(owner) } @@ -173,7 +173,7 @@ func NewTestLedger( } return nil }, - OnAllocateStorageIndex: func(owner []byte) (result atree.StorageIndex, err error) { + OnAllocateStorageIndex: func(owner []byte) (result atree.SlabIndex, err error) { index := storageIndices[string(owner)] + 1 storageIndices[string(owner)] = index binary.BigEndian.PutUint64(result[:], index) @@ -356,8 +356,8 @@ func (i *TestRuntimeInterface) SetValue(owner, key, value []byte) (err error) { return i.Storage.SetValue(owner, key, value) } -func (i *TestRuntimeInterface) AllocateStorageIndex(owner []byte) (atree.StorageIndex, error) { - return i.Storage.AllocateStorageIndex(owner) +func (i *TestRuntimeInterface) AllocateSlabIndex(owner []byte) (atree.SlabIndex, error) { + return i.Storage.AllocateSlabIndex(owner) } func (i *TestRuntimeInterface) CreateAccount(payer runtime.Address) (address runtime.Address, err error) { diff --git a/go.mod b/go.mod index 9e723ba34ac..8d8a9294bf9 100644 --- a/go.mod +++ b/go.mod @@ -50,13 +50,13 @@ require ( github.com/multiformats/go-multiaddr v0.12.2 github.com/multiformats/go-multiaddr-dns v0.3.1 github.com/multiformats/go-multihash v0.2.3 - github.com/onflow/atree v0.6.1-0.20240308163425-dc825c20b1a2 - github.com/onflow/cadence v1.0.0-preview.14 + github.com/onflow/atree v0.6.1-0.20240314011440-92714cac03fb + github.com/onflow/cadence v1.0.0-preview.14.0.20240315230827-7fc0649bf611 github.com/onflow/crypto v0.25.0 github.com/onflow/flow v0.3.4 github.com/onflow/flow-core-contracts/lib/go/contracts v0.15.2-0.20240305214031-d81e0c3b42f3 github.com/onflow/flow-core-contracts/lib/go/templates v0.15.2-0.20240305214031-d81e0c3b42f3 - github.com/onflow/flow-go-sdk v1.0.0-preview.12 + github.com/onflow/flow-go-sdk v1.0.0-M8 github.com/onflow/flow/protobuf/go/flow v0.3.7 github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 github.com/pierrec/lz4 v2.6.1+incompatible diff --git a/go.sum b/go.sum index 34ea53f95db..7ac085ac6d5 100644 --- a/go.sum +++ b/go.sum @@ -2483,13 +2483,13 @@ github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5/go.mod h1:v github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec= github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY= github.com/onflow/atree v0.6.1-0.20230711151834-86040b30171f/go.mod h1:xvP61FoOs95K7IYdIYRnNcYQGf4nbF/uuJ0tHf4DRuM= -github.com/onflow/atree v0.6.1-0.20240308163425-dc825c20b1a2 h1:jJLDswfAVB0bHCu1y1FPdKukPcTNmN+jYEX9S9phbv0= -github.com/onflow/atree v0.6.1-0.20240308163425-dc825c20b1a2/go.mod h1:xvP61FoOs95K7IYdIYRnNcYQGf4nbF/uuJ0tHf4DRuM= +github.com/onflow/atree v0.6.1-0.20240314011440-92714cac03fb h1:9w+8wseSv7TZ9ikGfKSZ6yJifgYLhm717Zo0SVKpGEQ= +github.com/onflow/atree v0.6.1-0.20240314011440-92714cac03fb/go.mod h1:7YNAyCd5JENq+NzH+fR1ABUZVzbSq9dkt0+5fZH3L2A= github.com/onflow/boxo v0.0.0-20240201202436-f2477b92f483 h1:LpiQhTAfM9CAmNVEs0n//cBBgCg+vJSiIxTHYUklZ84= github.com/onflow/boxo v0.0.0-20240201202436-f2477b92f483/go.mod h1:pIZgTWdm3k3pLF9Uq6MB8JEcW07UDwNJjlXW1HELW80= github.com/onflow/cadence v1.0.0-M3/go.mod h1:odXGZZ/wGNA5mwT8bC9v8u8EXACHllB2ABSZK65TGL8= -github.com/onflow/cadence v1.0.0-preview.14 h1:nQoKX7MBIkLj32UUqESQT0Oj6rvKZF+boUo0oDWB8Kw= -github.com/onflow/cadence v1.0.0-preview.14/go.mod h1:no8+e5V51B9mgfi4U9xdeH+GxcJdoKKDP9gdxEj9Jdg= +github.com/onflow/cadence v1.0.0-preview.14.0.20240315230827-7fc0649bf611 h1:YR32C64UhtaZNeTR4tg1kCxxFdA8eBxk/ZyOilAUEas= +github.com/onflow/cadence v1.0.0-preview.14.0.20240315230827-7fc0649bf611/go.mod h1:5szKvYbjOrFTySYRDFu/HR6rdUH4yBjDDv2LoQsDUig= github.com/onflow/crypto v0.25.0 h1:BeWbLsh3ZD13Ej+Uky6kg1PL1ZIVBDVX+2MVBNwqddg= github.com/onflow/crypto v0.25.0/go.mod h1:C8FbaX0x8y+FxWjbkHy0Q4EASCDR9bSPWZqlpCLYyVI= github.com/onflow/flow v0.3.4 h1:FXUWVdYB90f/rjNcY0Owo30gL790tiYff9Pb/sycXYE= @@ -2503,8 +2503,8 @@ github.com/onflow/flow-ft/lib/go/contracts v0.7.1-0.20240305212555-29d91e18f0c1/ github.com/onflow/flow-ft/lib/go/templates v0.7.1-0.20240305212555-29d91e18f0c1 h1:7BkqJ21EQAbYCZqLj2xCIqJvhrHNCXe9BfLTEiQqIx0= github.com/onflow/flow-ft/lib/go/templates v0.7.1-0.20240305212555-29d91e18f0c1/go.mod h1:uQ8XFqmMK2jxyBSVrmyuwdWjTEb+6zGjRYotfDJ5pAE= github.com/onflow/flow-go-sdk v1.0.0-M1/go.mod h1:TDW0MNuCs4SvqYRUzkbRnRmHQL1h4X8wURsCw9P9beo= -github.com/onflow/flow-go-sdk v1.0.0-preview.12 h1:stfIbPpBSJ7DDXFFLvHXrCBWDHipPwAR6lwuKjpOl0I= -github.com/onflow/flow-go-sdk v1.0.0-preview.12/go.mod h1:EnBbDE+q7VdTCnfw2d8ssTCeTKvLHjDEXdzLH48Mhq4= +github.com/onflow/flow-go-sdk v1.0.0-M8 h1:jnWA6X29sZlMsrIjdxoEkBAJsZBazXTcQJ/wPw1ZbfU= +github.com/onflow/flow-go-sdk v1.0.0-M8/go.mod h1:qFy2LF0DKfYsEnyEaIxD0frA/zAMc5hURbnqTrSZBjA= github.com/onflow/flow-nft/lib/go/contracts v1.1.1-0.20240305213046-9026973838d7 h1:0/s3DzQp3JHWuuz3DpGwniFShtgEy3wq1uBB+4uOG1I= github.com/onflow/flow-nft/lib/go/contracts v1.1.1-0.20240305213046-9026973838d7/go.mod h1:2gpbza+uzs1k7x31hkpBPlggIRkI53Suo0n2AyA2HcE= github.com/onflow/flow-nft/lib/go/templates v0.0.0-20240305213046-9026973838d7 h1:VU294Tk/Ra/UuuJbArLILA+x+4qX5qGPTY/kokNKyA4= diff --git a/insecure/go.mod b/insecure/go.mod index eeccc33452f..0e5a2a4726d 100644 --- a/insecure/go.mod +++ b/insecure/go.mod @@ -204,13 +204,13 @@ require ( github.com/multiformats/go-multistream v0.5.0 // indirect github.com/multiformats/go-varint v0.0.7 // indirect github.com/olekukonko/tablewriter v0.0.5 // indirect - github.com/onflow/atree v0.6.1-0.20240308163425-dc825c20b1a2 // indirect - github.com/onflow/cadence v1.0.0-preview.14 // indirect + github.com/onflow/atree v0.6.1-0.20240314011440-92714cac03fb // indirect + github.com/onflow/cadence v1.0.0-preview.14.0.20240315230827-7fc0649bf611 // indirect github.com/onflow/flow-core-contracts/lib/go/contracts v0.15.2-0.20240305214031-d81e0c3b42f3 // indirect github.com/onflow/flow-core-contracts/lib/go/templates v0.15.2-0.20240305214031-d81e0c3b42f3 // indirect github.com/onflow/flow-ft/lib/go/contracts v0.7.1-0.20240305212555-29d91e18f0c1 // indirect github.com/onflow/flow-ft/lib/go/templates v0.7.1-0.20240305212555-29d91e18f0c1 // indirect - github.com/onflow/flow-go-sdk v1.0.0-preview.12 // indirect + github.com/onflow/flow-go-sdk v1.0.0-M8 // indirect github.com/onflow/flow-nft/lib/go/contracts v1.1.1-0.20240305213046-9026973838d7 // indirect github.com/onflow/flow-nft/lib/go/templates v0.0.0-20240305213046-9026973838d7 // indirect github.com/onflow/flow/protobuf/go/flow v0.3.7 // indirect diff --git a/insecure/go.sum b/insecure/go.sum index 373272a7eb2..9704d1ba0c5 100644 --- a/insecure/go.sum +++ b/insecure/go.sum @@ -2470,11 +2470,11 @@ github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5/go.mod h1:v github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec= github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY= github.com/onflow/atree v0.6.1-0.20230711151834-86040b30171f/go.mod h1:xvP61FoOs95K7IYdIYRnNcYQGf4nbF/uuJ0tHf4DRuM= -github.com/onflow/atree v0.6.1-0.20240308163425-dc825c20b1a2 h1:jJLDswfAVB0bHCu1y1FPdKukPcTNmN+jYEX9S9phbv0= -github.com/onflow/atree v0.6.1-0.20240308163425-dc825c20b1a2/go.mod h1:xvP61FoOs95K7IYdIYRnNcYQGf4nbF/uuJ0tHf4DRuM= +github.com/onflow/atree v0.6.1-0.20240314011440-92714cac03fb h1:9w+8wseSv7TZ9ikGfKSZ6yJifgYLhm717Zo0SVKpGEQ= +github.com/onflow/atree v0.6.1-0.20240314011440-92714cac03fb/go.mod h1:7YNAyCd5JENq+NzH+fR1ABUZVzbSq9dkt0+5fZH3L2A= github.com/onflow/cadence v1.0.0-M3/go.mod h1:odXGZZ/wGNA5mwT8bC9v8u8EXACHllB2ABSZK65TGL8= -github.com/onflow/cadence v1.0.0-preview.14 h1:nQoKX7MBIkLj32UUqESQT0Oj6rvKZF+boUo0oDWB8Kw= -github.com/onflow/cadence v1.0.0-preview.14/go.mod h1:no8+e5V51B9mgfi4U9xdeH+GxcJdoKKDP9gdxEj9Jdg= +github.com/onflow/cadence v1.0.0-preview.14.0.20240315230827-7fc0649bf611 h1:YR32C64UhtaZNeTR4tg1kCxxFdA8eBxk/ZyOilAUEas= +github.com/onflow/cadence v1.0.0-preview.14.0.20240315230827-7fc0649bf611/go.mod h1:5szKvYbjOrFTySYRDFu/HR6rdUH4yBjDDv2LoQsDUig= github.com/onflow/crypto v0.25.0 h1:BeWbLsh3ZD13Ej+Uky6kg1PL1ZIVBDVX+2MVBNwqddg= github.com/onflow/crypto v0.25.0/go.mod h1:C8FbaX0x8y+FxWjbkHy0Q4EASCDR9bSPWZqlpCLYyVI= github.com/onflow/flow-core-contracts/lib/go/contracts v0.15.2-0.20240305214031-d81e0c3b42f3 h1:UNLms46HthnzKTR3DVQmEoDerf4EtNUMfkombIeT3U8= @@ -2486,8 +2486,8 @@ github.com/onflow/flow-ft/lib/go/contracts v0.7.1-0.20240305212555-29d91e18f0c1/ github.com/onflow/flow-ft/lib/go/templates v0.7.1-0.20240305212555-29d91e18f0c1 h1:7BkqJ21EQAbYCZqLj2xCIqJvhrHNCXe9BfLTEiQqIx0= github.com/onflow/flow-ft/lib/go/templates v0.7.1-0.20240305212555-29d91e18f0c1/go.mod h1:uQ8XFqmMK2jxyBSVrmyuwdWjTEb+6zGjRYotfDJ5pAE= github.com/onflow/flow-go-sdk v1.0.0-M1/go.mod h1:TDW0MNuCs4SvqYRUzkbRnRmHQL1h4X8wURsCw9P9beo= -github.com/onflow/flow-go-sdk v1.0.0-preview.12 h1:stfIbPpBSJ7DDXFFLvHXrCBWDHipPwAR6lwuKjpOl0I= -github.com/onflow/flow-go-sdk v1.0.0-preview.12/go.mod h1:EnBbDE+q7VdTCnfw2d8ssTCeTKvLHjDEXdzLH48Mhq4= +github.com/onflow/flow-go-sdk v1.0.0-M8 h1:jnWA6X29sZlMsrIjdxoEkBAJsZBazXTcQJ/wPw1ZbfU= +github.com/onflow/flow-go-sdk v1.0.0-M8/go.mod h1:qFy2LF0DKfYsEnyEaIxD0frA/zAMc5hURbnqTrSZBjA= github.com/onflow/flow-nft/lib/go/contracts v1.1.1-0.20240305213046-9026973838d7 h1:0/s3DzQp3JHWuuz3DpGwniFShtgEy3wq1uBB+4uOG1I= github.com/onflow/flow-nft/lib/go/contracts v1.1.1-0.20240305213046-9026973838d7/go.mod h1:2gpbza+uzs1k7x31hkpBPlggIRkI53Suo0n2AyA2HcE= github.com/onflow/flow-nft/lib/go/templates v0.0.0-20240305213046-9026973838d7 h1:VU294Tk/Ra/UuuJbArLILA+x+4qX5qGPTY/kokNKyA4= diff --git a/integration/go.mod b/integration/go.mod index 3b45317d218..674980fa7d6 100644 --- a/integration/go.mod +++ b/integration/go.mod @@ -21,13 +21,13 @@ require ( github.com/ipfs/go-ds-badger2 v0.1.3 github.com/ipfs/go-ipfs-blockstore v1.3.0 github.com/libp2p/go-libp2p v0.32.2 - github.com/onflow/cadence v1.0.0-preview.14 + github.com/onflow/cadence v1.0.0-preview.14.0.20240315230827-7fc0649bf611 github.com/onflow/crypto v0.25.0 github.com/onflow/flow-core-contracts/lib/go/contracts v0.15.2-0.20240305214031-d81e0c3b42f3 github.com/onflow/flow-core-contracts/lib/go/templates v0.15.2-0.20240305214031-d81e0c3b42f3 github.com/onflow/flow-emulator v1.0.0-M7.0.20240227020422-2ec59747f9be github.com/onflow/flow-go v0.34.0-crescendo-preview.2.0.20240227001756-cb6311412b78 - github.com/onflow/flow-go-sdk v1.0.0-preview.12 + github.com/onflow/flow-go-sdk v1.0.0-M8 github.com/onflow/flow-go/insecure v0.0.0-00010101000000-000000000000 github.com/onflow/flow/protobuf/go/flow v0.3.7 github.com/plus3it/gorecurcopy v0.0.1 @@ -251,7 +251,7 @@ require ( github.com/multiformats/go-multistream v0.5.0 // indirect github.com/multiformats/go-varint v0.0.7 // indirect github.com/olekukonko/tablewriter v0.0.5 // indirect - github.com/onflow/atree v0.6.1-0.20240308163425-dc825c20b1a2 // indirect + github.com/onflow/atree v0.6.1-0.20240314011440-92714cac03fb // indirect github.com/onflow/flow-ft/lib/go/contracts v0.7.1-0.20240305212555-29d91e18f0c1 // indirect github.com/onflow/flow-ft/lib/go/templates v0.7.1-0.20240305212555-29d91e18f0c1 // indirect github.com/onflow/flow-nft/lib/go/contracts v1.1.1-0.20240305213046-9026973838d7 // indirect diff --git a/integration/go.sum b/integration/go.sum index 18b46ea4230..873c3e617cd 100644 --- a/integration/go.sum +++ b/integration/go.sum @@ -2546,11 +2546,11 @@ github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5/go.mod h1:v github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec= github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY= github.com/onflow/atree v0.6.1-0.20230711151834-86040b30171f/go.mod h1:xvP61FoOs95K7IYdIYRnNcYQGf4nbF/uuJ0tHf4DRuM= -github.com/onflow/atree v0.6.1-0.20240308163425-dc825c20b1a2 h1:jJLDswfAVB0bHCu1y1FPdKukPcTNmN+jYEX9S9phbv0= -github.com/onflow/atree v0.6.1-0.20240308163425-dc825c20b1a2/go.mod h1:xvP61FoOs95K7IYdIYRnNcYQGf4nbF/uuJ0tHf4DRuM= +github.com/onflow/atree v0.6.1-0.20240314011440-92714cac03fb h1:9w+8wseSv7TZ9ikGfKSZ6yJifgYLhm717Zo0SVKpGEQ= +github.com/onflow/atree v0.6.1-0.20240314011440-92714cac03fb/go.mod h1:7YNAyCd5JENq+NzH+fR1ABUZVzbSq9dkt0+5fZH3L2A= github.com/onflow/cadence v1.0.0-M3/go.mod h1:odXGZZ/wGNA5mwT8bC9v8u8EXACHllB2ABSZK65TGL8= -github.com/onflow/cadence v1.0.0-preview.14 h1:nQoKX7MBIkLj32UUqESQT0Oj6rvKZF+boUo0oDWB8Kw= -github.com/onflow/cadence v1.0.0-preview.14/go.mod h1:no8+e5V51B9mgfi4U9xdeH+GxcJdoKKDP9gdxEj9Jdg= +github.com/onflow/cadence v1.0.0-preview.14.0.20240315230827-7fc0649bf611 h1:YR32C64UhtaZNeTR4tg1kCxxFdA8eBxk/ZyOilAUEas= +github.com/onflow/cadence v1.0.0-preview.14.0.20240315230827-7fc0649bf611/go.mod h1:5szKvYbjOrFTySYRDFu/HR6rdUH4yBjDDv2LoQsDUig= github.com/onflow/crypto v0.25.0 h1:BeWbLsh3ZD13Ej+Uky6kg1PL1ZIVBDVX+2MVBNwqddg= github.com/onflow/crypto v0.25.0/go.mod h1:C8FbaX0x8y+FxWjbkHy0Q4EASCDR9bSPWZqlpCLYyVI= github.com/onflow/flow-core-contracts/lib/go/contracts v0.15.2-0.20240305214031-d81e0c3b42f3 h1:UNLms46HthnzKTR3DVQmEoDerf4EtNUMfkombIeT3U8= @@ -2564,8 +2564,8 @@ github.com/onflow/flow-ft/lib/go/contracts v0.7.1-0.20240305212555-29d91e18f0c1/ github.com/onflow/flow-ft/lib/go/templates v0.7.1-0.20240305212555-29d91e18f0c1 h1:7BkqJ21EQAbYCZqLj2xCIqJvhrHNCXe9BfLTEiQqIx0= github.com/onflow/flow-ft/lib/go/templates v0.7.1-0.20240305212555-29d91e18f0c1/go.mod h1:uQ8XFqmMK2jxyBSVrmyuwdWjTEb+6zGjRYotfDJ5pAE= github.com/onflow/flow-go-sdk v1.0.0-M1/go.mod h1:TDW0MNuCs4SvqYRUzkbRnRmHQL1h4X8wURsCw9P9beo= -github.com/onflow/flow-go-sdk v1.0.0-preview.12 h1:stfIbPpBSJ7DDXFFLvHXrCBWDHipPwAR6lwuKjpOl0I= -github.com/onflow/flow-go-sdk v1.0.0-preview.12/go.mod h1:EnBbDE+q7VdTCnfw2d8ssTCeTKvLHjDEXdzLH48Mhq4= +github.com/onflow/flow-go-sdk v1.0.0-M8 h1:jnWA6X29sZlMsrIjdxoEkBAJsZBazXTcQJ/wPw1ZbfU= +github.com/onflow/flow-go-sdk v1.0.0-M8/go.mod h1:qFy2LF0DKfYsEnyEaIxD0frA/zAMc5hURbnqTrSZBjA= github.com/onflow/flow-nft/lib/go/contracts v1.1.1-0.20240305213046-9026973838d7 h1:0/s3DzQp3JHWuuz3DpGwniFShtgEy3wq1uBB+4uOG1I= github.com/onflow/flow-nft/lib/go/contracts v1.1.1-0.20240305213046-9026973838d7/go.mod h1:2gpbza+uzs1k7x31hkpBPlggIRkI53Suo0n2AyA2HcE= github.com/onflow/flow-nft/lib/go/templates v0.0.0-20240305213046-9026973838d7 h1:VU294Tk/Ra/UuuJbArLILA+x+4qX5qGPTY/kokNKyA4= diff --git a/model/flow/ledger_test.go b/model/flow/ledger_test.go index b287c3d3bb0..f188e1625cf 100644 --- a/model/flow/ledger_test.go +++ b/model/flow/ledger_test.go @@ -87,7 +87,7 @@ func TestRegisterID_IsInternalState(t *testing.T) { func TestRegisterID_String(t *testing.T) { t.Run("atree slab", func(t *testing.T) { // slab with 189 should result in \\xbd - slabIndex := atree.StorageIndex([8]byte{0, 0, 0, 0, 0, 0, 0, 189}) + slabIndex := atree.SlabIndex([8]byte{0, 0, 0, 0, 0, 0, 0, 189}) id := flow.NewRegisterID( flow.BytesToAddress([]byte{1, 2, 3, 10}), From 16b55430ecf69e1946e5b64ee339073f189e49a3 Mon Sep 17 00:00:00 2001 From: Faye Amacker <33205765+fxamacker@users.noreply.github.com> Date: Fri, 15 Mar 2024 18:20:36 -0500 Subject: [PATCH 02/60] Update to use new API in Cadence v1.0 --- cmd/util/ledger/migrations/cadence_value_diff.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/util/ledger/migrations/cadence_value_diff.go b/cmd/util/ledger/migrations/cadence_value_diff.go index 94e148920e3..c81195fc382 100644 --- a/cmd/util/ledger/migrations/cadence_value_diff.go +++ b/cmd/util/ledger/migrations/cadence_value_diff.go @@ -600,13 +600,13 @@ func (dr *CadenceValueDiffReporter) diffCadenceDictionaryValue( } oldKeys := make([]interpreter.Value, 0, v.Count()) - v.IterateKeys(vInterpreter, func(key interpreter.Value) (resume bool) { + v.IterateKeys(vInterpreter, interpreter.EmptyLocationRange, func(key interpreter.Value) (resume bool) { oldKeys = append(oldKeys, key) return true }) newKeys := make([]interpreter.Value, 0, otherDictionary.Count()) - otherDictionary.IterateKeys(otherInterpreter, func(key interpreter.Value) (resume bool) { + otherDictionary.IterateKeys(otherInterpreter, interpreter.EmptyLocationRange, func(key interpreter.Value) (resume bool) { newKeys = append(newKeys, key) return true }) From a3540c2668d56649aa5d33c418ea9490127b7eb4 Mon Sep 17 00:00:00 2001 From: Faye Amacker <33205765+fxamacker@users.noreply.github.com> Date: Fri, 15 Mar 2024 18:49:35 -0500 Subject: [PATCH 03/60] Update hashes in tests due to Atree format changes --- engine/execution/state/bootstrap/bootstrap_test.go | 2 +- fvm/accounts_test.go | 2 +- utils/unittest/execution_state.go | 6 +++--- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/engine/execution/state/bootstrap/bootstrap_test.go b/engine/execution/state/bootstrap/bootstrap_test.go index 64838a73661..ba3622e415f 100644 --- a/engine/execution/state/bootstrap/bootstrap_test.go +++ b/engine/execution/state/bootstrap/bootstrap_test.go @@ -53,7 +53,7 @@ func TestBootstrapLedger(t *testing.T) { } func TestBootstrapLedger_ZeroTokenSupply(t *testing.T) { - expectedStateCommitmentBytes, _ := hex.DecodeString("ef4d7cf4a852985e51bac9fefce8a04aa885fa716cc73814242ce55b51347e82") + expectedStateCommitmentBytes, _ := hex.DecodeString("0d34780e05327422d9a09a4ceae60ae012c1e93bb9a350cfbd204ccd709bc069") expectedStateCommitment, err := flow.ToStateCommitment(expectedStateCommitmentBytes) require.NoError(t, err) diff --git a/fvm/accounts_test.go b/fvm/accounts_test.go index 1568aec1378..56f398fb216 100644 --- a/fvm/accounts_test.go +++ b/fvm/accounts_test.go @@ -1544,7 +1544,7 @@ func TestAccountBalanceFields(t *testing.T) { _, output, err = vm.Run(ctx, script, snapshotTree) assert.NoError(t, err) assert.NoError(t, output.Err) - assert.Equal(t, cadence.UFix64(99_989_590), output.Value) + assert.Equal(t, cadence.UFix64(99_990_950), output.Value) }), ) diff --git a/utils/unittest/execution_state.go b/utils/unittest/execution_state.go index 22c25123403..a72f7e701bc 100644 --- a/utils/unittest/execution_state.go +++ b/utils/unittest/execution_state.go @@ -23,7 +23,7 @@ const ServiceAccountPrivateKeySignAlgo = crypto.ECDSAP256 const ServiceAccountPrivateKeyHashAlgo = hash.SHA2_256 // Pre-calculated state commitment with root account with the above private key -const GenesisStateCommitmentHex = "16dae2cbbf05691bad867549881dd01958350709df64e45481d8b235389c28ab" +const GenesisStateCommitmentHex = "55e4e3550b554518142f0f42ad271f51326264f5f904ea3236972a646e8cf164" var GenesisStateCommitment flow.StateCommitment @@ -87,10 +87,10 @@ func genesisCommitHexByChainID(chainID flow.ChainID) string { return GenesisStateCommitmentHex } if chainID == flow.Testnet { - return "bedd5fcc349f9a758e6854a42ab7481b465a69e6a51e29e7f4f6d980aade9ac2" + return "679d16fc24bf555d4dbad5ffa692186fb89819e9677af8abb92c3c33f506303e" } if chainID == flow.Sandboxnet { return "e1c08b17f9e5896f03fe28dd37ca396c19b26628161506924fbf785834646ea1" } - return "7148c32b2fa1f5c9baecb0823ea68d84fece1852c95ad9c0927a50f17fb0ff13" + return "c135dcdf0fe6ed8483212fb5f7ee14972a08328f7155e6125d538a9f046120da" } From 9c69b3bfb5c60c246fa639fdc77831d58070c4a6 Mon Sep 17 00:00:00 2001 From: Faye Amacker <33205765+fxamacker@users.noreply.github.com> Date: Thu, 11 Apr 2024 09:03:20 -0500 Subject: [PATCH 04/60] Bump Cadence v1.0 and Atree inlining version to latest --- go.mod | 4 ++-- go.sum | 8 ++++---- insecure/go.mod | 4 ++-- insecure/go.sum | 8 ++++---- integration/go.mod | 4 ++-- integration/go.sum | 8 ++++---- 6 files changed, 18 insertions(+), 18 deletions(-) diff --git a/go.mod b/go.mod index 8d8a9294bf9..befcde4d76a 100644 --- a/go.mod +++ b/go.mod @@ -50,8 +50,8 @@ require ( github.com/multiformats/go-multiaddr v0.12.2 github.com/multiformats/go-multiaddr-dns v0.3.1 github.com/multiformats/go-multihash v0.2.3 - github.com/onflow/atree v0.6.1-0.20240314011440-92714cac03fb - github.com/onflow/cadence v1.0.0-preview.14.0.20240315230827-7fc0649bf611 + github.com/onflow/atree v0.6.1-0.20240405171204-c6c951de7189 + github.com/onflow/cadence v1.0.0-preview.19.0.20240411132841-5de6f1c865b7 github.com/onflow/crypto v0.25.0 github.com/onflow/flow v0.3.4 github.com/onflow/flow-core-contracts/lib/go/contracts v0.15.2-0.20240305214031-d81e0c3b42f3 diff --git a/go.sum b/go.sum index 7ac085ac6d5..334554a00e9 100644 --- a/go.sum +++ b/go.sum @@ -2483,13 +2483,13 @@ github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5/go.mod h1:v github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec= github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY= github.com/onflow/atree v0.6.1-0.20230711151834-86040b30171f/go.mod h1:xvP61FoOs95K7IYdIYRnNcYQGf4nbF/uuJ0tHf4DRuM= -github.com/onflow/atree v0.6.1-0.20240314011440-92714cac03fb h1:9w+8wseSv7TZ9ikGfKSZ6yJifgYLhm717Zo0SVKpGEQ= -github.com/onflow/atree v0.6.1-0.20240314011440-92714cac03fb/go.mod h1:7YNAyCd5JENq+NzH+fR1ABUZVzbSq9dkt0+5fZH3L2A= +github.com/onflow/atree v0.6.1-0.20240405171204-c6c951de7189 h1:9PWA8BpJUrnSAGHXNNBvgzH3bv71fectqZokZw1++jE= +github.com/onflow/atree v0.6.1-0.20240405171204-c6c951de7189/go.mod h1:7YNAyCd5JENq+NzH+fR1ABUZVzbSq9dkt0+5fZH3L2A= github.com/onflow/boxo v0.0.0-20240201202436-f2477b92f483 h1:LpiQhTAfM9CAmNVEs0n//cBBgCg+vJSiIxTHYUklZ84= github.com/onflow/boxo v0.0.0-20240201202436-f2477b92f483/go.mod h1:pIZgTWdm3k3pLF9Uq6MB8JEcW07UDwNJjlXW1HELW80= github.com/onflow/cadence v1.0.0-M3/go.mod h1:odXGZZ/wGNA5mwT8bC9v8u8EXACHllB2ABSZK65TGL8= -github.com/onflow/cadence v1.0.0-preview.14.0.20240315230827-7fc0649bf611 h1:YR32C64UhtaZNeTR4tg1kCxxFdA8eBxk/ZyOilAUEas= -github.com/onflow/cadence v1.0.0-preview.14.0.20240315230827-7fc0649bf611/go.mod h1:5szKvYbjOrFTySYRDFu/HR6rdUH4yBjDDv2LoQsDUig= +github.com/onflow/cadence v1.0.0-preview.19.0.20240411132841-5de6f1c865b7 h1:bcjVFz6Pb6M4XZlu33ir8u8L+Us+ysPe5mQ4ad6mhR0= +github.com/onflow/cadence v1.0.0-preview.19.0.20240411132841-5de6f1c865b7/go.mod h1:lkWHLZU+X9b7vROtPqll4p4Pvz3rR1KZDTRbTei/F8w= github.com/onflow/crypto v0.25.0 h1:BeWbLsh3ZD13Ej+Uky6kg1PL1ZIVBDVX+2MVBNwqddg= github.com/onflow/crypto v0.25.0/go.mod h1:C8FbaX0x8y+FxWjbkHy0Q4EASCDR9bSPWZqlpCLYyVI= github.com/onflow/flow v0.3.4 h1:FXUWVdYB90f/rjNcY0Owo30gL790tiYff9Pb/sycXYE= diff --git a/insecure/go.mod b/insecure/go.mod index 0e5a2a4726d..9e13642f47f 100644 --- a/insecure/go.mod +++ b/insecure/go.mod @@ -204,8 +204,8 @@ require ( github.com/multiformats/go-multistream v0.5.0 // indirect github.com/multiformats/go-varint v0.0.7 // indirect github.com/olekukonko/tablewriter v0.0.5 // indirect - github.com/onflow/atree v0.6.1-0.20240314011440-92714cac03fb // indirect - github.com/onflow/cadence v1.0.0-preview.14.0.20240315230827-7fc0649bf611 // indirect + github.com/onflow/atree v0.6.1-0.20240405171204-c6c951de7189 // indirect + github.com/onflow/cadence v1.0.0-preview.19.0.20240411132841-5de6f1c865b7 // indirect github.com/onflow/flow-core-contracts/lib/go/contracts v0.15.2-0.20240305214031-d81e0c3b42f3 // indirect github.com/onflow/flow-core-contracts/lib/go/templates v0.15.2-0.20240305214031-d81e0c3b42f3 // indirect github.com/onflow/flow-ft/lib/go/contracts v0.7.1-0.20240305212555-29d91e18f0c1 // indirect diff --git a/insecure/go.sum b/insecure/go.sum index 9704d1ba0c5..ef5a1ea6a95 100644 --- a/insecure/go.sum +++ b/insecure/go.sum @@ -2470,11 +2470,11 @@ github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5/go.mod h1:v github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec= github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY= github.com/onflow/atree v0.6.1-0.20230711151834-86040b30171f/go.mod h1:xvP61FoOs95K7IYdIYRnNcYQGf4nbF/uuJ0tHf4DRuM= -github.com/onflow/atree v0.6.1-0.20240314011440-92714cac03fb h1:9w+8wseSv7TZ9ikGfKSZ6yJifgYLhm717Zo0SVKpGEQ= -github.com/onflow/atree v0.6.1-0.20240314011440-92714cac03fb/go.mod h1:7YNAyCd5JENq+NzH+fR1ABUZVzbSq9dkt0+5fZH3L2A= +github.com/onflow/atree v0.6.1-0.20240405171204-c6c951de7189 h1:9PWA8BpJUrnSAGHXNNBvgzH3bv71fectqZokZw1++jE= +github.com/onflow/atree v0.6.1-0.20240405171204-c6c951de7189/go.mod h1:7YNAyCd5JENq+NzH+fR1ABUZVzbSq9dkt0+5fZH3L2A= github.com/onflow/cadence v1.0.0-M3/go.mod h1:odXGZZ/wGNA5mwT8bC9v8u8EXACHllB2ABSZK65TGL8= -github.com/onflow/cadence v1.0.0-preview.14.0.20240315230827-7fc0649bf611 h1:YR32C64UhtaZNeTR4tg1kCxxFdA8eBxk/ZyOilAUEas= -github.com/onflow/cadence v1.0.0-preview.14.0.20240315230827-7fc0649bf611/go.mod h1:5szKvYbjOrFTySYRDFu/HR6rdUH4yBjDDv2LoQsDUig= +github.com/onflow/cadence v1.0.0-preview.19.0.20240411132841-5de6f1c865b7 h1:bcjVFz6Pb6M4XZlu33ir8u8L+Us+ysPe5mQ4ad6mhR0= +github.com/onflow/cadence v1.0.0-preview.19.0.20240411132841-5de6f1c865b7/go.mod h1:lkWHLZU+X9b7vROtPqll4p4Pvz3rR1KZDTRbTei/F8w= github.com/onflow/crypto v0.25.0 h1:BeWbLsh3ZD13Ej+Uky6kg1PL1ZIVBDVX+2MVBNwqddg= github.com/onflow/crypto v0.25.0/go.mod h1:C8FbaX0x8y+FxWjbkHy0Q4EASCDR9bSPWZqlpCLYyVI= github.com/onflow/flow-core-contracts/lib/go/contracts v0.15.2-0.20240305214031-d81e0c3b42f3 h1:UNLms46HthnzKTR3DVQmEoDerf4EtNUMfkombIeT3U8= diff --git a/integration/go.mod b/integration/go.mod index 674980fa7d6..546d5c8ea39 100644 --- a/integration/go.mod +++ b/integration/go.mod @@ -21,7 +21,7 @@ require ( github.com/ipfs/go-ds-badger2 v0.1.3 github.com/ipfs/go-ipfs-blockstore v1.3.0 github.com/libp2p/go-libp2p v0.32.2 - github.com/onflow/cadence v1.0.0-preview.14.0.20240315230827-7fc0649bf611 + github.com/onflow/cadence v1.0.0-preview.19.0.20240411132841-5de6f1c865b7 github.com/onflow/crypto v0.25.0 github.com/onflow/flow-core-contracts/lib/go/contracts v0.15.2-0.20240305214031-d81e0c3b42f3 github.com/onflow/flow-core-contracts/lib/go/templates v0.15.2-0.20240305214031-d81e0c3b42f3 @@ -251,7 +251,7 @@ require ( github.com/multiformats/go-multistream v0.5.0 // indirect github.com/multiformats/go-varint v0.0.7 // indirect github.com/olekukonko/tablewriter v0.0.5 // indirect - github.com/onflow/atree v0.6.1-0.20240314011440-92714cac03fb // indirect + github.com/onflow/atree v0.6.1-0.20240405171204-c6c951de7189 // indirect github.com/onflow/flow-ft/lib/go/contracts v0.7.1-0.20240305212555-29d91e18f0c1 // indirect github.com/onflow/flow-ft/lib/go/templates v0.7.1-0.20240305212555-29d91e18f0c1 // indirect github.com/onflow/flow-nft/lib/go/contracts v1.1.1-0.20240305213046-9026973838d7 // indirect diff --git a/integration/go.sum b/integration/go.sum index 873c3e617cd..c1fe86c0850 100644 --- a/integration/go.sum +++ b/integration/go.sum @@ -2546,11 +2546,11 @@ github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5/go.mod h1:v github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec= github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY= github.com/onflow/atree v0.6.1-0.20230711151834-86040b30171f/go.mod h1:xvP61FoOs95K7IYdIYRnNcYQGf4nbF/uuJ0tHf4DRuM= -github.com/onflow/atree v0.6.1-0.20240314011440-92714cac03fb h1:9w+8wseSv7TZ9ikGfKSZ6yJifgYLhm717Zo0SVKpGEQ= -github.com/onflow/atree v0.6.1-0.20240314011440-92714cac03fb/go.mod h1:7YNAyCd5JENq+NzH+fR1ABUZVzbSq9dkt0+5fZH3L2A= +github.com/onflow/atree v0.6.1-0.20240405171204-c6c951de7189 h1:9PWA8BpJUrnSAGHXNNBvgzH3bv71fectqZokZw1++jE= +github.com/onflow/atree v0.6.1-0.20240405171204-c6c951de7189/go.mod h1:7YNAyCd5JENq+NzH+fR1ABUZVzbSq9dkt0+5fZH3L2A= github.com/onflow/cadence v1.0.0-M3/go.mod h1:odXGZZ/wGNA5mwT8bC9v8u8EXACHllB2ABSZK65TGL8= -github.com/onflow/cadence v1.0.0-preview.14.0.20240315230827-7fc0649bf611 h1:YR32C64UhtaZNeTR4tg1kCxxFdA8eBxk/ZyOilAUEas= -github.com/onflow/cadence v1.0.0-preview.14.0.20240315230827-7fc0649bf611/go.mod h1:5szKvYbjOrFTySYRDFu/HR6rdUH4yBjDDv2LoQsDUig= +github.com/onflow/cadence v1.0.0-preview.19.0.20240411132841-5de6f1c865b7 h1:bcjVFz6Pb6M4XZlu33ir8u8L+Us+ysPe5mQ4ad6mhR0= +github.com/onflow/cadence v1.0.0-preview.19.0.20240411132841-5de6f1c865b7/go.mod h1:lkWHLZU+X9b7vROtPqll4p4Pvz3rR1KZDTRbTei/F8w= github.com/onflow/crypto v0.25.0 h1:BeWbLsh3ZD13Ej+Uky6kg1PL1ZIVBDVX+2MVBNwqddg= github.com/onflow/crypto v0.25.0/go.mod h1:C8FbaX0x8y+FxWjbkHy0Q4EASCDR9bSPWZqlpCLYyVI= github.com/onflow/flow-core-contracts/lib/go/contracts v0.15.2-0.20240305214031-d81e0c3b42f3 h1:UNLms46HthnzKTR3DVQmEoDerf4EtNUMfkombIeT3U8= From dd9d6643e5fabab585cde4131e07fbdf583e4821 Mon Sep 17 00:00:00 2001 From: Faye Amacker <33205765+fxamacker@users.noreply.github.com> Date: Tue, 23 Apr 2024 19:59:39 -0500 Subject: [PATCH 05/60] Fix test to be compatible with atree inlining --- cmd/util/ledger/migrations/fix_broken_data_migration_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/util/ledger/migrations/fix_broken_data_migration_test.go b/cmd/util/ledger/migrations/fix_broken_data_migration_test.go index 8fd5a74eaef..ed627b5e611 100644 --- a/cmd/util/ledger/migrations/fix_broken_data_migration_test.go +++ b/cmd/util/ledger/migrations/fix_broken_data_migration_test.go @@ -104,14 +104,14 @@ func TestFixSlabsWithBrokenReferences(t *testing.T) { slabIndexWithBrokenReferences := mustDecodeHex("240000000000000003") fixedSlabWithBrokenReferences := ledger.NewPayload( ledger.NewKey([]ledger.KeyPart{ownerKey, {Type: 2, Value: slabIndexWithBrokenReferences}}), - ledger.Value(mustDecodeHex("008883d8d982d8d582d8c0824848602d8056ff9d937046616e546f705065726d697373696f6e7546616e546f705065726d697373696f6e2e526f6c65d8ddf6001b535c9de83a38cab0008883005b00000000000000009b0000000000000000")), + ledger.Value(mustDecodeHex("108883d8d982d8d582d8c0824848602d8056ff9d937046616e546f705065726d697373696f6e7546616e546f705065726d697373696f6e2e526f6c65d8ddf6001b535c9de83a38cab08300590000990000")), ) // Account status register is updated to include address ID counter and new storage used. accountStatusRegisterID := mustDecodeHex("612e73") updatedAccountStatusRegister := ledger.NewPayload( ledger.NewKey([]ledger.KeyPart{ownerKey, {Type: 2, Value: accountStatusRegisterID}}), - ledger.Value([]byte{0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x7, 0xcc, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x9, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}), + ledger.Value([]byte{0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x7, 0xbe, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x9, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}), ) expectedNewPayloads := make([]*ledger.Payload, len(oldPayloads)) From d14848bcaacb6cdadf267630739d08d721407332 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastian=20M=C3=BCller?= Date: Thu, 25 Apr 2024 11:33:27 -0700 Subject: [PATCH 06/60] adjust state commitments --- engine/execution/state/bootstrap/bootstrap_test.go | 2 +- utils/unittest/execution_state.go | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/engine/execution/state/bootstrap/bootstrap_test.go b/engine/execution/state/bootstrap/bootstrap_test.go index a27e9eae96e..5bc51800f18 100644 --- a/engine/execution/state/bootstrap/bootstrap_test.go +++ b/engine/execution/state/bootstrap/bootstrap_test.go @@ -53,7 +53,7 @@ func TestBootstrapLedger(t *testing.T) { } func TestBootstrapLedger_ZeroTokenSupply(t *testing.T) { - expectedStateCommitmentBytes, _ := hex.DecodeString("df32a75f2c69e477775ae223781e95345c0e9fa680359e1ae66d5550cfb562d0") + expectedStateCommitmentBytes, _ := hex.DecodeString("786861df06d2bd079c62321deb7197b9c330a5da8eac252d1ce796b0c70511ee") expectedStateCommitment, err := flow.ToStateCommitment(expectedStateCommitmentBytes) require.NoError(t, err) diff --git a/utils/unittest/execution_state.go b/utils/unittest/execution_state.go index 8618c1e5767..eaae60bad39 100644 --- a/utils/unittest/execution_state.go +++ b/utils/unittest/execution_state.go @@ -23,7 +23,7 @@ const ServiceAccountPrivateKeySignAlgo = crypto.ECDSAP256 const ServiceAccountPrivateKeyHashAlgo = hash.SHA2_256 // Pre-calculated state commitment with root account with the above private key -const GenesisStateCommitmentHex = "7195f874909810af208f3248da49180ccfd69e748586d83b0a1fbda100473039" +const GenesisStateCommitmentHex = "d8979030806005862e7c1c268466d56bcf1f37046f587bc3968e03868ae58df5" var GenesisStateCommitment flow.StateCommitment @@ -87,10 +87,10 @@ func genesisCommitHexByChainID(chainID flow.ChainID) string { return GenesisStateCommitmentHex } if chainID == flow.Testnet { - return "eb9de5b5a3d342a7de42912d6edd3e28ed51101e961b0993d78145a577a60b4f" + return "0e8ec5fe399042ac2ae08e53459534969eaf437f27d01dc0c69230d01766fd1e" } if chainID == flow.Sandboxnet { return "e1c08b17f9e5896f03fe28dd37ca396c19b26628161506924fbf785834646ea1" } - return "25854b600d6f84010acb5877842375605340cfc1ba7be9ba7f5830acb5c1b19e" + return "32f9dc0919a79f4ed2b4a1b68eae00d4fa55c0e37dd9bbf2406b26511a7a5795" } From 32f376f8a4adfea2da6a274f21ac441d8f240069 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastian=20M=C3=BCller?= Date: Wed, 1 May 2024 14:13:53 -0700 Subject: [PATCH 07/60] adjust state commitments --- engine/execution/state/bootstrap/bootstrap_test.go | 2 +- utils/unittest/execution_state.go | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/engine/execution/state/bootstrap/bootstrap_test.go b/engine/execution/state/bootstrap/bootstrap_test.go index 7f1462fd25b..9effd429445 100644 --- a/engine/execution/state/bootstrap/bootstrap_test.go +++ b/engine/execution/state/bootstrap/bootstrap_test.go @@ -53,7 +53,7 @@ func TestBootstrapLedger(t *testing.T) { } func TestBootstrapLedger_ZeroTokenSupply(t *testing.T) { - expectedStateCommitmentBytes, _ := hex.DecodeString("2450d20aeb0b5219e9c2238437448f7095110ebc73b82bd1314b9b9d48041730") + expectedStateCommitmentBytes, _ := hex.DecodeString("2d82a1ab6bc31924d0e98485db01fb6887bf07875b6e816226e95adb39f28598") expectedStateCommitment, err := flow.ToStateCommitment(expectedStateCommitmentBytes) require.NoError(t, err) diff --git a/utils/unittest/execution_state.go b/utils/unittest/execution_state.go index 09d51d4934d..372cf0576d1 100644 --- a/utils/unittest/execution_state.go +++ b/utils/unittest/execution_state.go @@ -23,7 +23,7 @@ const ServiceAccountPrivateKeySignAlgo = crypto.ECDSAP256 const ServiceAccountPrivateKeyHashAlgo = hash.SHA2_256 // Pre-calculated state commitment with root account with the above private key -const GenesisStateCommitmentHex = "e56d08e1c4e80dbc355b86a751feb5f81d9db4cb71c76e49156fa4fa2c257ac8" +const GenesisStateCommitmentHex = "7c6bb57dfc565ee4653f67c0c075eb38a1b91fdb24f98041f6dec9eb7675014c" var GenesisStateCommitment flow.StateCommitment @@ -87,10 +87,10 @@ func genesisCommitHexByChainID(chainID flow.ChainID) string { return GenesisStateCommitmentHex } if chainID == flow.Testnet { - return "3c995d368b5edbf513e251b502ab9f08bfc64f4128f43877820d6e1b3f550e32" + return "ffe73c20388feff0e3833f6927d3c2856a63e0c423b3771508f2a4ecfbf97139" } if chainID == flow.Sandboxnet { return "e1c08b17f9e5896f03fe28dd37ca396c19b26628161506924fbf785834646ea1" } - return "5ae62f5555ea7e3fd14bd3f4d2eabb8d65370f5c6a8b0213f23bf572d892c96b" + return "c76d796b91d92c3640d43fef9e04e85bcff0d6a273a2c531f0bc9931957f1716" } From bd92fce4eaa63d070bbb48016a73de33a82969be Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastian=20M=C3=BCller?= Date: Wed, 1 May 2024 14:27:11 -0700 Subject: [PATCH 08/60] use Cadence's predicate function to determine if given broken value should be fixed --- cmd/util/ledger/migrations/fix_broken_data_migration.go | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/cmd/util/ledger/migrations/fix_broken_data_migration.go b/cmd/util/ledger/migrations/fix_broken_data_migration.go index 86fd0641af8..17dcbdf22fa 100644 --- a/cmd/util/ledger/migrations/fix_broken_data_migration.go +++ b/cmd/util/ledger/migrations/fix_broken_data_migration.go @@ -7,6 +7,7 @@ import ( "sync" "time" + "github.com/onflow/cadence/migrations" "github.com/rs/zerolog" "github.com/onflow/atree" @@ -89,10 +90,8 @@ func (m *FixSlabsWithBrokenReferencesMigration) MigrateAccount( } // Fix broken references - fixedStorageIDs, skippedStorageIDs, err := storage.FixLoadedBrokenReferences(func(old atree.Value) bool { - // TODO: Cadence may need to export functions to check type info, etc. - return true - }) + fixedStorageIDs, skippedStorageIDs, err := + storage.FixLoadedBrokenReferences(migrations.ShouldFixBrokenCompositeKeyedDictionary) if err != nil { return nil, err } From bc98c829be3762d34ef6cc1088c5f75081e074f2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastian=20M=C3=BCller?= Date: Tue, 7 May 2024 16:50:50 -0700 Subject: [PATCH 09/60] adjust state commitments --- engine/execution/state/bootstrap/bootstrap_test.go | 2 +- utils/unittest/execution_state.go | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/engine/execution/state/bootstrap/bootstrap_test.go b/engine/execution/state/bootstrap/bootstrap_test.go index 9effd429445..45b3e4119b4 100644 --- a/engine/execution/state/bootstrap/bootstrap_test.go +++ b/engine/execution/state/bootstrap/bootstrap_test.go @@ -53,7 +53,7 @@ func TestBootstrapLedger(t *testing.T) { } func TestBootstrapLedger_ZeroTokenSupply(t *testing.T) { - expectedStateCommitmentBytes, _ := hex.DecodeString("2d82a1ab6bc31924d0e98485db01fb6887bf07875b6e816226e95adb39f28598") + expectedStateCommitmentBytes, _ := hex.DecodeString("82b80015ac01a065f22b18ebfb739c492bacfddde79a584d19658dcfd7ff4673") expectedStateCommitment, err := flow.ToStateCommitment(expectedStateCommitmentBytes) require.NoError(t, err) diff --git a/utils/unittest/execution_state.go b/utils/unittest/execution_state.go index 372cf0576d1..6f064d05bd7 100644 --- a/utils/unittest/execution_state.go +++ b/utils/unittest/execution_state.go @@ -23,7 +23,7 @@ const ServiceAccountPrivateKeySignAlgo = crypto.ECDSAP256 const ServiceAccountPrivateKeyHashAlgo = hash.SHA2_256 // Pre-calculated state commitment with root account with the above private key -const GenesisStateCommitmentHex = "7c6bb57dfc565ee4653f67c0c075eb38a1b91fdb24f98041f6dec9eb7675014c" +const GenesisStateCommitmentHex = "89ee0925606973a401f37ca21dddf15c95121d4c2e4fb9ebeb7689ef38377e49" var GenesisStateCommitment flow.StateCommitment @@ -87,10 +87,10 @@ func genesisCommitHexByChainID(chainID flow.ChainID) string { return GenesisStateCommitmentHex } if chainID == flow.Testnet { - return "ffe73c20388feff0e3833f6927d3c2856a63e0c423b3771508f2a4ecfbf97139" + return "b5e7fae0db47a5ab087442e1c18942225bce7eab128b294baf8c5849f2d49276" } if chainID == flow.Sandboxnet { return "e1c08b17f9e5896f03fe28dd37ca396c19b26628161506924fbf785834646ea1" } - return "c76d796b91d92c3640d43fef9e04e85bcff0d6a273a2c531f0bc9931957f1716" + return "6b962792fa1f2ec6ed6b7ddee45f70bb0811b197f6bedef01be2f5c248c5a87a" } From ecaa5cda1b41592004b6d3f6ad3d6b153d85554e Mon Sep 17 00:00:00 2001 From: Peter Argue <89119817+peterargue@users.noreply.github.com> Date: Wed, 8 May 2024 12:22:27 -0700 Subject: [PATCH 10/60] [CI] Fix stale issue workflow --- .github/workflows/stale.yml | 36 +++++++++++++++++++----------------- 1 file changed, 19 insertions(+), 17 deletions(-) diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml index a446dc86c50..8b3ad2cbe13 100644 --- a/.github/workflows/stale.yml +++ b/.github/workflows/stale.yml @@ -1,17 +1,19 @@ -# Number of days of inactivity before an issue becomes stale -daysUntilStale: 90 -# Number of days of inactivity before a stale issue is closed -daysUntilClose: 7 -# Issues with these labels will never be considered stale -exemptLabels: - - Preserve - - Idea -# Label to use when marking an issue as stale -staleLabel: Stale -# Comment to post when marking an issue as stale. Set to `false` to disable -markComment: > - This issue has been automatically marked as stale because it has not had - recent activity. It will be closed if no further activity occurs. Thank you - for your contributions. -# Comment to post when closing a stale issue. Set to `false` to disable -closeComment: false +name: 'Mark and close stale issues' +on: + schedule: + - cron: '30 1 * * *' + +jobs: + stale: + runs-on: ubuntu-latest + steps: + - uses: actions/stale@v9 + with: + days-before-issue-stale: 90 + days-before-issue-close: 7 + exempt-issue-labels: 'Preserve,Idea' + stale-issue-label: 'Stale' + stale-issue-message: > + This issue has been automatically marked as stale because it has not had + recent activity. It will be closed if no further activity occurs. Thank you + for your contributions. From b621cadf15e287502df75c310e834ab1fb64596e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastian=20M=C3=BCller?= Date: Fri, 17 May 2024 17:35:53 -0700 Subject: [PATCH 11/60] adjust expected state commitments --- engine/execution/state/bootstrap/bootstrap_test.go | 2 +- utils/unittest/execution_state.go | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/engine/execution/state/bootstrap/bootstrap_test.go b/engine/execution/state/bootstrap/bootstrap_test.go index ba25d1e7d89..b230825706e 100644 --- a/engine/execution/state/bootstrap/bootstrap_test.go +++ b/engine/execution/state/bootstrap/bootstrap_test.go @@ -53,7 +53,7 @@ func TestBootstrapLedger(t *testing.T) { } func TestBootstrapLedger_ZeroTokenSupply(t *testing.T) { - expectedStateCommitmentBytes, _ := hex.DecodeString("3f0f1fea08e7e289c2b94f019f5921382bfba87dc1b52cf1fe8f004ed8d70b1b") + expectedStateCommitmentBytes, _ := hex.DecodeString("18f2d2bc725080af617fd0793987b35c40a0c68c886e5917b82ae44d51ea4a36") expectedStateCommitment, err := flow.ToStateCommitment(expectedStateCommitmentBytes) require.NoError(t, err) diff --git a/utils/unittest/execution_state.go b/utils/unittest/execution_state.go index c7297e0d2ee..2de06da7d66 100644 --- a/utils/unittest/execution_state.go +++ b/utils/unittest/execution_state.go @@ -23,7 +23,7 @@ const ServiceAccountPrivateKeySignAlgo = crypto.ECDSAP256 const ServiceAccountPrivateKeyHashAlgo = hash.SHA2_256 // Pre-calculated state commitment with root account with the above private key -const GenesisStateCommitmentHex = "1f001c1c7fc9a9e125fae9175ae040d0aa91741b9c8d5f42f2bec91f93ef63c9" +const GenesisStateCommitmentHex = "62a413fddb95adec3f57550816004322d70147fdc492a0d1a6b40d87202cf1e9" var GenesisStateCommitment flow.StateCommitment @@ -87,10 +87,10 @@ func genesisCommitHexByChainID(chainID flow.ChainID) string { return GenesisStateCommitmentHex } if chainID == flow.Testnet { - return "a01e2ae8c955c15bbf4e107b93e7a7edf9d56acc607063dfd338fb7efae29dcc" + return "dbe45308cd8230d5fecc4c92f986d77938fcf5eb9d9454b2b96e02355ec8e2dc" } if chainID == flow.Sandboxnet { return "e1c08b17f9e5896f03fe28dd37ca396c19b26628161506924fbf785834646ea1" } - return "aa285104f02969b3ad3798e88472f211d36a46e6f66191b827f4bc9560697938" + return "ce1e4238dea47fc393b41e05fa0798de8b20fd64816f89f1d233cdecec7c467b" } From f6d8d91e5aeefcc4a80bf7bc968787c63bcce5f8 Mon Sep 17 00:00:00 2001 From: Faye Amacker <33205765+fxamacker@users.noreply.github.com> Date: Tue, 21 May 2024 18:06:47 -0500 Subject: [PATCH 12/60] Bump Cadence version to latest with atree v0.8.0-rc.3 Update flow-go to use latest commit in Cadence feature branch feature/atree-register-inlining-v1.0. --- go.mod | 4 ++-- go.sum | 8 ++++---- insecure/go.mod | 4 ++-- insecure/go.sum | 8 ++++---- integration/go.mod | 4 ++-- integration/go.sum | 8 ++++---- 6 files changed, 18 insertions(+), 18 deletions(-) diff --git a/go.mod b/go.mod index 9c055cecf3c..6892f109868 100644 --- a/go.mod +++ b/go.mod @@ -46,8 +46,8 @@ require ( github.com/multiformats/go-multiaddr v0.12.2 github.com/multiformats/go-multiaddr-dns v0.3.1 github.com/multiformats/go-multihash v0.2.3 - github.com/onflow/atree v0.8.0-rc.2 - github.com/onflow/cadence v1.0.0-preview-atree-register-inlining.29 + github.com/onflow/atree v0.8.0-rc.3 + github.com/onflow/cadence v1.0.0-preview-atree-register-inlining.29.0.20240521223412-72e083b3cf80 github.com/onflow/crypto v0.25.1 github.com/onflow/flow v0.3.4 github.com/onflow/flow-core-contracts/lib/go/contracts v1.1.0 diff --git a/go.sum b/go.sum index da85f3d00cf..141e8ad7ed4 100644 --- a/go.sum +++ b/go.sum @@ -2152,13 +2152,13 @@ github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5/go.mod h1:v github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec= github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY= github.com/onflow/atree v0.6.1-0.20230711151834-86040b30171f/go.mod h1:xvP61FoOs95K7IYdIYRnNcYQGf4nbF/uuJ0tHf4DRuM= -github.com/onflow/atree v0.8.0-rc.2 h1:7XYaOiiYJqLadzmyLyju2ztoqyTw/ikzcI0HI2LA+bI= -github.com/onflow/atree v0.8.0-rc.2/go.mod h1:7YNAyCd5JENq+NzH+fR1ABUZVzbSq9dkt0+5fZH3L2A= +github.com/onflow/atree v0.8.0-rc.3 h1:BHVkJLrBHhHo7ET8gkuS1+lyQGNekYYOyoICGK3RFNM= +github.com/onflow/atree v0.8.0-rc.3/go.mod h1:7YNAyCd5JENq+NzH+fR1ABUZVzbSq9dkt0+5fZH3L2A= github.com/onflow/boxo v0.0.0-20240201202436-f2477b92f483 h1:LpiQhTAfM9CAmNVEs0n//cBBgCg+vJSiIxTHYUklZ84= github.com/onflow/boxo v0.0.0-20240201202436-f2477b92f483/go.mod h1:pIZgTWdm3k3pLF9Uq6MB8JEcW07UDwNJjlXW1HELW80= github.com/onflow/cadence v1.0.0-M3/go.mod h1:odXGZZ/wGNA5mwT8bC9v8u8EXACHllB2ABSZK65TGL8= -github.com/onflow/cadence v1.0.0-preview-atree-register-inlining.29 h1:qLjWwmFh3HYHogVwccw6ZViPiLAKjo7T7IpHj3Xvco0= -github.com/onflow/cadence v1.0.0-preview-atree-register-inlining.29/go.mod h1:KclJlSGWG4USgPK4CsI3V/YtCHYOwPpjyzb6iEfWlbM= +github.com/onflow/cadence v1.0.0-preview-atree-register-inlining.29.0.20240521223412-72e083b3cf80 h1:Jqj/5POMSXfzJh0wPFyGimxQRtuCcFx0BWC/xoivvqg= +github.com/onflow/cadence v1.0.0-preview-atree-register-inlining.29.0.20240521223412-72e083b3cf80/go.mod h1:ulmbwFrXnKgqRpY18L1J5irMOerclScfEfAURx2uHFw= github.com/onflow/crypto v0.25.0/go.mod h1:C8FbaX0x8y+FxWjbkHy0Q4EASCDR9bSPWZqlpCLYyVI= github.com/onflow/crypto v0.25.1 h1:0txy2PKPMM873JbpxQNbJmuOJtD56bfs48RQfm0ts5A= github.com/onflow/crypto v0.25.1/go.mod h1:C8FbaX0x8y+FxWjbkHy0Q4EASCDR9bSPWZqlpCLYyVI= diff --git a/insecure/go.mod b/insecure/go.mod index 47c79a0bfad..41819d07b5a 100644 --- a/insecure/go.mod +++ b/insecure/go.mod @@ -197,8 +197,8 @@ require ( github.com/multiformats/go-multistream v0.5.0 // indirect github.com/multiformats/go-varint v0.0.7 // indirect github.com/olekukonko/tablewriter v0.0.5 // indirect - github.com/onflow/atree v0.8.0-rc.2 // indirect - github.com/onflow/cadence v1.0.0-preview-atree-register-inlining.29 // indirect + github.com/onflow/atree v0.8.0-rc.3 // indirect + github.com/onflow/cadence v1.0.0-preview-atree-register-inlining.29.0.20240521223412-72e083b3cf80 // indirect github.com/onflow/flow-core-contracts/lib/go/contracts v1.1.0 // indirect github.com/onflow/flow-core-contracts/lib/go/templates v1.0.0 // indirect github.com/onflow/flow-ft/lib/go/contracts v1.0.0 // indirect diff --git a/insecure/go.sum b/insecure/go.sum index b6f8cc5864f..7a93b36629d 100644 --- a/insecure/go.sum +++ b/insecure/go.sum @@ -2140,11 +2140,11 @@ github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5/go.mod h1:v github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec= github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY= github.com/onflow/atree v0.6.1-0.20230711151834-86040b30171f/go.mod h1:xvP61FoOs95K7IYdIYRnNcYQGf4nbF/uuJ0tHf4DRuM= -github.com/onflow/atree v0.8.0-rc.2 h1:7XYaOiiYJqLadzmyLyju2ztoqyTw/ikzcI0HI2LA+bI= -github.com/onflow/atree v0.8.0-rc.2/go.mod h1:7YNAyCd5JENq+NzH+fR1ABUZVzbSq9dkt0+5fZH3L2A= +github.com/onflow/atree v0.8.0-rc.3 h1:BHVkJLrBHhHo7ET8gkuS1+lyQGNekYYOyoICGK3RFNM= +github.com/onflow/atree v0.8.0-rc.3/go.mod h1:7YNAyCd5JENq+NzH+fR1ABUZVzbSq9dkt0+5fZH3L2A= github.com/onflow/cadence v1.0.0-M3/go.mod h1:odXGZZ/wGNA5mwT8bC9v8u8EXACHllB2ABSZK65TGL8= -github.com/onflow/cadence v1.0.0-preview-atree-register-inlining.29 h1:qLjWwmFh3HYHogVwccw6ZViPiLAKjo7T7IpHj3Xvco0= -github.com/onflow/cadence v1.0.0-preview-atree-register-inlining.29/go.mod h1:KclJlSGWG4USgPK4CsI3V/YtCHYOwPpjyzb6iEfWlbM= +github.com/onflow/cadence v1.0.0-preview-atree-register-inlining.29.0.20240521223412-72e083b3cf80 h1:Jqj/5POMSXfzJh0wPFyGimxQRtuCcFx0BWC/xoivvqg= +github.com/onflow/cadence v1.0.0-preview-atree-register-inlining.29.0.20240521223412-72e083b3cf80/go.mod h1:ulmbwFrXnKgqRpY18L1J5irMOerclScfEfAURx2uHFw= github.com/onflow/crypto v0.25.0/go.mod h1:C8FbaX0x8y+FxWjbkHy0Q4EASCDR9bSPWZqlpCLYyVI= github.com/onflow/crypto v0.25.1 h1:0txy2PKPMM873JbpxQNbJmuOJtD56bfs48RQfm0ts5A= github.com/onflow/crypto v0.25.1/go.mod h1:C8FbaX0x8y+FxWjbkHy0Q4EASCDR9bSPWZqlpCLYyVI= diff --git a/integration/go.mod b/integration/go.mod index 788855e9f97..b94fcd227b3 100644 --- a/integration/go.mod +++ b/integration/go.mod @@ -19,7 +19,7 @@ require ( github.com/ipfs/go-datastore v0.6.0 github.com/ipfs/go-ds-badger2 v0.1.3 github.com/libp2p/go-libp2p v0.32.2 - github.com/onflow/cadence v1.0.0-preview-atree-register-inlining.29 + github.com/onflow/cadence v1.0.0-preview-atree-register-inlining.29.0.20240521223412-72e083b3cf80 github.com/onflow/crypto v0.25.1 github.com/onflow/flow-core-contracts/lib/go/contracts v1.1.0 github.com/onflow/flow-core-contracts/lib/go/templates v1.0.0 @@ -241,7 +241,7 @@ require ( github.com/multiformats/go-multistream v0.5.0 // indirect github.com/multiformats/go-varint v0.0.7 // indirect github.com/olekukonko/tablewriter v0.0.5 // indirect - github.com/onflow/atree v0.8.0-rc.2 // indirect + github.com/onflow/atree v0.8.0-rc.3 // indirect github.com/onflow/flow-ft/lib/go/contracts v1.0.0 // indirect github.com/onflow/flow-ft/lib/go/templates v1.0.0 // indirect github.com/onflow/flow-nft/lib/go/contracts v1.2.1 // indirect diff --git a/integration/go.sum b/integration/go.sum index ad470cc69e8..07c339d2850 100644 --- a/integration/go.sum +++ b/integration/go.sum @@ -2134,11 +2134,11 @@ github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec= github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY= github.com/onflow/atree v0.6.1-0.20230711151834-86040b30171f/go.mod h1:xvP61FoOs95K7IYdIYRnNcYQGf4nbF/uuJ0tHf4DRuM= -github.com/onflow/atree v0.8.0-rc.2 h1:7XYaOiiYJqLadzmyLyju2ztoqyTw/ikzcI0HI2LA+bI= -github.com/onflow/atree v0.8.0-rc.2/go.mod h1:7YNAyCd5JENq+NzH+fR1ABUZVzbSq9dkt0+5fZH3L2A= +github.com/onflow/atree v0.8.0-rc.3 h1:BHVkJLrBHhHo7ET8gkuS1+lyQGNekYYOyoICGK3RFNM= +github.com/onflow/atree v0.8.0-rc.3/go.mod h1:7YNAyCd5JENq+NzH+fR1ABUZVzbSq9dkt0+5fZH3L2A= github.com/onflow/cadence v1.0.0-M3/go.mod h1:odXGZZ/wGNA5mwT8bC9v8u8EXACHllB2ABSZK65TGL8= -github.com/onflow/cadence v1.0.0-preview-atree-register-inlining.29 h1:qLjWwmFh3HYHogVwccw6ZViPiLAKjo7T7IpHj3Xvco0= -github.com/onflow/cadence v1.0.0-preview-atree-register-inlining.29/go.mod h1:KclJlSGWG4USgPK4CsI3V/YtCHYOwPpjyzb6iEfWlbM= +github.com/onflow/cadence v1.0.0-preview-atree-register-inlining.29.0.20240521223412-72e083b3cf80 h1:Jqj/5POMSXfzJh0wPFyGimxQRtuCcFx0BWC/xoivvqg= +github.com/onflow/cadence v1.0.0-preview-atree-register-inlining.29.0.20240521223412-72e083b3cf80/go.mod h1:ulmbwFrXnKgqRpY18L1J5irMOerclScfEfAURx2uHFw= github.com/onflow/crypto v0.25.0/go.mod h1:C8FbaX0x8y+FxWjbkHy0Q4EASCDR9bSPWZqlpCLYyVI= github.com/onflow/crypto v0.25.1 h1:0txy2PKPMM873JbpxQNbJmuOJtD56bfs48RQfm0ts5A= github.com/onflow/crypto v0.25.1/go.mod h1:C8FbaX0x8y+FxWjbkHy0Q4EASCDR9bSPWZqlpCLYyVI= From 9349bd1f68e4831132780f4b81b1820d496e4730 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastian=20M=C3=BCller?= Date: Wed, 29 May 2024 11:41:48 -0700 Subject: [PATCH 13/60] temporarily adjust expected numbers --- .../ledger/migrations/migration_matrics_collector_test.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/cmd/util/ledger/migrations/migration_matrics_collector_test.go b/cmd/util/ledger/migrations/migration_matrics_collector_test.go index f4db4a56ca6..747ee804d8d 100644 --- a/cmd/util/ledger/migrations/migration_matrics_collector_test.go +++ b/cmd/util/ledger/migrations/migration_matrics_collector_test.go @@ -80,7 +80,8 @@ func TestMigrationMetricsCollection(t *testing.T) { require.Equal( t, Metrics{ - TotalValues: 752, + // TODO: should be 752, like on master + TotalValues: 750, TotalErrors: 6, ErrorsPerContract: map[string]int{ "A.01cf0e2f2f715450.Test": 6, @@ -172,7 +173,8 @@ func TestMigrationMetricsCollection(t *testing.T) { require.Equal( t, Metrics{ - TotalValues: 752, + // TODO: should be 752, like on master + TotalValues: 750, TotalErrors: 6, ErrorsPerContract: map[string]int{ "A.01cf0e2f2f715450.Test": 6, From c6c04090b96a43b5bfd21925721d096af626def6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastian=20M=C3=BCller?= Date: Tue, 4 Jun 2024 11:05:16 -0700 Subject: [PATCH 14/60] adjust numbers issue got resolved by https://github.com/onflow/cadence/pull/3386 --- .../ledger/migrations/migration_matrics_collector_test.go | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/cmd/util/ledger/migrations/migration_matrics_collector_test.go b/cmd/util/ledger/migrations/migration_matrics_collector_test.go index e6a8e4f5466..1bd70a82739 100644 --- a/cmd/util/ledger/migrations/migration_matrics_collector_test.go +++ b/cmd/util/ledger/migrations/migration_matrics_collector_test.go @@ -87,8 +87,7 @@ func TestMigrationMetricsCollection(t *testing.T) { require.Equal( t, Metrics{ - // TODO: should be 752, like on master - TotalValues: 750, + TotalValues: 752, TotalErrors: 6, ErrorsPerContract: map[string]int{ "A.01cf0e2f2f715450.Test": 6, @@ -188,8 +187,7 @@ func TestMigrationMetricsCollection(t *testing.T) { require.Equal( t, Metrics{ - // TODO: should be 752, like on master - TotalValues: 750, + TotalValues: 752, TotalErrors: 6, ErrorsPerContract: map[string]int{ "A.01cf0e2f2f715450.Test": 6, From d3af3c6cbde79df90abae09a66197bccb133b050 Mon Sep 17 00:00:00 2001 From: Ardit Marku Date: Mon, 10 Jun 2024 15:24:52 +0300 Subject: [PATCH 15/60] Take into account gas refunds in EVM.dryRun To correctly compute the gas estimation with EVM.dryRun, we need to add any gas refunds to the used gas. Any potential gas refunds are required during transaction execution, and only after they are refunded to the transaction author. --- fvm/evm/emulator/emulator.go | 13 ++- fvm/evm/evm_test.go | 202 +++++++++++++++++++++++++++++++++++ fvm/evm/types/result.go | 4 + 3 files changed, 218 insertions(+), 1 deletion(-) diff --git a/fvm/evm/emulator/emulator.go b/fvm/evm/emulator/emulator.go index ee06deb3d01..8d1c7a2cdd9 100644 --- a/fvm/evm/emulator/emulator.go +++ b/fvm/evm/emulator/emulator.go @@ -234,7 +234,17 @@ func (bl *BlockView) DryRunTransaction( msg.SkipAccountChecks = true // return without commiting the state - return proc.run(msg, tx.Hash(), 0, tx.Type()) + txResult, err := proc.run(msg, tx.Hash(), 0, tx.Type()) + if txResult.VMError == nil && txResult.ValidationError == nil { + // Adding `gethParams.SstoreSentryGasEIP2200` is needed for this condition: + // https://github.com/onflow/go-ethereum/blob/master/core/vm/operations_acl.go#L29-L32 + txResult.GasConsumed += gethParams.SstoreSentryGasEIP2200 + // Take into account any gas refunds, which are calculated only after + // transaction execution. + txResult.GasConsumed += txResult.GasRefund + } + + return txResult, err } func (bl *BlockView) newProcedure() (*procedure, error) { @@ -522,6 +532,7 @@ func (proc *procedure) run( // if prechecks are passed, the exec result won't be nil if execResult != nil { res.GasConsumed = execResult.UsedGas + res.GasRefund = proc.state.GetRefund() res.Index = uint16(txIndex) // we need to capture the returned value no matter the status // if the tx is reverted the error message is returned as returned value diff --git a/fvm/evm/evm_test.go b/fvm/evm/evm_test.go index 12cd3994b24..fb332792cb5 100644 --- a/fvm/evm/evm_test.go +++ b/fvm/evm/evm_test.go @@ -12,6 +12,7 @@ import ( "github.com/onflow/cadence/encoding/ccf" gethTypes "github.com/onflow/go-ethereum/core/types" + "github.com/onflow/go-ethereum/params" "github.com/onflow/go-ethereum/rlp" "github.com/stretchr/testify/assert" @@ -1462,6 +1463,207 @@ func TestDryRun(t *testing.T) { }) }) + t.Run("test dry run storing current value", func(t *testing.T) { + RunWithNewEnvironment(t, + chain, func( + ctx fvm.Context, + vm fvm.VM, + snapshot snapshot.SnapshotTree, + testContract *TestContract, + testAccount *EOATestAccount, + ) { + data := testContract.MakeCallData(t, "store", big.NewInt(0)) + + limit := uint64(math.MaxUint64 - 1) + tx := gethTypes.NewTransaction( + 0, + testContract.DeployedAt.ToCommon(), + big.NewInt(0), + limit, + big.NewInt(0), + data, + ) + result := dryRunTx(t, tx, ctx, vm, snapshot, testContract) + require.Equal(t, types.ErrCodeNoError, result.ErrorCode) + require.Equal(t, types.StatusSuccessful, result.Status) + require.Greater(t, result.GasConsumed, uint64(0)) + require.Equal( + t, + uint64(23873)+params.SstoreSentryGasEIP2200, + result.GasConsumed, + ) + require.Equal(t, uint64(0), result.GasRefund) + }) + }) + + t.Run("test dry run storing new value", func(t *testing.T) { + RunWithNewEnvironment(t, + chain, func( + ctx fvm.Context, + vm fvm.VM, + snapshot snapshot.SnapshotTree, + testContract *TestContract, + testAccount *EOATestAccount, + ) { + sc := systemcontracts.SystemContractsForChain(chain.ChainID()) + code := []byte(fmt.Sprintf( + ` + import EVM from %s + + transaction(tx: [UInt8], coinbaseBytes: [UInt8; 20]){ + prepare(account: &Account) { + let coinbase = EVM.EVMAddress(bytes: coinbaseBytes) + let res = EVM.run(tx: tx, coinbase: coinbase) + + assert(res.status == EVM.Status.successful, message: "unexpected status") + assert(res.errorCode == 0, message: "unexpected error code") + } + } + `, + sc.EVMContract.Address.HexWithPrefix(), + )) + + num := int64(12) + innerTxBytes := testAccount.PrepareSignAndEncodeTx(t, + testContract.DeployedAt.ToCommon(), + testContract.MakeCallData(t, "store", big.NewInt(num)), + big.NewInt(0), + uint64(50_000), + big.NewInt(0), + ) + + innerTx := cadence.NewArray( + ConvertToCadence(innerTxBytes), + ).WithType(stdlib.EVMTransactionBytesCadenceType) + + coinbase := cadence.NewArray( + ConvertToCadence(testAccount.Address().Bytes()), + ).WithType(stdlib.EVMAddressBytesCadenceType) + + tx := fvm.Transaction( + flow.NewTransactionBody(). + SetScript(code). + AddAuthorizer(sc.FlowServiceAccount.Address). + AddArgument(json.MustEncode(innerTx)). + AddArgument(json.MustEncode(coinbase)), + 0) + + _, output, err := vm.Run( + ctx, + tx, + snapshot, + ) + require.NoError(t, err) + require.NoError(t, output.Err) + + data := testContract.MakeCallData(t, "store", big.NewInt(100)) + + tx1 := gethTypes.NewTransaction( + 0, + testContract.DeployedAt.ToCommon(), + big.NewInt(0), + uint64(50_000), + big.NewInt(0), + data, + ) + result := dryRunTx(t, tx1, ctx, vm, snapshot, testContract) + require.Equal(t, types.ErrCodeNoError, result.ErrorCode) + require.Equal(t, types.StatusSuccessful, result.Status) + require.Greater(t, result.GasConsumed, uint64(0)) + require.Equal( + t, + uint64(43785)+params.SstoreSentryGasEIP2200, + result.GasConsumed, + ) + require.Equal(t, uint64(0), result.GasRefund) + }) + }) + + t.Run("test dry run clear current value", func(t *testing.T) { + RunWithNewEnvironment(t, + chain, func( + ctx fvm.Context, + vm fvm.VM, + snapshot snapshot.SnapshotTree, + testContract *TestContract, + testAccount *EOATestAccount, + ) { + sc := systemcontracts.SystemContractsForChain(chain.ChainID()) + code := []byte(fmt.Sprintf( + ` + import EVM from %s + + transaction(tx: [UInt8], coinbaseBytes: [UInt8; 20]){ + prepare(account: &Account) { + let coinbase = EVM.EVMAddress(bytes: coinbaseBytes) + let res = EVM.run(tx: tx, coinbase: coinbase) + + assert(res.status == EVM.Status.successful, message: "unexpected status") + assert(res.errorCode == 0, message: "unexpected error code") + } + } + `, + sc.EVMContract.Address.HexWithPrefix(), + )) + + num := int64(100) + innerTxBytes := testAccount.PrepareSignAndEncodeTx(t, + testContract.DeployedAt.ToCommon(), + testContract.MakeCallData(t, "store", big.NewInt(num)), + big.NewInt(0), + uint64(50_000), + big.NewInt(0), + ) + + innerTx := cadence.NewArray( + ConvertToCadence(innerTxBytes), + ).WithType(stdlib.EVMTransactionBytesCadenceType) + + coinbase := cadence.NewArray( + ConvertToCadence(testAccount.Address().Bytes()), + ).WithType(stdlib.EVMAddressBytesCadenceType) + + tx := fvm.Transaction( + flow.NewTransactionBody(). + SetScript(code). + AddAuthorizer(sc.FlowServiceAccount.Address). + AddArgument(json.MustEncode(innerTx)). + AddArgument(json.MustEncode(coinbase)), + 0) + + state, output, err := vm.Run( + ctx, + tx, + snapshot, + ) + require.NoError(t, err) + require.NoError(t, output.Err) + snapshot = snapshot.Append(state) + + data := testContract.MakeCallData(t, "store", big.NewInt(0)) + + tx1 := gethTypes.NewTransaction( + 0, + testContract.DeployedAt.ToCommon(), + big.NewInt(0), + uint64(50_000), + big.NewInt(0), + data, + ) + result := dryRunTx(t, tx1, ctx, vm, snapshot, testContract) + require.Equal(t, types.ErrCodeNoError, result.ErrorCode) + require.Equal(t, types.StatusSuccessful, result.Status) + require.Greater(t, result.GasConsumed, uint64(0)) + require.Equal( + t, + uint64(21873)+params.SstoreSentryGasEIP2200+params.SstoreClearsScheduleRefundEIP3529, + result.GasConsumed, + ) + // We might want to think about exposing GasRefuned to EVM contract. + // require.Equal(t, uint64(4800), result.GasRefund) + }) + }) + // this test makes sure the dry-run that updates the value on the contract // doesn't persist the change, and after when the value is read it isn't updated. t.Run("test dry run for any side-effects", func(t *testing.T) { diff --git a/fvm/evm/types/result.go b/fvm/evm/types/result.go index ac22760464c..3c77a821ad8 100644 --- a/fvm/evm/types/result.go +++ b/fvm/evm/types/result.go @@ -39,6 +39,7 @@ type ResultSummary struct { ErrorCode ErrorCode ErrorMessage string GasConsumed uint64 + GasRefund uint64 DeployedContractAddress *Address ReturnedData Data } @@ -71,6 +72,8 @@ type Result struct { TxType uint8 // total gas consumed during an opeartion GasConsumed uint64 + // total gas refunds after transaction execution + GasRefund uint64 // the address where the contract is deployed (if any) DeployedContractAddress *Address // returned data from a function call @@ -142,6 +145,7 @@ func (res *Result) Receipt() *gethTypes.Receipt { func (res *Result) ResultSummary() *ResultSummary { rs := &ResultSummary{ GasConsumed: res.GasConsumed, + GasRefund: res.GasRefund, DeployedContractAddress: res.DeployedContractAddress, ReturnedData: res.ReturnedData, Status: StatusSuccessful, From 058e3c4e76f292c060957b7a6111fa2a25f3dc57 Mon Sep 17 00:00:00 2001 From: Jordan Schalm Date: Mon, 10 Jun 2024 13:03:22 -0700 Subject: [PATCH 16/60] use 250ms block time across all epoch tests --- integration/tests/epochs/base_suite.go | 3 +-- .../tests/epochs/cohort2/epoch_join_and_leave_sn_test.go | 6 ------ .../tests/epochs/cohort2/epoch_join_and_leave_vn_test.go | 3 --- 3 files changed, 1 insertion(+), 11 deletions(-) diff --git a/integration/tests/epochs/base_suite.go b/integration/tests/epochs/base_suite.go index 3f2d037a146..69dce17d98f 100644 --- a/integration/tests/epochs/base_suite.go +++ b/integration/tests/epochs/base_suite.go @@ -50,9 +50,8 @@ type BaseSuite struct { // SetupTest is run automatically by the testing framework before each test case. func (s *BaseSuite) SetupTest() { - // If unset, use default value 100ms if s.ConsensusProposalDuration == 0 { - s.ConsensusProposalDuration = time.Millisecond * 100 + s.ConsensusProposalDuration = time.Millisecond * 250 } minEpochLength := s.StakingAuctionLen + s.DKGPhaseLen*3 + 20 diff --git a/integration/tests/epochs/cohort2/epoch_join_and_leave_sn_test.go b/integration/tests/epochs/cohort2/epoch_join_and_leave_sn_test.go index d101af6371d..1dfd01fec0a 100644 --- a/integration/tests/epochs/cohort2/epoch_join_and_leave_sn_test.go +++ b/integration/tests/epochs/cohort2/epoch_join_and_leave_sn_test.go @@ -2,7 +2,6 @@ package cohort2 import ( "testing" - "time" "github.com/stretchr/testify/suite" @@ -19,11 +18,6 @@ type EpochJoinAndLeaveSNSuite struct { } func (s *EpochJoinAndLeaveSNSuite) SetupTest() { - // slow down the block rate. This is needed since the crypto module - // update provides faster BLS operations. - // TODO: fix the access integration test logic to function without slowing down - // the block rate - s.ConsensusProposalDuration = time.Millisecond * 250 s.DynamicEpochTransitionSuite.SetupTest() } diff --git a/integration/tests/epochs/cohort2/epoch_join_and_leave_vn_test.go b/integration/tests/epochs/cohort2/epoch_join_and_leave_vn_test.go index 9f87c156ed7..716927ddfc3 100644 --- a/integration/tests/epochs/cohort2/epoch_join_and_leave_vn_test.go +++ b/integration/tests/epochs/cohort2/epoch_join_and_leave_vn_test.go @@ -2,7 +2,6 @@ package cohort2 import ( "testing" - "time" "github.com/stretchr/testify/suite" @@ -21,8 +20,6 @@ type EpochJoinAndLeaveVNSuite struct { func (s *EpochJoinAndLeaveVNSuite) SetupTest() { // require approvals for seals to verify that the joining VN is producing valid seals in the second epoch s.RequiredSealApprovals = 1 - // slow down consensus, as sealing tends to lag behind - s.ConsensusProposalDuration = time.Millisecond * 250 // increase epoch length to account for greater sealing lag due to above // NOTE: this value is set fairly aggressively to ensure shorter test times. // If flakiness due to failure to complete staking operations in time is observed, From b6fc9e1ea349062b7fbbb9d575421f9d213518e2 Mon Sep 17 00:00:00 2001 From: Ardit Marku Date: Tue, 11 Jun 2024 13:00:52 +0300 Subject: [PATCH 17/60] Add the Successful() method on Result and improve tests for EVM.dryRun --- fvm/evm/emulator/emulator.go | 2 +- fvm/evm/evm_test.go | 202 +++++++++++++++++++++++++++++------ fvm/evm/types/result.go | 5 + 3 files changed, 178 insertions(+), 31 deletions(-) diff --git a/fvm/evm/emulator/emulator.go b/fvm/evm/emulator/emulator.go index 8d1c7a2cdd9..b86821e9f7f 100644 --- a/fvm/evm/emulator/emulator.go +++ b/fvm/evm/emulator/emulator.go @@ -235,7 +235,7 @@ func (bl *BlockView) DryRunTransaction( // return without commiting the state txResult, err := proc.run(msg, tx.Hash(), 0, tx.Type()) - if txResult.VMError == nil && txResult.ValidationError == nil { + if txResult.Successful() { // Adding `gethParams.SstoreSentryGasEIP2200` is needed for this condition: // https://github.com/onflow/go-ethereum/blob/master/core/vm/operations_acl.go#L29-L32 txResult.GasConsumed += gethParams.SstoreSentryGasEIP2200 diff --git a/fvm/evm/evm_test.go b/fvm/evm/evm_test.go index fb332792cb5..3346238a30a 100644 --- a/fvm/evm/evm_test.go +++ b/fvm/evm/evm_test.go @@ -12,7 +12,7 @@ import ( "github.com/onflow/cadence/encoding/ccf" gethTypes "github.com/onflow/go-ethereum/core/types" - "github.com/onflow/go-ethereum/params" + gethParams "github.com/onflow/go-ethereum/params" "github.com/onflow/go-ethereum/rlp" "github.com/stretchr/testify/assert" @@ -1463,7 +1463,7 @@ func TestDryRun(t *testing.T) { }) }) - t.Run("test dry run storing current value", func(t *testing.T) { + t.Run("test dry run store current value", func(t *testing.T) { RunWithNewEnvironment(t, chain, func( ctx fvm.Context, @@ -1473,30 +1473,76 @@ func TestDryRun(t *testing.T) { testAccount *EOATestAccount, ) { data := testContract.MakeCallData(t, "store", big.NewInt(0)) - - limit := uint64(math.MaxUint64 - 1) tx := gethTypes.NewTransaction( 0, testContract.DeployedAt.ToCommon(), big.NewInt(0), - limit, + uint64(50_000), big.NewInt(0), data, ) - result := dryRunTx(t, tx, ctx, vm, snapshot, testContract) - require.Equal(t, types.ErrCodeNoError, result.ErrorCode) - require.Equal(t, types.StatusSuccessful, result.Status) - require.Greater(t, result.GasConsumed, uint64(0)) + dryRunResult := dryRunTx(t, tx, ctx, vm, snapshot, testContract) + + require.Equal(t, types.ErrCodeNoError, dryRunResult.ErrorCode) + require.Equal(t, types.StatusSuccessful, dryRunResult.Status) + require.Greater(t, dryRunResult.GasConsumed, uint64(0)) + + code := []byte(fmt.Sprintf( + ` + import EVM from %s + access(all) + fun main(tx: [UInt8], coinbaseBytes: [UInt8; 20]): EVM.Result { + let coinbase = EVM.EVMAddress(bytes: coinbaseBytes) + return EVM.run(tx: tx, coinbase: coinbase) + } + `, + evmAddress, + )) + + innerTxBytes := testAccount.PrepareSignAndEncodeTx(t, + testContract.DeployedAt.ToCommon(), + data, + big.NewInt(0), + dryRunResult.GasConsumed, // use the gas estimation from Evm.dryRun + big.NewInt(0), + ) + + innerTx := cadence.NewArray( + ConvertToCadence(innerTxBytes), + ).WithType(stdlib.EVMTransactionBytesCadenceType) + + coinbase := cadence.NewArray( + ConvertToCadence(testAccount.Address().Bytes()), + ).WithType(stdlib.EVMAddressBytesCadenceType) + + script := fvm.Script(code).WithArguments( + json.MustEncode(innerTx), + json.MustEncode(coinbase), + ) + + _, output, err := vm.Run( + ctx, + script, + snapshot) + require.NoError(t, err) + require.NoError(t, output.Err) + + res, err := stdlib.ResultSummaryFromEVMResultValue(output.Value) + require.NoError(t, err) + require.Equal(t, types.StatusSuccessful, res.Status) + require.Equal(t, types.ErrCodeNoError, res.ErrorCode) + // Make sure that gas consumed from `EVM.dryRun` is bigger + // than the actual gas consumption of the equivalent + // `EVM.run`. require.Equal( t, - uint64(23873)+params.SstoreSentryGasEIP2200, - result.GasConsumed, + res.GasConsumed+gethParams.SstoreSentryGasEIP2200, + dryRunResult.GasConsumed, ) - require.Equal(t, uint64(0), result.GasRefund) }) }) - t.Run("test dry run storing new value", func(t *testing.T) { + t.Run("test dry run store new value", func(t *testing.T) { RunWithNewEnvironment(t, chain, func( ctx fvm.Context, @@ -1557,7 +1603,6 @@ func TestDryRun(t *testing.T) { require.NoError(t, output.Err) data := testContract.MakeCallData(t, "store", big.NewInt(100)) - tx1 := gethTypes.NewTransaction( 0, testContract.DeployedAt.ToCommon(), @@ -1566,16 +1611,67 @@ func TestDryRun(t *testing.T) { big.NewInt(0), data, ) - result := dryRunTx(t, tx1, ctx, vm, snapshot, testContract) - require.Equal(t, types.ErrCodeNoError, result.ErrorCode) - require.Equal(t, types.StatusSuccessful, result.Status) - require.Greater(t, result.GasConsumed, uint64(0)) + dryRunResult := dryRunTx(t, tx1, ctx, vm, snapshot, testContract) + + require.Equal(t, types.ErrCodeNoError, dryRunResult.ErrorCode) + require.Equal(t, types.StatusSuccessful, dryRunResult.Status) + require.Greater(t, dryRunResult.GasConsumed, uint64(0)) + + code = []byte(fmt.Sprintf( + ` + import EVM from %s + access(all) + fun main(tx: [UInt8], coinbaseBytes: [UInt8; 20]): EVM.Result { + let coinbase = EVM.EVMAddress(bytes: coinbaseBytes) + return EVM.run(tx: tx, coinbase: coinbase) + } + `, + evmAddress, + )) + + // Decrease nonce because we are Cadence using scripts, and not + // transactions, which means that no state change is happening. + testAccount.SetNonce(testAccount.Nonce() - 1) + innerTxBytes = testAccount.PrepareSignAndEncodeTx(t, + testContract.DeployedAt.ToCommon(), + data, + big.NewInt(0), + dryRunResult.GasConsumed, // use the gas estimation from Evm.dryRun + big.NewInt(0), + ) + + innerTx = cadence.NewArray( + ConvertToCadence(innerTxBytes), + ).WithType(stdlib.EVMTransactionBytesCadenceType) + + coinbase = cadence.NewArray( + ConvertToCadence(testAccount.Address().Bytes()), + ).WithType(stdlib.EVMAddressBytesCadenceType) + + script := fvm.Script(code).WithArguments( + json.MustEncode(innerTx), + json.MustEncode(coinbase), + ) + + _, output, err = vm.Run( + ctx, + script, + snapshot) + require.NoError(t, err) + require.NoError(t, output.Err) + + res, err := stdlib.ResultSummaryFromEVMResultValue(output.Value) + require.NoError(t, err) + require.Equal(t, types.StatusSuccessful, res.Status) + require.Equal(t, types.ErrCodeNoError, res.ErrorCode) + // Make sure that gas consumed from `EVM.dryRun` is bigger + // than the actual gas consumption of the equivalent + // `EVM.run`. require.Equal( t, - uint64(43785)+params.SstoreSentryGasEIP2200, - result.GasConsumed, + res.GasConsumed+gethParams.SstoreSentryGasEIP2200, + dryRunResult.GasConsumed, ) - require.Equal(t, uint64(0), result.GasRefund) }) }) @@ -1641,7 +1737,6 @@ func TestDryRun(t *testing.T) { snapshot = snapshot.Append(state) data := testContract.MakeCallData(t, "store", big.NewInt(0)) - tx1 := gethTypes.NewTransaction( 0, testContract.DeployedAt.ToCommon(), @@ -1650,17 +1745,64 @@ func TestDryRun(t *testing.T) { big.NewInt(0), data, ) - result := dryRunTx(t, tx1, ctx, vm, snapshot, testContract) - require.Equal(t, types.ErrCodeNoError, result.ErrorCode) - require.Equal(t, types.StatusSuccessful, result.Status) - require.Greater(t, result.GasConsumed, uint64(0)) + dryRunResult := dryRunTx(t, tx1, ctx, vm, snapshot, testContract) + + require.Equal(t, types.ErrCodeNoError, dryRunResult.ErrorCode) + require.Equal(t, types.StatusSuccessful, dryRunResult.Status) + require.Greater(t, dryRunResult.GasConsumed, uint64(0)) + + code = []byte(fmt.Sprintf( + ` + import EVM from %s + access(all) + fun main(tx: [UInt8], coinbaseBytes: [UInt8; 20]): EVM.Result { + let coinbase = EVM.EVMAddress(bytes: coinbaseBytes) + return EVM.run(tx: tx, coinbase: coinbase) + } + `, + evmAddress, + )) + + innerTxBytes = testAccount.PrepareSignAndEncodeTx(t, + testContract.DeployedAt.ToCommon(), + data, + big.NewInt(0), + dryRunResult.GasConsumed, // use the gas estimation from Evm.dryRun + big.NewInt(0), + ) + + innerTx = cadence.NewArray( + ConvertToCadence(innerTxBytes), + ).WithType(stdlib.EVMTransactionBytesCadenceType) + + coinbase = cadence.NewArray( + ConvertToCadence(testAccount.Address().Bytes()), + ).WithType(stdlib.EVMAddressBytesCadenceType) + + script := fvm.Script(code).WithArguments( + json.MustEncode(innerTx), + json.MustEncode(coinbase), + ) + + _, output, err = vm.Run( + ctx, + script, + snapshot) + require.NoError(t, err) + require.NoError(t, output.Err) + + res, err := stdlib.ResultSummaryFromEVMResultValue(output.Value) + require.NoError(t, err) + //require.Equal(t, types.StatusSuccessful, res.Status) + require.Equal(t, types.ErrCodeNoError, res.ErrorCode) + // Make sure that gas consumed from `EVM.dryRun` is bigger + // than the actual gas consumption of the equivalent + // `EVM.run`. require.Equal( t, - uint64(21873)+params.SstoreSentryGasEIP2200+params.SstoreClearsScheduleRefundEIP3529, - result.GasConsumed, + res.GasConsumed+gethParams.SstoreSentryGasEIP2200+gethParams.SstoreClearsScheduleRefundEIP3529, + dryRunResult.GasConsumed, ) - // We might want to think about exposing GasRefuned to EVM contract. - // require.Equal(t, uint64(4800), result.GasRefund) }) }) diff --git a/fvm/evm/types/result.go b/fvm/evm/types/result.go index 3c77a821ad8..aba686c1a9f 100644 --- a/fvm/evm/types/result.go +++ b/fvm/evm/types/result.go @@ -96,6 +96,11 @@ func (res *Result) Failed() bool { return res.VMError != nil } +// Successful returns true if transaction has been executed without any errors +func (res *Result) Successful() bool { + return !res.Failed() && !res.Invalid() +} + // SetValidationError sets the validation error // and also sets the gas used to the fixed invalid gas usage func (res *Result) SetValidationError(err error) { From a96af388b551e3b7211d7d0d3d79bd839798d6fb Mon Sep 17 00:00:00 2001 From: "Leo Zhang (zhangchiqing)" Date: Wed, 5 Jun 2024 10:56:48 -0700 Subject: [PATCH 18/60] update chunk data pack storage interface --- storage/chunkDataPacks.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/storage/chunkDataPacks.go b/storage/chunkDataPacks.go index d9f49735d90..a17acaf0e86 100644 --- a/storage/chunkDataPacks.go +++ b/storage/chunkDataPacks.go @@ -15,9 +15,6 @@ type ChunkDataPacks interface { // No errors are expected during normal operation, but it may return generic error Remove(cs []flow.Identifier) error - // BatchStore inserts the chunk header, keyed by chunk ID into a given batch - BatchStore(c *flow.ChunkDataPack, batch BatchStorage) error - // ByChunkID returns the chunk data for the given a chunk ID. ByChunkID(chunkID flow.Identifier) (*flow.ChunkDataPack, error) From d907833726e22b27bbf234231163362f427b8eda Mon Sep 17 00:00:00 2001 From: "Leo Zhang (zhangchiqing)" Date: Wed, 5 Jun 2024 11:59:46 -0700 Subject: [PATCH 19/60] move StoredChunkDataPack to storage package --- storage/badger/chunkDataPacks.go | 23 +++++++------- storage/badger/model/storedChunkDataPack.go | 17 ---------- storage/badger/operation/chunkDataPacks.go | 8 ++--- .../badger/operation/chunkDataPacks_test.go | 10 +++--- storage/chunkDataPacks.go | 31 +++++++++++++++++++ 5 files changed, 51 insertions(+), 38 deletions(-) delete mode 100644 storage/badger/model/storedChunkDataPack.go diff --git a/storage/badger/chunkDataPacks.go b/storage/badger/chunkDataPacks.go index 63865a574ce..7d08aef934f 100644 --- a/storage/badger/chunkDataPacks.go +++ b/storage/badger/chunkDataPacks.go @@ -9,7 +9,6 @@ import ( "github.com/onflow/flow-go/module" "github.com/onflow/flow-go/module/metrics" "github.com/onflow/flow-go/storage" - badgermodel "github.com/onflow/flow-go/storage/badger/model" "github.com/onflow/flow-go/storage/badger/operation" "github.com/onflow/flow-go/storage/badger/transaction" ) @@ -17,25 +16,25 @@ import ( type ChunkDataPacks struct { db *badger.DB collections storage.Collections - byChunkIDCache *Cache[flow.Identifier, *badgermodel.StoredChunkDataPack] + byChunkIDCache *Cache[flow.Identifier, *storage.StoredChunkDataPack] } func NewChunkDataPacks(collector module.CacheMetrics, db *badger.DB, collections storage.Collections, byChunkIDCacheSize uint) *ChunkDataPacks { - store := func(key flow.Identifier, val *badgermodel.StoredChunkDataPack) func(*transaction.Tx) error { + store := func(key flow.Identifier, val *storage.StoredChunkDataPack) func(*transaction.Tx) error { return transaction.WithTx(operation.SkipDuplicates(operation.InsertChunkDataPack(val))) } - retrieve := func(key flow.Identifier) func(tx *badger.Txn) (*badgermodel.StoredChunkDataPack, error) { - return func(tx *badger.Txn) (*badgermodel.StoredChunkDataPack, error) { - var c badgermodel.StoredChunkDataPack + retrieve := func(key flow.Identifier) func(tx *badger.Txn) (*storage.StoredChunkDataPack, error) { + return func(tx *badger.Txn) (*storage.StoredChunkDataPack, error) { + var c storage.StoredChunkDataPack err := operation.RetrieveChunkDataPack(key, &c)(tx) return &c, err } } cache := newCache(collector, metrics.ResourceChunkDataPack, - withLimit[flow.Identifier, *badgermodel.StoredChunkDataPack](byChunkIDCacheSize), + withLimit[flow.Identifier, *storage.StoredChunkDataPack](byChunkIDCacheSize), withStore(store), withRetrieve(retrieve), ) @@ -133,7 +132,7 @@ func (ch *ChunkDataPacks) ByChunkID(chunkID flow.Identifier) (*flow.ChunkDataPac return chdp, nil } -func (ch *ChunkDataPacks) byChunkID(chunkID flow.Identifier) (*badgermodel.StoredChunkDataPack, error) { +func (ch *ChunkDataPacks) byChunkID(chunkID flow.Identifier) (*storage.StoredChunkDataPack, error) { tx := ch.db.NewTransaction(false) defer tx.Discard() @@ -145,8 +144,8 @@ func (ch *ChunkDataPacks) byChunkID(chunkID flow.Identifier) (*badgermodel.Store return schdp, nil } -func (ch *ChunkDataPacks) retrieveCHDP(chunkID flow.Identifier) func(*badger.Txn) (*badgermodel.StoredChunkDataPack, error) { - return func(tx *badger.Txn) (*badgermodel.StoredChunkDataPack, error) { +func (ch *ChunkDataPacks) retrieveCHDP(chunkID flow.Identifier) func(*badger.Txn) (*storage.StoredChunkDataPack, error) { + return func(tx *badger.Txn) (*storage.StoredChunkDataPack, error) { val, err := ch.byChunkIDCache.Get(chunkID)(tx) if err != nil { return nil, err @@ -155,8 +154,8 @@ func (ch *ChunkDataPacks) retrieveCHDP(chunkID flow.Identifier) func(*badger.Txn } } -func toStoredChunkDataPack(c *flow.ChunkDataPack) *badgermodel.StoredChunkDataPack { - sc := &badgermodel.StoredChunkDataPack{ +func toStoredChunkDataPack(c *flow.ChunkDataPack) *storage.StoredChunkDataPack { + sc := &storage.StoredChunkDataPack{ ChunkID: c.ChunkID, StartState: c.StartState, Proof: c.Proof, diff --git a/storage/badger/model/storedChunkDataPack.go b/storage/badger/model/storedChunkDataPack.go deleted file mode 100644 index 31349604070..00000000000 --- a/storage/badger/model/storedChunkDataPack.go +++ /dev/null @@ -1,17 +0,0 @@ -package badgermodel - -import ( - "github.com/onflow/flow-go/model/flow" -) - -// StoredChunkDataPack is an in-storage representation of chunk data pack. -// Its prime difference is instead of an actual collection, it keeps a collection ID hence relying on maintaining -// the collection on a secondary storage. -type StoredChunkDataPack struct { - ChunkID flow.Identifier - StartState flow.StateCommitment - Proof flow.StorageProof - CollectionID flow.Identifier - SystemChunk bool - ExecutionDataRoot flow.BlockExecutionDataRoot -} diff --git a/storage/badger/operation/chunkDataPacks.go b/storage/badger/operation/chunkDataPacks.go index 687712985d4..e0f2deb2ce2 100644 --- a/storage/badger/operation/chunkDataPacks.go +++ b/storage/badger/operation/chunkDataPacks.go @@ -4,16 +4,16 @@ import ( "github.com/dgraph-io/badger/v2" "github.com/onflow/flow-go/model/flow" - badgermodel "github.com/onflow/flow-go/storage/badger/model" + "github.com/onflow/flow-go/storage" ) // InsertChunkDataPack inserts a chunk data pack keyed by chunk ID. -func InsertChunkDataPack(c *badgermodel.StoredChunkDataPack) func(*badger.Txn) error { +func InsertChunkDataPack(c *storage.StoredChunkDataPack) func(*badger.Txn) error { return insert(makePrefix(codeChunkDataPack, c.ChunkID), c) } // BatchInsertChunkDataPack inserts a chunk data pack keyed by chunk ID into a batch -func BatchInsertChunkDataPack(c *badgermodel.StoredChunkDataPack) func(batch *badger.WriteBatch) error { +func BatchInsertChunkDataPack(c *storage.StoredChunkDataPack) func(batch *badger.WriteBatch) error { return batchWrite(makePrefix(codeChunkDataPack, c.ChunkID), c) } @@ -25,7 +25,7 @@ func BatchRemoveChunkDataPack(chunkID flow.Identifier) func(batch *badger.WriteB } // RetrieveChunkDataPack retrieves a chunk data pack by chunk ID. -func RetrieveChunkDataPack(chunkID flow.Identifier, c *badgermodel.StoredChunkDataPack) func(*badger.Txn) error { +func RetrieveChunkDataPack(chunkID flow.Identifier, c *storage.StoredChunkDataPack) func(*badger.Txn) error { return retrieve(makePrefix(codeChunkDataPack, chunkID), c) } diff --git a/storage/badger/operation/chunkDataPacks_test.go b/storage/badger/operation/chunkDataPacks_test.go index 0dc79ef7266..f3a90af8d00 100644 --- a/storage/badger/operation/chunkDataPacks_test.go +++ b/storage/badger/operation/chunkDataPacks_test.go @@ -7,14 +7,14 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - storagemodel "github.com/onflow/flow-go/storage/badger/model" + "github.com/onflow/flow-go/storage" "github.com/onflow/flow-go/utils/unittest" ) func TestChunkDataPack(t *testing.T) { unittest.RunWithBadgerDB(t, func(db *badger.DB) { collectionID := unittest.IdentifierFixture() - expected := &storagemodel.StoredChunkDataPack{ + expected := &storage.StoredChunkDataPack{ ChunkID: unittest.IdentifierFixture(), StartState: unittest.StateCommitmentFixture(), Proof: []byte{'p'}, @@ -22,7 +22,7 @@ func TestChunkDataPack(t *testing.T) { } t.Run("Retrieve non-existent", func(t *testing.T) { - var actual storagemodel.StoredChunkDataPack + var actual storage.StoredChunkDataPack err := db.View(RetrieveChunkDataPack(expected.ChunkID, &actual)) assert.Error(t, err) }) @@ -31,7 +31,7 @@ func TestChunkDataPack(t *testing.T) { err := db.Update(InsertChunkDataPack(expected)) require.NoError(t, err) - var actual storagemodel.StoredChunkDataPack + var actual storage.StoredChunkDataPack err = db.View(RetrieveChunkDataPack(expected.ChunkID, &actual)) assert.NoError(t, err) @@ -42,7 +42,7 @@ func TestChunkDataPack(t *testing.T) { err := db.Update(RemoveChunkDataPack(expected.ChunkID)) require.NoError(t, err) - var actual storagemodel.StoredChunkDataPack + var actual storage.StoredChunkDataPack err = db.View(RetrieveChunkDataPack(expected.ChunkID, &actual)) assert.Error(t, err) }) diff --git a/storage/chunkDataPacks.go b/storage/chunkDataPacks.go index a17acaf0e86..60d4fe2375d 100644 --- a/storage/chunkDataPacks.go +++ b/storage/chunkDataPacks.go @@ -23,3 +23,34 @@ type ChunkDataPacks interface { // If Badger unexpectedly fails to process the request, the error is wrapped in a generic error and returned. BatchRemove(chunkID flow.Identifier, batch BatchStorage) error } + +// StoredChunkDataPack is an in-storage representation of chunk data pack. +// Its prime difference is instead of an actual collection, it keeps a collection ID hence relying on maintaining +// the collection on a secondary storage. +type StoredChunkDataPack struct { + ChunkID flow.Identifier + StartState flow.StateCommitment + Proof flow.StorageProof + CollectionID flow.Identifier + SystemChunk bool + ExecutionDataRoot flow.BlockExecutionDataRoot +} + +func ToStoredChunkDataPack(c *flow.ChunkDataPack) *StoredChunkDataPack { + sc := &StoredChunkDataPack{ + ChunkID: c.ChunkID, + StartState: c.StartState, + Proof: c.Proof, + SystemChunk: false, + ExecutionDataRoot: c.ExecutionDataRoot, + } + + if c.Collection != nil { + // non system chunk + sc.CollectionID = c.Collection.ID() + } else { + sc.SystemChunk = true + } + + return sc +} From 14dcad5e8f45044d75cfc410462ccaf98846dd07 Mon Sep 17 00:00:00 2001 From: "Leo Zhang (zhangchiqing)" Date: Wed, 5 Jun 2024 15:55:16 -0700 Subject: [PATCH 20/60] implement pebble chunk data pack --- model/flow/chunk.go | 51 ++++++++++ storage/badger/chunkDataPacks.go | 21 +--- storage/pebble/chunk_data_packs.go | 128 ++++++++++++++++++++++++ storage/pebble/chunk_data_packs_test.go | 90 +++++++++++++++++ 4 files changed, 270 insertions(+), 20 deletions(-) create mode 100644 storage/pebble/chunk_data_packs.go create mode 100644 storage/pebble/chunk_data_packs_test.go diff --git a/model/flow/chunk.go b/model/flow/chunk.go index e8aeafea8bf..90fd760c0f4 100644 --- a/model/flow/chunk.go +++ b/model/flow/chunk.go @@ -4,6 +4,7 @@ import ( "log" "github.com/ipfs/go-cid" + "github.com/vmihailenco/msgpack/v4" ) var EmptyEventCollectionID Identifier @@ -203,3 +204,53 @@ type BlockExecutionDataRoot struct { // associated with this block. ChunkExecutionDataIDs []cid.Cid } + +// MarshalMsgpack implements the msgpack.Marshaler interface +func (b BlockExecutionDataRoot) MarshalMsgpack() ([]byte, error) { + return msgpack.Marshal(struct { + BlockID Identifier + ChunkExecutionDataIDs []string + }{ + BlockID: b.BlockID, + ChunkExecutionDataIDs: cidsToStrings(b.ChunkExecutionDataIDs), + }) +} + +// UnmarshalMsgpack implements the msgpack.Unmarshaler interface +func (b *BlockExecutionDataRoot) UnmarshalMsgpack(data []byte) error { + var temp struct { + BlockID Identifier + ChunkExecutionDataIDs []string + } + + if err := msgpack.Unmarshal(data, &temp); err != nil { + return err + } + + b.BlockID = temp.BlockID + b.ChunkExecutionDataIDs = stringsToCids(temp.ChunkExecutionDataIDs) + + return nil +} + +// Helper function to convert a slice of cid.Cid to a slice of strings +func cidsToStrings(cids []cid.Cid) []string { + strs := make([]string, len(cids)) + for i, c := range cids { + strs[i] = c.String() + } + return strs +} + +// Helper function to convert a slice of strings to a slice of cid.Cid +func stringsToCids(strs []string) []cid.Cid { + cids := make([]cid.Cid, len(strs)) + for i, s := range strs { + c, err := cid.Decode(s) + if err != nil { + panic(err) // Handle error appropriately in real code + } + cids[i] = c + } + return cids +} diff --git a/storage/badger/chunkDataPacks.go b/storage/badger/chunkDataPacks.go index 7d08aef934f..05f42cf7856 100644 --- a/storage/badger/chunkDataPacks.go +++ b/storage/badger/chunkDataPacks.go @@ -70,7 +70,7 @@ func (ch *ChunkDataPacks) Remove(chunkIDs []flow.Identifier) error { // No errors are expected during normal operation, but it may return generic error // if entity is not serializable or Badger unexpectedly fails to process request func (ch *ChunkDataPacks) BatchStore(c *flow.ChunkDataPack, batch storage.BatchStorage) error { - sc := toStoredChunkDataPack(c) + sc := storage.ToStoredChunkDataPack(c) writeBatch := batch.GetWriter() batch.OnSucceed(func() { ch.byChunkIDCache.Insert(sc.ChunkID, sc) @@ -153,22 +153,3 @@ func (ch *ChunkDataPacks) retrieveCHDP(chunkID flow.Identifier) func(*badger.Txn return val, nil } } - -func toStoredChunkDataPack(c *flow.ChunkDataPack) *storage.StoredChunkDataPack { - sc := &storage.StoredChunkDataPack{ - ChunkID: c.ChunkID, - StartState: c.StartState, - Proof: c.Proof, - SystemChunk: false, - ExecutionDataRoot: c.ExecutionDataRoot, - } - - if c.Collection != nil { - // non system chunk - sc.CollectionID = c.Collection.ID() - } else { - sc.SystemChunk = true - } - - return sc -} diff --git a/storage/pebble/chunk_data_packs.go b/storage/pebble/chunk_data_packs.go new file mode 100644 index 00000000000..4573db65e79 --- /dev/null +++ b/storage/pebble/chunk_data_packs.go @@ -0,0 +1,128 @@ +package pebble + +import ( + "fmt" + + "github.com/cockroachdb/pebble" + "github.com/vmihailenco/msgpack" + + "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/module/irrecoverable" + "github.com/onflow/flow-go/storage" +) + +type ChunkDataPacks struct { + db *pebble.DB + collections storage.Collections +} + +var _ storage.ChunkDataPacks = (*ChunkDataPacks)(nil) + +func NewChunkDataPacks(db *pebble.DB, collections storage.Collections) *ChunkDataPacks { + return &ChunkDataPacks{ + db: db, + collections: collections, + } +} + +func (ch *ChunkDataPacks) Store(cs []*flow.ChunkDataPack) error { + batch := ch.db.NewBatch() + defer batch.Close() + for _, c := range cs { + err := ch.batchStore(c, batch) + if err != nil { + return fmt.Errorf("cannot store chunk data pack: %w", err) + } + } + + err := batch.Commit(pebble.Sync) + if err != nil { + return fmt.Errorf("cannot commit batch: %w", err) + } + + return nil +} + +func (ch *ChunkDataPacks) Remove(cs []flow.Identifier) error { + return nil +} + +func (ch *ChunkDataPacks) ByChunkID(chunkID flow.Identifier) (*flow.ChunkDataPack, error) { + var sc storage.StoredChunkDataPack + err := RetrieveChunkDataPack(ch.db, chunkID, &sc) + if err != nil { + return nil, fmt.Errorf("could not retrieve stored chunk data pack: %w", err) + } + + chdp := &flow.ChunkDataPack{ + ChunkID: sc.ChunkID, + StartState: sc.StartState, + Proof: sc.Proof, + Collection: nil, // to be filled in later + ExecutionDataRoot: sc.ExecutionDataRoot, + } + if !sc.SystemChunk { + collection, err := ch.collections.ByID(sc.CollectionID) + if err != nil { + return nil, fmt.Errorf("could not retrive collection (id: %x) for stored chunk data pack: %w", sc.CollectionID, err) + } + + chdp.Collection = collection + } + return chdp, nil +} + +func (ch *ChunkDataPacks) BatchRemove(chunkID flow.Identifier, batch storage.BatchStorage) error { + return nil +} + +func (ch *ChunkDataPacks) batchStore(c *flow.ChunkDataPack, batch *pebble.Batch) error { + sc := storage.ToStoredChunkDataPack(c) + return InsertChunkDataPack(batch, sc) +} + +func InsertChunkDataPack(batch *pebble.Batch, sc *storage.StoredChunkDataPack) error { + key := makeKey(codeChunkDataPack, sc.ChunkID) + return batchWrite(batch, key, sc) +} + +func RetrieveChunkDataPack(db *pebble.DB, chunkID flow.Identifier, sc *storage.StoredChunkDataPack) error { + key := makeKey(codeChunkDataPack, chunkID) + return retrieve(db, key, sc) +} + +func batchWrite(batch *pebble.Batch, key []byte, val interface{}) error { + value, err := msgpack.Marshal(val) + if err != nil { + return irrecoverable.NewExceptionf("failed to encode value: %w", err) + } + + err = batch.Set(key, value, nil) + if err != nil { + return irrecoverable.NewExceptionf("failed to store data: %w", err) + } + + return nil +} + +func retrieve(db *pebble.DB, key []byte, sc interface{}) error { + val, closer, err := db.Get(key) + if err != nil { + return convertNotFoundError(err) + } + defer closer.Close() + + err = msgpack.Unmarshal(val, &sc) + if err != nil { + return irrecoverable.NewExceptionf("failed to decode value: %w", err) + } + return nil +} + +const ( + codeChunkDataPack = 100 +) + +func makeKey(prefix byte, chunkID flow.Identifier) []byte { + return append([]byte{prefix}, chunkID[:]...) +} diff --git a/storage/pebble/chunk_data_packs_test.go b/storage/pebble/chunk_data_packs_test.go new file mode 100644 index 00000000000..79d6c5bb908 --- /dev/null +++ b/storage/pebble/chunk_data_packs_test.go @@ -0,0 +1,90 @@ +package pebble + +import ( + "path/filepath" + "testing" + + "github.com/cockroachdb/pebble" + "github.com/dgraph-io/badger/v2" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/vmihailenco/msgpack" + + "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/module/metrics" + "github.com/onflow/flow-go/storage" + badgerstorage "github.com/onflow/flow-go/storage/badger" + "github.com/onflow/flow-go/utils/unittest" +) + +func TestMsgPacks(t *testing.T) { + chunkDataPacks := unittest.ChunkDataPacksFixture(10) + for _, chunkDataPack := range chunkDataPacks { + sc := storage.ToStoredChunkDataPack(chunkDataPack) + value, err := msgpack.Marshal(sc) + require.NoError(t, err) + + var actual storage.StoredChunkDataPack + err = msgpack.Unmarshal(value, &actual) + require.NoError(t, err) + + require.Equal(t, *sc, actual) + } +} + +// TestChunkDataPacks_Store evaluates correct storage and retrieval of chunk data packs in the storage. +// It also evaluates that re-inserting is idempotent. +func TestChunkDataPacks_Store(t *testing.T) { + WithChunkDataPacks(t, 100, func(t *testing.T, chunkDataPacks []*flow.ChunkDataPack, chunkDataPackStore *ChunkDataPacks, _ *pebble.DB) { + require.NoError(t, chunkDataPackStore.Store(chunkDataPacks)) + require.NoError(t, chunkDataPackStore.Store(chunkDataPacks)) + }) +} + +// WithChunkDataPacks is a test helper that generates specified number of chunk data packs, store them using the storeFunc, and +// then evaluates whether they are successfully retrieved from storage. +func WithChunkDataPacks(t *testing.T, chunks int, storeFunc func(*testing.T, []*flow.ChunkDataPack, *ChunkDataPacks, *pebble.DB)) { + RunWithBadgerDBAndPebbleDB(t, func(badgerDB *badger.DB, db *pebble.DB) { + transactions := badgerstorage.NewTransactions(&metrics.NoopCollector{}, badgerDB) + collections := badgerstorage.NewCollections(badgerDB, transactions) + // keep the cache size at 1 to make sure that entries are written and read from storage itself. + store := NewChunkDataPacks(db, collections) + + chunkDataPacks := unittest.ChunkDataPacksFixture(chunks) + for _, chunkDataPack := range chunkDataPacks { + // stores collection in Collections storage (which ChunkDataPacks store uses internally) + err := collections.Store(chunkDataPack.Collection) + require.NoError(t, err) + } + + // stores chunk data packs in the memory using provided store function. + storeFunc(t, chunkDataPacks, store, db) + + // stored chunk data packs should be retrieved successfully. + for _, expected := range chunkDataPacks { + actual, err := store.ByChunkID(expected.ChunkID) + require.NoError(t, err) + + assert.Equal(t, expected, actual) + } + }) +} + +func RunWithBadgerDBAndPebbleDB(t *testing.T, fn func(*badger.DB, *pebble.DB)) { + unittest.RunWithTempDir(t, func(dir string) { + badgerDB := unittest.BadgerDB(t, filepath.Join(dir, "badger")) + defer func() { + require.NoError(t, badgerDB.Close()) + }() + + cache := pebble.NewCache(1 << 20) + defer cache.Unref() + // currently pebble is only used for registers + opts := DefaultPebbleOptions(cache, pebble.DefaultComparer) + pebbledb, err := pebble.Open(filepath.Join(dir, "pebble"), opts) + require.NoError(t, err) + defer pebbledb.Close() + + fn(badgerDB, pebbledb) + }) +} From fc774476f32d8d095cbc74faa33dd7bb07035a00 Mon Sep 17 00:00:00 2001 From: "Leo Zhang (zhangchiqing)" Date: Wed, 5 Jun 2024 16:13:48 -0700 Subject: [PATCH 21/60] use pebble based chunk data pack storage --- cmd/execution_builder.go | 59 ++++++++++++++++++++++++---------------- 1 file changed, 35 insertions(+), 24 deletions(-) diff --git a/cmd/execution_builder.go b/cmd/execution_builder.go index 3841f70f5f5..ec7dbdbe9a2 100644 --- a/cmd/execution_builder.go +++ b/cmd/execution_builder.go @@ -13,7 +13,7 @@ import ( awsconfig "github.com/aws/aws-sdk-go-v2/config" "github.com/aws/aws-sdk-go-v2/service/s3" - badgerDB "github.com/dgraph-io/badger/v2" + "github.com/cockroachdb/pebble" "github.com/ipfs/boxo/bitswap" "github.com/ipfs/go-cid" badger "github.com/ipfs/go-ds-badger2" @@ -94,7 +94,6 @@ import ( storage "github.com/onflow/flow-go/storage/badger" "github.com/onflow/flow-go/storage/badger/procedure" storagepebble "github.com/onflow/flow-go/storage/pebble" - sutil "github.com/onflow/flow-go/storage/util" ) const ( @@ -691,28 +690,39 @@ func (exeNode *ExecutionNode) LoadExecutionDataGetter(node *NodeConfig) error { return nil } -func openChunkDataPackDB(dbPath string, logger zerolog.Logger) (*badgerDB.DB, error) { - log := sutil.NewLogger(logger) - - opts := badgerDB. - DefaultOptions(dbPath). - WithKeepL0InMemory(true). - WithLogger(log). - - // the ValueLogFileSize option specifies how big the value of a - // key-value pair is allowed to be saved into badger. - // exceeding this limit, will fail with an error like this: - // could not store data: Value with size exceeded 1073741824 limit - // Maximum value size is 10G, needed by execution node - // TODO: finding a better max value for each node type - WithValueLogFileSize(256 << 23). - WithValueLogMaxEntries(100000) // Default is 1000000 - - db, err := badgerDB.Open(opts) +// func openChunkDataPackDB(dbPath string, logger zerolog.Logger) (*badgerDB.DB, error) { +// log := sutil.NewLogger(logger) +// +// opts := badgerDB. +// DefaultOptions(dbPath). +// WithKeepL0InMemory(true). +// WithLogger(log). +// +// // the ValueLogFileSize option specifies how big the value of a +// // key-value pair is allowed to be saved into badger. +// // exceeding this limit, will fail with an error like this: +// // could not store data: Value with size exceeded 1073741824 limit +// // Maximum value size is 10G, needed by execution node +// // TODO: finding a better max value for each node type +// WithValueLogFileSize(256 << 23). +// WithValueLogMaxEntries(100000) // Default is 1000000 +// +// db, err := badgerDB.Open(opts) +// if err != nil { +// return nil, fmt.Errorf("could not open chunk data pack badger db at path %v: %w", dbPath, err) +// } +// return db, nil +// } + +func openPebbleChunkDataPackDB(dbPath string, logger zerolog.Logger) (*pebble.DB, error) { + cache := pebble.NewCache(1 << 20) + defer cache.Unref() + opts := storagepebble.DefaultPebbleOptions(cache, pebble.DefaultComparer) + pebbledb, err := pebble.Open(dbPath, opts) if err != nil { - return nil, fmt.Errorf("could not open chunk data pack badger db at path %v: %w", dbPath, err) + return nil, fmt.Errorf("could not open chunk data pack pebble db at path %v: %w", dbPath, err) } - return db, nil + return pebbledb, nil } func (exeNode *ExecutionNode) LoadExecutionState( @@ -722,7 +732,7 @@ func (exeNode *ExecutionNode) LoadExecutionState( error, ) { - chunkDataPackDB, err := openChunkDataPackDB(exeNode.exeConf.chunkDataPackDir, node.Logger) + chunkDataPackDB, err := openPebbleChunkDataPackDB(exeNode.exeConf.chunkDataPackDir, node.Logger) if err != nil { return nil, err } @@ -732,7 +742,8 @@ func (exeNode *ExecutionNode) LoadExecutionState( } return nil }) - chunkDataPacks := storage.NewChunkDataPacks(node.Metrics.Cache, chunkDataPackDB, node.Storage.Collections, exeNode.exeConf.chunkDataPackCacheSize) + // chunkDataPacks := storage.NewChunkDataPacks(node.Metrics.Cache, chunkDataPackDB, node.Storage.Collections, exeNode.exeConf.chunkDataPackCacheSize) + chunkDataPacks := storagepebble.NewChunkDataPacks(chunkDataPackDB, node.Storage.Collections) // Needed for gRPC server, make sure to assign to main scoped vars exeNode.events = storage.NewEvents(node.Metrics.Cache, node.DB) From b511b0a6f1097139d138ad706c040cfb4ccd326c Mon Sep 17 00:00:00 2001 From: "Leo Zhang (zhangchiqing)" Date: Thu, 6 Jun 2024 15:44:48 -0700 Subject: [PATCH 22/60] tmp --- cmd/execution_builder.go | 17 +++-------------- storage/pebble/chunk_data_packs.go | 28 ++++++++++++++++++++++++++-- storage/pebble/open.go | 16 +++++++++++++++- 3 files changed, 44 insertions(+), 17 deletions(-) diff --git a/cmd/execution_builder.go b/cmd/execution_builder.go index ec7dbdbe9a2..5dcb36fe20f 100644 --- a/cmd/execution_builder.go +++ b/cmd/execution_builder.go @@ -13,7 +13,6 @@ import ( awsconfig "github.com/aws/aws-sdk-go-v2/config" "github.com/aws/aws-sdk-go-v2/service/s3" - "github.com/cockroachdb/pebble" "github.com/ipfs/boxo/bitswap" "github.com/ipfs/go-cid" badger "github.com/ipfs/go-ds-badger2" @@ -714,17 +713,6 @@ func (exeNode *ExecutionNode) LoadExecutionDataGetter(node *NodeConfig) error { // return db, nil // } -func openPebbleChunkDataPackDB(dbPath string, logger zerolog.Logger) (*pebble.DB, error) { - cache := pebble.NewCache(1 << 20) - defer cache.Unref() - opts := storagepebble.DefaultPebbleOptions(cache, pebble.DefaultComparer) - pebbledb, err := pebble.Open(dbPath, opts) - if err != nil { - return nil, fmt.Errorf("could not open chunk data pack pebble db at path %v: %w", dbPath, err) - } - return pebbledb, nil -} - func (exeNode *ExecutionNode) LoadExecutionState( node *NodeConfig, ) ( @@ -732,10 +720,11 @@ func (exeNode *ExecutionNode) LoadExecutionState( error, ) { - chunkDataPackDB, err := openPebbleChunkDataPackDB(exeNode.exeConf.chunkDataPackDir, node.Logger) + chunkDataPackDB, err := storagepebble.OpenDefaultPebbleDB(exeNode.exeConf.chunkDataPackDir) if err != nil { - return nil, err + return nil, fmt.Errorf("could not open chunk data pack database: %w", err) } + exeNode.builder.ShutdownFunc(func() error { if err := chunkDataPackDB.Close(); err != nil { return fmt.Errorf("error closing chunk data pack database: %w", err) diff --git a/storage/pebble/chunk_data_packs.go b/storage/pebble/chunk_data_packs.go index 4573db65e79..e8e6e311db0 100644 --- a/storage/pebble/chunk_data_packs.go +++ b/storage/pebble/chunk_data_packs.go @@ -4,21 +4,45 @@ import ( "fmt" "github.com/cockroachdb/pebble" + "github.com/dgraph-io/badger" "github.com/vmihailenco/msgpack" "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/module/irrecoverable" + "github.com/onflow/flow-go/module/metrics" "github.com/onflow/flow-go/storage" + "github.com/onflow/flow-go/storage/badger/operation" + "github.com/onflow/flow-go/storage/badger/transaction" ) type ChunkDataPacks struct { - db *pebble.DB - collections storage.Collections + db *pebble.DB + collections storage.Collections + byChunkIDCache *Cache[flow.Identifier, *storage.StoredChunkDataPack] } var _ storage.ChunkDataPacks = (*ChunkDataPacks)(nil) func NewChunkDataPacks(db *pebble.DB, collections storage.Collections) *ChunkDataPacks { + + store := func(key flow.Identifier, val *storage.StoredChunkDataPack) func(*transaction.Tx) error { + return transaction.WithTx(operation.SkipDuplicates(operation.InsertChunkDataPack(val))) + } + + retrieve := func(key flow.Identifier) func(tx *badger.Txn) (*storage.StoredChunkDataPack, error) { + return func(tx *badger.Txn) (*storage.StoredChunkDataPack, error) { + var c storage.StoredChunkDataPack + err := operation.RetrieveChunkDataPack(key, &c)(tx) + return &c, err + } + } + + cache := newCache(collector, metrics.ResourceChunkDataPack, + withLimit[flow.Identifier, *storage.StoredChunkDataPack](byChunkIDCacheSize), + withStore(store), + withRetrieve(retrieve), + ) + return &ChunkDataPacks{ db: db, collections: collections, diff --git a/storage/pebble/open.go b/storage/pebble/open.go index a0d7ea6c0d5..c17756949ca 100644 --- a/storage/pebble/open.go +++ b/storage/pebble/open.go @@ -12,6 +12,8 @@ import ( "github.com/onflow/flow-go/storage/pebble/registers" ) +const DefaultPebbleCacheSize = 1 << 20 + // NewBootstrappedRegistersWithPath initializes a new Registers instance with a pebble db // if the database is not initialized, it close the database and return storage.ErrNotBootstrapped func NewBootstrappedRegistersWithPath(dir string) (*Registers, *pebble.DB, error) { @@ -35,7 +37,7 @@ func NewBootstrappedRegistersWithPath(dir string) (*Registers, *pebble.DB, error // OpenRegisterPebbleDB opens the database func OpenRegisterPebbleDB(dir string) (*pebble.DB, error) { - cache := pebble.NewCache(1 << 20) + cache := pebble.NewCache(DefaultPebbleCacheSize) defer cache.Unref() // currently pebble is only used for registers opts := DefaultPebbleOptions(cache, registers.NewMVCCComparer()) @@ -47,6 +49,18 @@ func OpenRegisterPebbleDB(dir string) (*pebble.DB, error) { return db, nil } +func OpenDefaultPebbleDB(dir string) (*pebble.DB, error) { + cache := pebble.NewCache(DefaultPebbleCacheSize) + defer cache.Unref() + opts := DefaultPebbleOptions(cache, pebble.DefaultComparer) + db, err := pebble.Open(dir, opts) + if err != nil { + return nil, fmt.Errorf("failed to open db: %w", err) + } + + return db, nil +} + // ReadHeightsFromBootstrappedDB reads the first and latest height from a bootstrapped register db // If the register db is not bootstrapped, it returns storage.ErrNotBootstrapped // If the register db is corrupted, it returns an error From 7efc768e95421a810a7a89ce2dd5a204d5e0f2c7 Mon Sep 17 00:00:00 2001 From: "Leo Zhang (zhangchiqing)" Date: Fri, 7 Jun 2024 11:41:23 -0700 Subject: [PATCH 23/60] implement pebble chunk data pack with cache --- cmd/execution_builder.go | 2 +- storage/pebble/chunk_data_packs.go | 119 ++++++++++++------- storage/pebble/chunk_data_packs_test.go | 50 ++++++-- storage/pebble/value_cache.go | 150 ++++++++++++++++++++++++ 4 files changed, 265 insertions(+), 56 deletions(-) create mode 100644 storage/pebble/value_cache.go diff --git a/cmd/execution_builder.go b/cmd/execution_builder.go index 5dcb36fe20f..10c34aa804b 100644 --- a/cmd/execution_builder.go +++ b/cmd/execution_builder.go @@ -732,7 +732,7 @@ func (exeNode *ExecutionNode) LoadExecutionState( return nil }) // chunkDataPacks := storage.NewChunkDataPacks(node.Metrics.Cache, chunkDataPackDB, node.Storage.Collections, exeNode.exeConf.chunkDataPackCacheSize) - chunkDataPacks := storagepebble.NewChunkDataPacks(chunkDataPackDB, node.Storage.Collections) + chunkDataPacks := storagepebble.NewChunkDataPacks(node.Metrics.Cache, chunkDataPackDB, node.Storage.Collections, exeNode.exeConf.chunkDataPackCacheSize) // Needed for gRPC server, make sure to assign to main scoped vars exeNode.events = storage.NewEvents(node.Metrics.Cache, node.DB) diff --git a/storage/pebble/chunk_data_packs.go b/storage/pebble/chunk_data_packs.go index e8e6e311db0..0b200af893b 100644 --- a/storage/pebble/chunk_data_packs.go +++ b/storage/pebble/chunk_data_packs.go @@ -4,15 +4,13 @@ import ( "fmt" "github.com/cockroachdb/pebble" - "github.com/dgraph-io/badger" "github.com/vmihailenco/msgpack" "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/module" "github.com/onflow/flow-go/module/irrecoverable" "github.com/onflow/flow-go/module/metrics" "github.com/onflow/flow-go/storage" - "github.com/onflow/flow-go/storage/badger/operation" - "github.com/onflow/flow-go/storage/badger/transaction" ) type ChunkDataPacks struct { @@ -23,40 +21,39 @@ type ChunkDataPacks struct { var _ storage.ChunkDataPacks = (*ChunkDataPacks)(nil) -func NewChunkDataPacks(db *pebble.DB, collections storage.Collections) *ChunkDataPacks { +func NewChunkDataPacks(collector module.CacheMetrics, db *pebble.DB, collections storage.Collections, byChunkIDCacheSize uint) *ChunkDataPacks { - store := func(key flow.Identifier, val *storage.StoredChunkDataPack) func(*transaction.Tx) error { - return transaction.WithTx(operation.SkipDuplicates(operation.InsertChunkDataPack(val))) - } - - retrieve := func(key flow.Identifier) func(tx *badger.Txn) (*storage.StoredChunkDataPack, error) { - return func(tx *badger.Txn) (*storage.StoredChunkDataPack, error) { + retrieve := func(key flow.Identifier) func(pebble.Reader) (*storage.StoredChunkDataPack, error) { + return func(r pebble.Reader) (*storage.StoredChunkDataPack, error) { var c storage.StoredChunkDataPack - err := operation.RetrieveChunkDataPack(key, &c)(tx) + err := RetrieveChunkDataPack(key, &c)(r) return &c, err } } cache := newCache(collector, metrics.ResourceChunkDataPack, withLimit[flow.Identifier, *storage.StoredChunkDataPack](byChunkIDCacheSize), - withStore(store), withRetrieve(retrieve), ) return &ChunkDataPacks{ - db: db, - collections: collections, + db: db, + collections: collections, + byChunkIDCache: cache, } } func (ch *ChunkDataPacks) Store(cs []*flow.ChunkDataPack) error { batch := ch.db.NewBatch() defer batch.Close() + + scs := make([]*storage.StoredChunkDataPack, 0, len(cs)) for _, c := range cs { - err := ch.batchStore(c, batch) + sc, err := ch.batchStore(c, batch) if err != nil { return fmt.Errorf("cannot store chunk data pack: %w", err) } + scs = append(scs, sc) } err := batch.Commit(pebble.Sync) @@ -64,16 +61,39 @@ func (ch *ChunkDataPacks) Store(cs []*flow.ChunkDataPack) error { return fmt.Errorf("cannot commit batch: %w", err) } + // TODO: move to batchStore + for _, sc := range scs { + ch.byChunkIDCache.Insert(sc.ChunkID, sc) + } + return nil } func (ch *ChunkDataPacks) Remove(cs []flow.Identifier) error { + batch := ch.db.NewBatch() + + for _, c := range cs { + err := ch.batchRemove(c, batch) + if err != nil { + return fmt.Errorf("cannot remove chunk data pack: %w", err) + } + } + + err := batch.Commit(pebble.Sync) + if err != nil { + return fmt.Errorf("cannot commit batch: %w", err) + } + + for _, c := range cs { + ch.byChunkIDCache.Remove(c) + } + return nil } func (ch *ChunkDataPacks) ByChunkID(chunkID flow.Identifier) (*flow.ChunkDataPack, error) { var sc storage.StoredChunkDataPack - err := RetrieveChunkDataPack(ch.db, chunkID, &sc) + err := RetrieveChunkDataPack(chunkID, &sc)(ch.db) if err != nil { return nil, fmt.Errorf("could not retrieve stored chunk data pack: %w", err) } @@ -97,50 +117,63 @@ func (ch *ChunkDataPacks) ByChunkID(chunkID flow.Identifier) (*flow.ChunkDataPac } func (ch *ChunkDataPacks) BatchRemove(chunkID flow.Identifier, batch storage.BatchStorage) error { - return nil + return fmt.Errorf("not implemented") } -func (ch *ChunkDataPacks) batchStore(c *flow.ChunkDataPack, batch *pebble.Batch) error { +func (ch *ChunkDataPacks) batchRemove(chunkID flow.Identifier, batch pebble.Writer) error { + return batch.Delete(makeKey(codeChunkDataPack, chunkID), nil) +} + +func (ch *ChunkDataPacks) batchStore(c *flow.ChunkDataPack, batch *pebble.Batch) (*storage.StoredChunkDataPack, error) { sc := storage.ToStoredChunkDataPack(c) - return InsertChunkDataPack(batch, sc) + err := InsertChunkDataPack(sc)(batch) + if err != nil { + return nil, fmt.Errorf("failed to store chunk data pack: %w", err) + } + return sc, nil } -func InsertChunkDataPack(batch *pebble.Batch, sc *storage.StoredChunkDataPack) error { +// TODO: move to operation package +func InsertChunkDataPack(sc *storage.StoredChunkDataPack) func(w pebble.Writer) error { key := makeKey(codeChunkDataPack, sc.ChunkID) - return batchWrite(batch, key, sc) + return insert(key, sc) } -func RetrieveChunkDataPack(db *pebble.DB, chunkID flow.Identifier, sc *storage.StoredChunkDataPack) error { +func RetrieveChunkDataPack(chunkID flow.Identifier, sc *storage.StoredChunkDataPack) func(r pebble.Reader) error { key := makeKey(codeChunkDataPack, chunkID) - return retrieve(db, key, sc) + return retrieve(key, sc) } -func batchWrite(batch *pebble.Batch, key []byte, val interface{}) error { - value, err := msgpack.Marshal(val) - if err != nil { - return irrecoverable.NewExceptionf("failed to encode value: %w", err) - } +func insert(key []byte, val interface{}) func(pebble.Writer) error { + return func(w pebble.Writer) error { + value, err := msgpack.Marshal(val) + if err != nil { + return irrecoverable.NewExceptionf("failed to encode value: %w", err) + } - err = batch.Set(key, value, nil) - if err != nil { - return irrecoverable.NewExceptionf("failed to store data: %w", err) - } + err = w.Set(key, value, nil) + if err != nil { + return irrecoverable.NewExceptionf("failed to store data: %w", err) + } - return nil + return nil + } } -func retrieve(db *pebble.DB, key []byte, sc interface{}) error { - val, closer, err := db.Get(key) - if err != nil { - return convertNotFoundError(err) - } - defer closer.Close() +func retrieve(key []byte, sc interface{}) func(r pebble.Reader) error { + return func(r pebble.Reader) error { + val, closer, err := r.Get(key) + if err != nil { + return convertNotFoundError(err) + } + defer closer.Close() - err = msgpack.Unmarshal(val, &sc) - if err != nil { - return irrecoverable.NewExceptionf("failed to decode value: %w", err) + err = msgpack.Unmarshal(val, &sc) + if err != nil { + return irrecoverable.NewExceptionf("failed to decode value: %w", err) + } + return nil } - return nil } const ( diff --git a/storage/pebble/chunk_data_packs_test.go b/storage/pebble/chunk_data_packs_test.go index 79d6c5bb908..f170b22114c 100644 --- a/storage/pebble/chunk_data_packs_test.go +++ b/storage/pebble/chunk_data_packs_test.go @@ -1,12 +1,12 @@ package pebble import ( + "errors" "path/filepath" "testing" "github.com/cockroachdb/pebble" "github.com/dgraph-io/badger/v2" - "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/vmihailenco/msgpack" @@ -35,20 +35,54 @@ func TestMsgPacks(t *testing.T) { // TestChunkDataPacks_Store evaluates correct storage and retrieval of chunk data packs in the storage. // It also evaluates that re-inserting is idempotent. func TestChunkDataPacks_Store(t *testing.T) { - WithChunkDataPacks(t, 100, func(t *testing.T, chunkDataPacks []*flow.ChunkDataPack, chunkDataPackStore *ChunkDataPacks, _ *pebble.DB) { + WithChunkDataPacks(t, 100, func(t *testing.T, chunkDataPacks []*flow.ChunkDataPack, chunkDataPackStore storage.ChunkDataPacks, _ *pebble.DB) { + // can store require.NoError(t, chunkDataPackStore.Store(chunkDataPacks)) + + // can read back + for _, c := range chunkDataPacks { + c2, err := chunkDataPackStore.ByChunkID(c.ChunkID) + require.NoError(t, err) + require.Equal(t, c, c2) + } + + // can store again require.NoError(t, chunkDataPackStore.Store(chunkDataPacks)) + + cids := make([]flow.Identifier, 0, len(chunkDataPacks)) + for i, c := range chunkDataPacks { + // remove everything except the first one + if i > 0 { + cids = append(cids, c.ChunkID) + } + } + // can remove + require.NoError(t, chunkDataPackStore.Remove(cids)) + for i, c := range chunkDataPacks { + if i == 0 { + // the first one is not removed + _, err := chunkDataPackStore.ByChunkID(c.ChunkID) + require.NoError(t, err) + continue + } + // the rest are removed + _, err := chunkDataPackStore.ByChunkID(c.ChunkID) + require.True(t, errors.Is(err, storage.ErrNotFound)) + } + + // can remove again + require.NoError(t, chunkDataPackStore.Remove(cids)) }) } // WithChunkDataPacks is a test helper that generates specified number of chunk data packs, store them using the storeFunc, and // then evaluates whether they are successfully retrieved from storage. -func WithChunkDataPacks(t *testing.T, chunks int, storeFunc func(*testing.T, []*flow.ChunkDataPack, *ChunkDataPacks, *pebble.DB)) { +func WithChunkDataPacks(t *testing.T, chunks int, storeFunc func(*testing.T, []*flow.ChunkDataPack, storage.ChunkDataPacks, *pebble.DB)) { RunWithBadgerDBAndPebbleDB(t, func(badgerDB *badger.DB, db *pebble.DB) { transactions := badgerstorage.NewTransactions(&metrics.NoopCollector{}, badgerDB) collections := badgerstorage.NewCollections(badgerDB, transactions) // keep the cache size at 1 to make sure that entries are written and read from storage itself. - store := NewChunkDataPacks(db, collections) + store := NewChunkDataPacks(&metrics.NoopCollector{}, db, collections, 1) chunkDataPacks := unittest.ChunkDataPacksFixture(chunks) for _, chunkDataPack := range chunkDataPacks { @@ -59,14 +93,6 @@ func WithChunkDataPacks(t *testing.T, chunks int, storeFunc func(*testing.T, []* // stores chunk data packs in the memory using provided store function. storeFunc(t, chunkDataPacks, store, db) - - // stored chunk data packs should be retrieved successfully. - for _, expected := range chunkDataPacks { - actual, err := store.ByChunkID(expected.ChunkID) - require.NoError(t, err) - - assert.Equal(t, expected, actual) - } }) } diff --git a/storage/pebble/value_cache.go b/storage/pebble/value_cache.go new file mode 100644 index 00000000000..64ca2c91a81 --- /dev/null +++ b/storage/pebble/value_cache.go @@ -0,0 +1,150 @@ +package pebble + +import ( + "errors" + "fmt" + + "github.com/cockroachdb/pebble" + lru "github.com/hashicorp/golang-lru/v2" + + "github.com/onflow/flow-go/module" + "github.com/onflow/flow-go/storage" +) + +func withLimit[K comparable, V any](limit uint) func(*Cache[K, V]) { + return func(c *Cache[K, V]) { + c.limit = limit + } +} + +type storeFunc[K comparable, V any] func(key K, val V) func(pebble.Writer) error + +func withStore[K comparable, V any](store storeFunc[K, V]) func(*Cache[K, V]) { + return func(c *Cache[K, V]) { + c.store = store + } +} + +func noStore[K comparable, V any](_ K, _ V) func(pebble.Writer) error { + return func(pebble.Writer) error { + return fmt.Errorf("no store function for cache put available") + } +} + +func noopStore[K comparable, V any](_ K, _ V) func(pebble.Reader) error { + return func(pebble.Reader) error { + return nil + } +} + +type retrieveFunc[K comparable, V any] func(key K) func(pebble.Reader) (V, error) + +func withRetrieve[K comparable, V any](retrieve retrieveFunc[K, V]) func(*Cache[K, V]) { + return func(c *Cache[K, V]) { + c.retrieve = retrieve + } +} + +func noRetrieve[K comparable, V any](_ K) func(pebble.Reader) (V, error) { + return func(pebble.Reader) (V, error) { + var nullV V + return nullV, fmt.Errorf("no retrieve function for cache get available") + } +} + +type Cache[K comparable, V any] struct { + metrics module.CacheMetrics + limit uint + store storeFunc[K, V] + retrieve retrieveFunc[K, V] + resource string + cache *lru.Cache[K, V] +} + +func newCache[K comparable, V any](collector module.CacheMetrics, resourceName string, options ...func(*Cache[K, V])) *Cache[K, V] { + c := Cache[K, V]{ + metrics: collector, + limit: 1000, + store: noStore[K, V], + retrieve: noRetrieve[K, V], + resource: resourceName, + } + for _, option := range options { + option(&c) + } + c.cache, _ = lru.New[K, V](int(c.limit)) + c.metrics.CacheEntries(c.resource, uint(c.cache.Len())) + return &c +} + +// IsCached returns true if the key exists in the cache. +// It DOES NOT check whether the key exists in the underlying data store. +func (c *Cache[K, V]) IsCached(key K) bool { + return c.cache.Contains(key) +} + +// Get will try to retrieve the resource from cache first, and then from the +// injected. During normal operations, the following error returns are expected: +// - `storage.ErrNotFound` if key is unknown. +func (c *Cache[K, V]) Get(key K) func(pebble.Reader) (V, error) { + return func(r pebble.Reader) (V, error) { + + // check if we have it in the cache + resource, cached := c.cache.Get(key) + if cached { + c.metrics.CacheHit(c.resource) + return resource, nil + } + + // get it from the database + resource, err := c.retrieve(key)(r) + if err != nil { + if errors.Is(err, storage.ErrNotFound) { + c.metrics.CacheNotFound(c.resource) + } + var nullV V + return nullV, fmt.Errorf("could not retrieve resource: %w", err) + } + + c.metrics.CacheMiss(c.resource) + + // cache the resource and eject least recently used one if we reached limit + evicted := c.cache.Add(key, resource) + if !evicted { + c.metrics.CacheEntries(c.resource, uint(c.cache.Len())) + } + + return resource, nil + } +} + +func (c *Cache[K, V]) Remove(key K) { + c.cache.Remove(key) +} + +// Insert will add a resource directly to the cache with the given ID +// assuming the resource has been added to storage already. +func (c *Cache[K, V]) Insert(key K, resource V) { + // cache the resource and eject least recently used one if we reached limit + evicted := c.cache.Add(key, resource) + if !evicted { + c.metrics.CacheEntries(c.resource, uint(c.cache.Len())) + } +} + +// PutTx will return tx which adds a resource to the cache with the given ID. +func (c *Cache[K, V]) PutTx(key K, resource V) func(pebble.Writer) error { + storeOps := c.store(key, resource) // assemble DB operations to store resource (no execution) + + return func(w pebble.Writer) error { + // the storeOps must be sync operation + err := storeOps(w) // execute operations to store resource + if err != nil { + return fmt.Errorf("could not store resource: %w", err) + } + + c.Insert(key, resource) + + return nil + } +} From 86e6267666475ca12bac231c47bfe9a39591fceb Mon Sep 17 00:00:00 2001 From: "Leo Zhang (zhangchiqing)" Date: Fri, 7 Jun 2024 11:46:57 -0700 Subject: [PATCH 24/60] move modules to operations package --- storage/pebble/chunk_data_packs.go | 62 ++------------------ storage/pebble/operations/chunk_data_pack.go | 25 ++++++++ storage/pebble/operations/codes.go | 5 ++ storage/pebble/operations/common.go | 55 +++++++++++++++++ 4 files changed, 90 insertions(+), 57 deletions(-) create mode 100644 storage/pebble/operations/chunk_data_pack.go create mode 100644 storage/pebble/operations/codes.go create mode 100644 storage/pebble/operations/common.go diff --git a/storage/pebble/chunk_data_packs.go b/storage/pebble/chunk_data_packs.go index 0b200af893b..68632dc8ec7 100644 --- a/storage/pebble/chunk_data_packs.go +++ b/storage/pebble/chunk_data_packs.go @@ -4,13 +4,12 @@ import ( "fmt" "github.com/cockroachdb/pebble" - "github.com/vmihailenco/msgpack" "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/module" - "github.com/onflow/flow-go/module/irrecoverable" "github.com/onflow/flow-go/module/metrics" "github.com/onflow/flow-go/storage" + "github.com/onflow/flow-go/storage/pebble/operations" ) type ChunkDataPacks struct { @@ -26,7 +25,7 @@ func NewChunkDataPacks(collector module.CacheMetrics, db *pebble.DB, collections retrieve := func(key flow.Identifier) func(pebble.Reader) (*storage.StoredChunkDataPack, error) { return func(r pebble.Reader) (*storage.StoredChunkDataPack, error) { var c storage.StoredChunkDataPack - err := RetrieveChunkDataPack(key, &c)(r) + err := operations.RetrieveChunkDataPack(key, &c)(r) return &c, err } } @@ -93,7 +92,7 @@ func (ch *ChunkDataPacks) Remove(cs []flow.Identifier) error { func (ch *ChunkDataPacks) ByChunkID(chunkID flow.Identifier) (*flow.ChunkDataPack, error) { var sc storage.StoredChunkDataPack - err := RetrieveChunkDataPack(chunkID, &sc)(ch.db) + err := operations.RetrieveChunkDataPack(chunkID, &sc)(ch.db) if err != nil { return nil, fmt.Errorf("could not retrieve stored chunk data pack: %w", err) } @@ -121,65 +120,14 @@ func (ch *ChunkDataPacks) BatchRemove(chunkID flow.Identifier, batch storage.Bat } func (ch *ChunkDataPacks) batchRemove(chunkID flow.Identifier, batch pebble.Writer) error { - return batch.Delete(makeKey(codeChunkDataPack, chunkID), nil) + return operations.RemoveChunkDataPack(chunkID)(batch) } func (ch *ChunkDataPacks) batchStore(c *flow.ChunkDataPack, batch *pebble.Batch) (*storage.StoredChunkDataPack, error) { sc := storage.ToStoredChunkDataPack(c) - err := InsertChunkDataPack(sc)(batch) + err := operations.InsertChunkDataPack(sc)(batch) if err != nil { return nil, fmt.Errorf("failed to store chunk data pack: %w", err) } return sc, nil } - -// TODO: move to operation package -func InsertChunkDataPack(sc *storage.StoredChunkDataPack) func(w pebble.Writer) error { - key := makeKey(codeChunkDataPack, sc.ChunkID) - return insert(key, sc) -} - -func RetrieveChunkDataPack(chunkID flow.Identifier, sc *storage.StoredChunkDataPack) func(r pebble.Reader) error { - key := makeKey(codeChunkDataPack, chunkID) - return retrieve(key, sc) -} - -func insert(key []byte, val interface{}) func(pebble.Writer) error { - return func(w pebble.Writer) error { - value, err := msgpack.Marshal(val) - if err != nil { - return irrecoverable.NewExceptionf("failed to encode value: %w", err) - } - - err = w.Set(key, value, nil) - if err != nil { - return irrecoverable.NewExceptionf("failed to store data: %w", err) - } - - return nil - } -} - -func retrieve(key []byte, sc interface{}) func(r pebble.Reader) error { - return func(r pebble.Reader) error { - val, closer, err := r.Get(key) - if err != nil { - return convertNotFoundError(err) - } - defer closer.Close() - - err = msgpack.Unmarshal(val, &sc) - if err != nil { - return irrecoverable.NewExceptionf("failed to decode value: %w", err) - } - return nil - } -} - -const ( - codeChunkDataPack = 100 -) - -func makeKey(prefix byte, chunkID flow.Identifier) []byte { - return append([]byte{prefix}, chunkID[:]...) -} diff --git a/storage/pebble/operations/chunk_data_pack.go b/storage/pebble/operations/chunk_data_pack.go new file mode 100644 index 00000000000..e4ede3dd9e3 --- /dev/null +++ b/storage/pebble/operations/chunk_data_pack.go @@ -0,0 +1,25 @@ +package operations + +import ( + "github.com/cockroachdb/pebble" + + "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/storage" +) + +func InsertChunkDataPack(sc *storage.StoredChunkDataPack) func(w pebble.Writer) error { + key := makeKey(codeChunkDataPack, sc.ChunkID) + return insert(key, sc) +} + +func RetrieveChunkDataPack(chunkID flow.Identifier, sc *storage.StoredChunkDataPack) func(r pebble.Reader) error { + key := makeKey(codeChunkDataPack, chunkID) + return retrieve(key, sc) +} + +func RemoveChunkDataPack(chunkID flow.Identifier) func(w pebble.Writer) error { + key := makeKey(codeChunkDataPack, chunkID) + return func(w pebble.Writer) error { + return w.Delete(key, nil) + } +} diff --git a/storage/pebble/operations/codes.go b/storage/pebble/operations/codes.go new file mode 100644 index 00000000000..0f803e5e0c0 --- /dev/null +++ b/storage/pebble/operations/codes.go @@ -0,0 +1,5 @@ +package operations + +const ( + codeChunkDataPack = 100 +) diff --git a/storage/pebble/operations/common.go b/storage/pebble/operations/common.go new file mode 100644 index 00000000000..a093ca459c7 --- /dev/null +++ b/storage/pebble/operations/common.go @@ -0,0 +1,55 @@ +package operations + +import ( + "errors" + + "github.com/cockroachdb/pebble" + "github.com/vmihailenco/msgpack" + + "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/module/irrecoverable" + "github.com/onflow/flow-go/storage" +) + +func insert(key []byte, val interface{}) func(pebble.Writer) error { + return func(w pebble.Writer) error { + value, err := msgpack.Marshal(val) + if err != nil { + return irrecoverable.NewExceptionf("failed to encode value: %w", err) + } + + err = w.Set(key, value, nil) + if err != nil { + return irrecoverable.NewExceptionf("failed to store data: %w", err) + } + + return nil + } +} + +func retrieve(key []byte, sc interface{}) func(r pebble.Reader) error { + return func(r pebble.Reader) error { + val, closer, err := r.Get(key) + if err != nil { + return convertNotFoundError(err) + } + defer closer.Close() + + err = msgpack.Unmarshal(val, &sc) + if err != nil { + return irrecoverable.NewExceptionf("failed to decode value: %w", err) + } + return nil + } +} + +func makeKey(prefix byte, identifier flow.Identifier) []byte { + return append([]byte{prefix}, identifier[:]...) +} + +func convertNotFoundError(err error) error { + if errors.Is(err, pebble.ErrNotFound) { + return storage.ErrNotFound + } + return err +} From 03fdb3924837fa73e282c680d96a950ad9cb535b Mon Sep 17 00:00:00 2001 From: "Leo Zhang (zhangchiqing)" Date: Fri, 7 Jun 2024 12:57:59 -0700 Subject: [PATCH 25/60] extract batch --- storage/pebble/batch.go | 57 ++++++++++++++++++++++++++++++ storage/pebble/chunk_data_packs.go | 25 ++++++------- storage/pebble/value_cache.go | 22 ++++++------ 3 files changed, 78 insertions(+), 26 deletions(-) create mode 100644 storage/pebble/batch.go diff --git a/storage/pebble/batch.go b/storage/pebble/batch.go new file mode 100644 index 00000000000..e952d06c2d2 --- /dev/null +++ b/storage/pebble/batch.go @@ -0,0 +1,57 @@ +package pebble + +import ( + "sync" + + "github.com/cockroachdb/pebble" +) + +type Batch struct { + writer *pebble.Batch + + lock sync.RWMutex + callbacks []func() +} + +func NewBatch(db *pebble.DB) *Batch { + batch := db.NewBatch() + return &Batch{ + writer: batch, + callbacks: make([]func(), 0), + } +} + +func (b *Batch) GetWriter() *pebble.Batch { + return b.writer +} + +// OnSucceed adds a callback to execute after the batch has +// been successfully flushed. +// useful for implementing the cache where we will only cache +// after the batch has been successfully flushed +func (b *Batch) OnSucceed(callback func()) { + b.lock.Lock() + defer b.lock.Unlock() + b.callbacks = append(b.callbacks, callback) +} + +// Flush will call the badger Batch's Flush method, in +// addition, it will call the callbacks added by +// OnSucceed +func (b *Batch) Flush() error { + err := b.writer.Commit(nil) + if err != nil { + return err + } + + b.lock.RLock() + defer b.lock.RUnlock() + for _, callback := range b.callbacks { + callback() + } + return nil +} + +func (b *Batch) Close() error { + return b.writer.Close() +} diff --git a/storage/pebble/chunk_data_packs.go b/storage/pebble/chunk_data_packs.go index 68632dc8ec7..43e7e6cdc8b 100644 --- a/storage/pebble/chunk_data_packs.go +++ b/storage/pebble/chunk_data_packs.go @@ -43,28 +43,21 @@ func NewChunkDataPacks(collector module.CacheMetrics, db *pebble.DB, collections } func (ch *ChunkDataPacks) Store(cs []*flow.ChunkDataPack) error { - batch := ch.db.NewBatch() + batch := NewBatch(ch.db) defer batch.Close() - scs := make([]*storage.StoredChunkDataPack, 0, len(cs)) for _, c := range cs { - sc, err := ch.batchStore(c, batch) + err := ch.batchStore(c, batch) if err != nil { return fmt.Errorf("cannot store chunk data pack: %w", err) } - scs = append(scs, sc) } - err := batch.Commit(pebble.Sync) + err := batch.Flush() if err != nil { return fmt.Errorf("cannot commit batch: %w", err) } - // TODO: move to batchStore - for _, sc := range scs { - ch.byChunkIDCache.Insert(sc.ChunkID, sc) - } - return nil } @@ -123,11 +116,15 @@ func (ch *ChunkDataPacks) batchRemove(chunkID flow.Identifier, batch pebble.Writ return operations.RemoveChunkDataPack(chunkID)(batch) } -func (ch *ChunkDataPacks) batchStore(c *flow.ChunkDataPack, batch *pebble.Batch) (*storage.StoredChunkDataPack, error) { +func (ch *ChunkDataPacks) batchStore(c *flow.ChunkDataPack, batch *Batch) error { sc := storage.ToStoredChunkDataPack(c) - err := operations.InsertChunkDataPack(sc)(batch) + writer := batch.GetWriter() + batch.OnSucceed(func() { + ch.byChunkIDCache.Insert(sc.ChunkID, sc) + }) + err := operations.InsertChunkDataPack(sc)(writer) if err != nil { - return nil, fmt.Errorf("failed to store chunk data pack: %w", err) + return fmt.Errorf("failed to store chunk data pack: %w", err) } - return sc, nil + return nil } diff --git a/storage/pebble/value_cache.go b/storage/pebble/value_cache.go index 64ca2c91a81..fdf2b9044c7 100644 --- a/storage/pebble/value_cache.go +++ b/storage/pebble/value_cache.go @@ -19,24 +19,22 @@ func withLimit[K comparable, V any](limit uint) func(*Cache[K, V]) { type storeFunc[K comparable, V any] func(key K, val V) func(pebble.Writer) error -func withStore[K comparable, V any](store storeFunc[K, V]) func(*Cache[K, V]) { - return func(c *Cache[K, V]) { - c.store = store - } -} - +// func withStore[K comparable, V any](store storeFunc[K, V]) func(*Cache[K, V]) { +// return func(c *Cache[K, V]) { +// c.store = store +// } +// } func noStore[K comparable, V any](_ K, _ V) func(pebble.Writer) error { return func(pebble.Writer) error { return fmt.Errorf("no store function for cache put available") } } -func noopStore[K comparable, V any](_ K, _ V) func(pebble.Reader) error { - return func(pebble.Reader) error { - return nil - } -} - +// func noopStore[K comparable, V any](_ K, _ V) func(pebble.Reader) error { +// return func(pebble.Reader) error { +// return nil +// } +// } type retrieveFunc[K comparable, V any] func(key K) func(pebble.Reader) (V, error) func withRetrieve[K comparable, V any](retrieve retrieveFunc[K, V]) func(*Cache[K, V]) { From ad06637cee71a5fe41f6c1121fe78238c24b2636 Mon Sep 17 00:00:00 2001 From: "Leo Zhang (zhangchiqing)" Date: Fri, 7 Jun 2024 15:02:51 -0700 Subject: [PATCH 26/60] fix lint --- cmd/execution_builder.go | 48 +++++++++++++++++++++------------------- 1 file changed, 25 insertions(+), 23 deletions(-) diff --git a/cmd/execution_builder.go b/cmd/execution_builder.go index 10c34aa804b..0cac773bb43 100644 --- a/cmd/execution_builder.go +++ b/cmd/execution_builder.go @@ -13,6 +13,7 @@ import ( awsconfig "github.com/aws/aws-sdk-go-v2/config" "github.com/aws/aws-sdk-go-v2/service/s3" + badgerDB "github.com/dgraph-io/badger/v2" "github.com/ipfs/boxo/bitswap" "github.com/ipfs/go-cid" badger "github.com/ipfs/go-ds-badger2" @@ -93,6 +94,7 @@ import ( storage "github.com/onflow/flow-go/storage/badger" "github.com/onflow/flow-go/storage/badger/procedure" storagepebble "github.com/onflow/flow-go/storage/pebble" + sutil "github.com/onflow/flow-go/storage/util" ) const ( @@ -689,29 +691,29 @@ func (exeNode *ExecutionNode) LoadExecutionDataGetter(node *NodeConfig) error { return nil } -// func openChunkDataPackDB(dbPath string, logger zerolog.Logger) (*badgerDB.DB, error) { -// log := sutil.NewLogger(logger) -// -// opts := badgerDB. -// DefaultOptions(dbPath). -// WithKeepL0InMemory(true). -// WithLogger(log). -// -// // the ValueLogFileSize option specifies how big the value of a -// // key-value pair is allowed to be saved into badger. -// // exceeding this limit, will fail with an error like this: -// // could not store data: Value with size exceeded 1073741824 limit -// // Maximum value size is 10G, needed by execution node -// // TODO: finding a better max value for each node type -// WithValueLogFileSize(256 << 23). -// WithValueLogMaxEntries(100000) // Default is 1000000 -// -// db, err := badgerDB.Open(opts) -// if err != nil { -// return nil, fmt.Errorf("could not open chunk data pack badger db at path %v: %w", dbPath, err) -// } -// return db, nil -// } +func OpenChunkDataPackDB(dbPath string, logger zerolog.Logger) (*badgerDB.DB, error) { + log := sutil.NewLogger(logger) + + opts := badgerDB. + DefaultOptions(dbPath). + WithKeepL0InMemory(true). + WithLogger(log). + + // the ValueLogFileSize option specifies how big the value of a + // key-value pair is allowed to be saved into badger. + // exceeding this limit, will fail with an error like this: + // could not store data: Value with size exceeded 1073741824 limit + // Maximum value size is 10G, needed by execution node + // TODO: finding a better max value for each node type + WithValueLogFileSize(256 << 23). + WithValueLogMaxEntries(100000) // Default is 1000000 + + db, err := badgerDB.Open(opts) + if err != nil { + return nil, fmt.Errorf("could not open chunk data pack badger db at path %v: %w", dbPath, err) + } + return db, nil +} func (exeNode *ExecutionNode) LoadExecutionState( node *NodeConfig, From 7e8d0d6bde956ed33a36bd6839ce8667759a573c Mon Sep 17 00:00:00 2001 From: "Leo Zhang (zhangchiqing)" Date: Mon, 10 Jun 2024 09:06:53 -0700 Subject: [PATCH 27/60] update mock --- storage/mock/chunk_data_packs.go | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/storage/mock/chunk_data_packs.go b/storage/mock/chunk_data_packs.go index 3fbacab10d8..2db2f6dbe63 100644 --- a/storage/mock/chunk_data_packs.go +++ b/storage/mock/chunk_data_packs.go @@ -32,24 +32,6 @@ func (_m *ChunkDataPacks) BatchRemove(chunkID flow.Identifier, batch storage.Bat return r0 } -// BatchStore provides a mock function with given fields: c, batch -func (_m *ChunkDataPacks) BatchStore(c *flow.ChunkDataPack, batch storage.BatchStorage) error { - ret := _m.Called(c, batch) - - if len(ret) == 0 { - panic("no return value specified for BatchStore") - } - - var r0 error - if rf, ok := ret.Get(0).(func(*flow.ChunkDataPack, storage.BatchStorage) error); ok { - r0 = rf(c, batch) - } else { - r0 = ret.Error(0) - } - - return r0 -} - // ByChunkID provides a mock function with given fields: chunkID func (_m *ChunkDataPacks) ByChunkID(chunkID flow.Identifier) (*flow.ChunkDataPack, error) { ret := _m.Called(chunkID) From 466f9a6e1cac5b3f8650cea8651aca4f1f181557 Mon Sep 17 00:00:00 2001 From: "Leo Zhang (zhangchiqing)" Date: Mon, 10 Jun 2024 09:30:10 -0700 Subject: [PATCH 28/60] fix chunk data pack codec --- model/flow/chunk.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/model/flow/chunk.go b/model/flow/chunk.go index 90fd760c0f4..2de03307a5c 100644 --- a/model/flow/chunk.go +++ b/model/flow/chunk.go @@ -235,6 +235,9 @@ func (b *BlockExecutionDataRoot) UnmarshalMsgpack(data []byte) error { // Helper function to convert a slice of cid.Cid to a slice of strings func cidsToStrings(cids []cid.Cid) []string { + if cids == nil { + return nil + } strs := make([]string, len(cids)) for i, c := range cids { strs[i] = c.String() @@ -244,6 +247,9 @@ func cidsToStrings(cids []cid.Cid) []string { // Helper function to convert a slice of strings to a slice of cid.Cid func stringsToCids(strs []string) []cid.Cid { + if strs == nil { + return nil + } cids := make([]cid.Cid, len(strs)) for i, s := range strs { c, err := cid.Decode(s) From cde5e81fa433856afaee118f8cda6e4115dd4eee Mon Sep 17 00:00:00 2001 From: "Leo Zhang (zhangchiqing)" Date: Mon, 10 Jun 2024 16:35:24 -0700 Subject: [PATCH 29/60] add comments --- cmd/execution_builder.go | 6 ++++-- model/flow/chunk.go | 17 ++++++++++++----- storage/pebble/batch.go | 1 + storage/pebble/chunk_data_packs.go | 8 ++++++++ storage/pebble/open.go | 6 ++++++ storage/pebble/operations/chunk_data_pack.go | 6 ++++++ storage/pebble/value_cache.go | 4 ++++ 7 files changed, 41 insertions(+), 7 deletions(-) diff --git a/cmd/execution_builder.go b/cmd/execution_builder.go index 0cac773bb43..2f1795cd8e0 100644 --- a/cmd/execution_builder.go +++ b/cmd/execution_builder.go @@ -733,8 +733,10 @@ func (exeNode *ExecutionNode) LoadExecutionState( } return nil }) - // chunkDataPacks := storage.NewChunkDataPacks(node.Metrics.Cache, chunkDataPackDB, node.Storage.Collections, exeNode.exeConf.chunkDataPackCacheSize) - chunkDataPacks := storagepebble.NewChunkDataPacks(node.Metrics.Cache, chunkDataPackDB, node.Storage.Collections, exeNode.exeConf.chunkDataPackCacheSize) + // chunkDataPacks := storage.NewChunkDataPacks(node.Metrics.Cache, + // chunkDataPackDB, node.Storage.Collections, exeNode.exeConf.chunkDataPackCacheSize) + chunkDataPacks := storagepebble.NewChunkDataPacks(node.Metrics.Cache, + chunkDataPackDB, node.Storage.Collections, exeNode.exeConf.chunkDataPackCacheSize) // Needed for gRPC server, make sure to assign to main scoped vars exeNode.events = storage.NewEvents(node.Metrics.Cache, node.DB) diff --git a/model/flow/chunk.go b/model/flow/chunk.go index 2de03307a5c..83eabde4b1e 100644 --- a/model/flow/chunk.go +++ b/model/flow/chunk.go @@ -1,6 +1,7 @@ package flow import ( + "fmt" "log" "github.com/ipfs/go-cid" @@ -228,7 +229,13 @@ func (b *BlockExecutionDataRoot) UnmarshalMsgpack(data []byte) error { } b.BlockID = temp.BlockID - b.ChunkExecutionDataIDs = stringsToCids(temp.ChunkExecutionDataIDs) + cids, err := stringsToCids(temp.ChunkExecutionDataIDs) + + if err != nil { + return fmt.Errorf("failed to decode chunk execution data ids: %w", err) + } + + b.ChunkExecutionDataIDs = cids return nil } @@ -246,17 +253,17 @@ func cidsToStrings(cids []cid.Cid) []string { } // Helper function to convert a slice of strings to a slice of cid.Cid -func stringsToCids(strs []string) []cid.Cid { +func stringsToCids(strs []string) ([]cid.Cid, error) { if strs == nil { - return nil + return nil, nil } cids := make([]cid.Cid, len(strs)) for i, s := range strs { c, err := cid.Decode(s) if err != nil { - panic(err) // Handle error appropriately in real code + return nil, fmt.Errorf("failed to decode cid %v: %w", s, err) } cids[i] = c } - return cids + return cids, nil } diff --git a/storage/pebble/batch.go b/storage/pebble/batch.go index e952d06c2d2..9a45e55bc02 100644 --- a/storage/pebble/batch.go +++ b/storage/pebble/batch.go @@ -38,6 +38,7 @@ func (b *Batch) OnSucceed(callback func()) { // Flush will call the badger Batch's Flush method, in // addition, it will call the callbacks added by // OnSucceed +// any error are exceptions func (b *Batch) Flush() error { err := b.writer.Commit(nil) if err != nil { diff --git a/storage/pebble/chunk_data_packs.go b/storage/pebble/chunk_data_packs.go index 43e7e6cdc8b..73037e988ed 100644 --- a/storage/pebble/chunk_data_packs.go +++ b/storage/pebble/chunk_data_packs.go @@ -42,6 +42,8 @@ func NewChunkDataPacks(collector module.CacheMetrics, db *pebble.DB, collections } } +// Store stores the given chunk data pack lists, it stores them atomically. +// Any error are exceptions func (ch *ChunkDataPacks) Store(cs []*flow.ChunkDataPack) error { batch := NewBatch(ch.db) defer batch.Close() @@ -61,6 +63,8 @@ func (ch *ChunkDataPacks) Store(cs []*flow.ChunkDataPack) error { return nil } +// Remove removes chunk data packs by IDs, it removes them atomically. +// Any errors are exceptions func (ch *ChunkDataPacks) Remove(cs []flow.Identifier) error { batch := ch.db.NewBatch() @@ -83,6 +87,9 @@ func (ch *ChunkDataPacks) Remove(cs []flow.Identifier) error { return nil } +// ByChunkID finds the chunk data pack by chunk ID. +// it returns storage.ErrNotFound if not found +// other errors are exceptions func (ch *ChunkDataPacks) ByChunkID(chunkID flow.Identifier) (*flow.ChunkDataPack, error) { var sc storage.StoredChunkDataPack err := operations.RetrieveChunkDataPack(chunkID, &sc)(ch.db) @@ -108,6 +115,7 @@ func (ch *ChunkDataPacks) ByChunkID(chunkID flow.Identifier) (*flow.ChunkDataPac return chdp, nil } +// BatchRemove is not used in pebble implementation func (ch *ChunkDataPacks) BatchRemove(chunkID flow.Identifier, batch storage.BatchStorage) error { return fmt.Errorf("not implemented") } diff --git a/storage/pebble/open.go b/storage/pebble/open.go index c17756949ca..80f328ce87a 100644 --- a/storage/pebble/open.go +++ b/storage/pebble/open.go @@ -36,6 +36,10 @@ func NewBootstrappedRegistersWithPath(dir string) (*Registers, *pebble.DB, error } // OpenRegisterPebbleDB opens the database +// The difference between OpenDefaultPebbleDB is that it uses +// a customized comparer (NewMVCCComparer) which is needed to +// implement finding register values at any given height using +// pebble's SeekPrefixGE function func OpenRegisterPebbleDB(dir string) (*pebble.DB, error) { cache := pebble.NewCache(DefaultPebbleCacheSize) defer cache.Unref() @@ -49,6 +53,8 @@ func OpenRegisterPebbleDB(dir string) (*pebble.DB, error) { return db, nil } +// OpenDefaultPebbleDB opens a pebble database using default options, +// such as cache size and comparer func OpenDefaultPebbleDB(dir string) (*pebble.DB, error) { cache := pebble.NewCache(DefaultPebbleCacheSize) defer cache.Unref() diff --git a/storage/pebble/operations/chunk_data_pack.go b/storage/pebble/operations/chunk_data_pack.go index e4ede3dd9e3..7b93f72d61a 100644 --- a/storage/pebble/operations/chunk_data_pack.go +++ b/storage/pebble/operations/chunk_data_pack.go @@ -7,16 +7,22 @@ import ( "github.com/onflow/flow-go/storage" ) +// InsertChunkDataPack inserts a chunk data pack keyed by chunk ID. +// any error are exceptions func InsertChunkDataPack(sc *storage.StoredChunkDataPack) func(w pebble.Writer) error { key := makeKey(codeChunkDataPack, sc.ChunkID) return insert(key, sc) } +// RetrieveChunkDataPack retrieves a chunk data pack by chunk ID. +// it returns storage.ErrNotFound if the chunk data pack is not found func RetrieveChunkDataPack(chunkID flow.Identifier, sc *storage.StoredChunkDataPack) func(r pebble.Reader) error { key := makeKey(codeChunkDataPack, chunkID) return retrieve(key, sc) } +// RemoveChunkDataPack removes the chunk data pack with the given chunk ID. +// any error are exceptions func RemoveChunkDataPack(chunkID flow.Identifier) func(w pebble.Writer) error { key := makeKey(codeChunkDataPack, chunkID) return func(w pebble.Writer) error { diff --git a/storage/pebble/value_cache.go b/storage/pebble/value_cache.go index fdf2b9044c7..38f1f394910 100644 --- a/storage/pebble/value_cache.go +++ b/storage/pebble/value_cache.go @@ -50,6 +50,10 @@ func noRetrieve[K comparable, V any](_ K) func(pebble.Reader) (V, error) { } } +// Cache is a read-through cache for underlying storage layer. +// Note: when a resource is not found in the cache nor the underlying storage, then +// it will not be cached. In other words, finding the missing item again will +// query the underlying storage again. type Cache[K comparable, V any] struct { metrics module.CacheMetrics limit uint From 22daffdb72b8fbd1f7434d0e1aa44a1b5e909f13 Mon Sep 17 00:00:00 2001 From: "Leo Zhang (zhangchiqing)" Date: Tue, 11 Jun 2024 11:50:24 -0700 Subject: [PATCH 30/60] address review comments --- storage/pebble/chunk_data_packs.go | 18 ++++++++++++------ .../chunk_data_pack.go | 6 +++++- .../pebble/{operations => operation}/codes.go | 2 +- .../pebble/{operations => operation}/common.go | 7 +------ 4 files changed, 19 insertions(+), 14 deletions(-) rename storage/pebble/{operations => operation}/chunk_data_pack.go (88%) rename storage/pebble/{operations => operation}/codes.go (65%) rename storage/pebble/{operations => operation}/common.go (86%) diff --git a/storage/pebble/chunk_data_packs.go b/storage/pebble/chunk_data_packs.go index 73037e988ed..c0b5b47eeab 100644 --- a/storage/pebble/chunk_data_packs.go +++ b/storage/pebble/chunk_data_packs.go @@ -4,12 +4,13 @@ import ( "fmt" "github.com/cockroachdb/pebble" + "github.com/rs/zerolog/log" "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/module" "github.com/onflow/flow-go/module/metrics" "github.com/onflow/flow-go/storage" - "github.com/onflow/flow-go/storage/pebble/operations" + "github.com/onflow/flow-go/storage/pebble/operation" ) type ChunkDataPacks struct { @@ -25,7 +26,7 @@ func NewChunkDataPacks(collector module.CacheMetrics, db *pebble.DB, collections retrieve := func(key flow.Identifier) func(pebble.Reader) (*storage.StoredChunkDataPack, error) { return func(r pebble.Reader) (*storage.StoredChunkDataPack, error) { var c storage.StoredChunkDataPack - err := operations.RetrieveChunkDataPack(key, &c)(r) + err := operation.RetrieveChunkDataPack(key, &c)(r) return &c, err } } @@ -46,7 +47,12 @@ func NewChunkDataPacks(collector module.CacheMetrics, db *pebble.DB, collections // Any error are exceptions func (ch *ChunkDataPacks) Store(cs []*flow.ChunkDataPack) error { batch := NewBatch(ch.db) - defer batch.Close() + defer func() { + err := batch.Close() + if err != nil { + log.Error().Err(err).Msgf("failed to close batch when storing chunk data pack") + } + }() for _, c := range cs { err := ch.batchStore(c, batch) @@ -92,7 +98,7 @@ func (ch *ChunkDataPacks) Remove(cs []flow.Identifier) error { // other errors are exceptions func (ch *ChunkDataPacks) ByChunkID(chunkID flow.Identifier) (*flow.ChunkDataPack, error) { var sc storage.StoredChunkDataPack - err := operations.RetrieveChunkDataPack(chunkID, &sc)(ch.db) + err := operation.RetrieveChunkDataPack(chunkID, &sc)(ch.db) if err != nil { return nil, fmt.Errorf("could not retrieve stored chunk data pack: %w", err) } @@ -121,7 +127,7 @@ func (ch *ChunkDataPacks) BatchRemove(chunkID flow.Identifier, batch storage.Bat } func (ch *ChunkDataPacks) batchRemove(chunkID flow.Identifier, batch pebble.Writer) error { - return operations.RemoveChunkDataPack(chunkID)(batch) + return operation.RemoveChunkDataPack(chunkID)(batch) } func (ch *ChunkDataPacks) batchStore(c *flow.ChunkDataPack, batch *Batch) error { @@ -130,7 +136,7 @@ func (ch *ChunkDataPacks) batchStore(c *flow.ChunkDataPack, batch *Batch) error batch.OnSucceed(func() { ch.byChunkIDCache.Insert(sc.ChunkID, sc) }) - err := operations.InsertChunkDataPack(sc)(writer) + err := operation.InsertChunkDataPack(sc)(writer) if err != nil { return fmt.Errorf("failed to store chunk data pack: %w", err) } diff --git a/storage/pebble/operations/chunk_data_pack.go b/storage/pebble/operation/chunk_data_pack.go similarity index 88% rename from storage/pebble/operations/chunk_data_pack.go rename to storage/pebble/operation/chunk_data_pack.go index 7b93f72d61a..f5cec13cdbe 100644 --- a/storage/pebble/operations/chunk_data_pack.go +++ b/storage/pebble/operation/chunk_data_pack.go @@ -1,4 +1,4 @@ -package operations +package operation import ( "github.com/cockroachdb/pebble" @@ -29,3 +29,7 @@ func RemoveChunkDataPack(chunkID flow.Identifier) func(w pebble.Writer) error { return w.Delete(key, nil) } } + +func makeKey(prefix byte, identifier flow.Identifier) []byte { + return append([]byte{prefix}, identifier[:]...) +} diff --git a/storage/pebble/operations/codes.go b/storage/pebble/operation/codes.go similarity index 65% rename from storage/pebble/operations/codes.go rename to storage/pebble/operation/codes.go index 0f803e5e0c0..1d9057646c3 100644 --- a/storage/pebble/operations/codes.go +++ b/storage/pebble/operation/codes.go @@ -1,4 +1,4 @@ -package operations +package operation const ( codeChunkDataPack = 100 diff --git a/storage/pebble/operations/common.go b/storage/pebble/operation/common.go similarity index 86% rename from storage/pebble/operations/common.go rename to storage/pebble/operation/common.go index a093ca459c7..ad9e96c2c8b 100644 --- a/storage/pebble/operations/common.go +++ b/storage/pebble/operation/common.go @@ -1,4 +1,4 @@ -package operations +package operation import ( "errors" @@ -6,7 +6,6 @@ import ( "github.com/cockroachdb/pebble" "github.com/vmihailenco/msgpack" - "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/module/irrecoverable" "github.com/onflow/flow-go/storage" ) @@ -43,10 +42,6 @@ func retrieve(key []byte, sc interface{}) func(r pebble.Reader) error { } } -func makeKey(prefix byte, identifier flow.Identifier) []byte { - return append([]byte{prefix}, identifier[:]...) -} - func convertNotFoundError(err error) error { if errors.Is(err, pebble.ErrNotFound) { return storage.ErrNotFound From 6beaff85bc30a1a302e352db998b0c5f6ba2c2aa Mon Sep 17 00:00:00 2001 From: Faye Amacker <33205765+fxamacker@users.noreply.github.com> Date: Wed, 12 Jun 2024 10:14:05 -0500 Subject: [PATCH 31/60] Update Cadence migration tests to include path_cap --- .../cadence_values_migration_test.go | 40 +++++++++++++++++++ .../migration_matrics_collector_test.go | 4 +- 2 files changed, 42 insertions(+), 2 deletions(-) diff --git a/cmd/util/ledger/migrations/cadence_values_migration_test.go b/cmd/util/ledger/migrations/cadence_values_migration_test.go index f63d2fa99e5..be88cdd7825 100644 --- a/cmd/util/ledger/migrations/cadence_values_migration_test.go +++ b/cmd/util/ledger/migrations/cadence_values_migration_test.go @@ -457,6 +457,46 @@ func checkMigratedState( storageMapKey: interpreter.Uint64StorageMapKey(0x3), value: "StorageCapabilityController(borrowType: Type<&A.0ae53cb6e3f42a79.FlowToken.Vault>(), capabilityID: 3, target: /storage/flowTokenVault)", }, + { + storageKey: interpreter.StorageKey{Key: "path_cap", Address: address}, + storageMapKey: interpreter.StringStorageMapKey("flowTokenVault"), + value: "3", + }, + { + storageKey: interpreter.StorageKey{Key: "path_cap", Address: address}, + storageMapKey: interpreter.StringStorageMapKey("flowTokenVault"), + value: "1", + }, + { + storageKey: interpreter.StorageKey{Key: "path_cap", Address: address}, + storageMapKey: interpreter.StringStorageMapKey("flowTokenVault"), + value: "nil", + }, + { + storageKey: interpreter.StorageKey{Key: "path_cap", Address: address}, + storageMapKey: interpreter.StringStorageMapKey("flowTokenVault"), + value: "nil", + }, + { + storageKey: interpreter.StorageKey{Key: "path_cap", Address: address}, + storageMapKey: interpreter.StringStorageMapKey("flowTokenVault"), + value: "{3: nil, 1: nil}", + }, + { + storageKey: interpreter.StorageKey{Key: "path_cap", Address: address}, + storageMapKey: interpreter.StringStorageMapKey("r"), + value: "2", + }, + { + storageKey: interpreter.StorageKey{Key: "path_cap", Address: address}, + storageMapKey: interpreter.StringStorageMapKey("r"), + value: "nil", + }, + { + storageKey: interpreter.StorageKey{Key: "path_cap", Address: address}, + storageMapKey: interpreter.StringStorageMapKey("r"), + value: "{2: nil}", + }, }, visitMigration.visits, ) diff --git a/cmd/util/ledger/migrations/migration_matrics_collector_test.go b/cmd/util/ledger/migrations/migration_matrics_collector_test.go index 1bd70a82739..29afafb19ef 100644 --- a/cmd/util/ledger/migrations/migration_matrics_collector_test.go +++ b/cmd/util/ledger/migrations/migration_matrics_collector_test.go @@ -87,7 +87,7 @@ func TestMigrationMetricsCollection(t *testing.T) { require.Equal( t, Metrics{ - TotalValues: 752, + TotalValues: 789, TotalErrors: 6, ErrorsPerContract: map[string]int{ "A.01cf0e2f2f715450.Test": 6, @@ -187,7 +187,7 @@ func TestMigrationMetricsCollection(t *testing.T) { require.Equal( t, Metrics{ - TotalValues: 752, + TotalValues: 789, TotalErrors: 6, ErrorsPerContract: map[string]int{ "A.01cf0e2f2f715450.Test": 6, From d4b552769b5ff765c9177a11c7c924086713b858 Mon Sep 17 00:00:00 2001 From: Faye Amacker <33205765+fxamacker@users.noreply.github.com> Date: Wed, 12 Jun 2024 10:34:20 -0500 Subject: [PATCH 32/60] Update mocks --- fvm/environment/mock/accounts.go | 2 +- fvm/environment/mock/environment.go | 2 +- fvm/environment/mock/value_store.go | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/fvm/environment/mock/accounts.go b/fvm/environment/mock/accounts.go index 6452179acd2..83c42d5955e 100644 --- a/fvm/environment/mock/accounts.go +++ b/fvm/environment/mock/accounts.go @@ -20,7 +20,7 @@ func (_m *Accounts) AllocateSlabIndex(address flow.Address) (atree.SlabIndex, er ret := _m.Called(address) if len(ret) == 0 { - panic("no return value specified for AllocateStorageIndex") + panic("no return value specified for AllocateSlabIndex") } var r0 atree.SlabIndex diff --git a/fvm/environment/mock/environment.go b/fvm/environment/mock/environment.go index 93308b8453b..a15f8774d32 100644 --- a/fvm/environment/mock/environment.go +++ b/fvm/environment/mock/environment.go @@ -137,7 +137,7 @@ func (_m *Environment) AllocateSlabIndex(owner []byte) (atree.SlabIndex, error) ret := _m.Called(owner) if len(ret) == 0 { - panic("no return value specified for AllocateStorageIndex") + panic("no return value specified for AllocateSlabIndex") } var r0 atree.SlabIndex diff --git a/fvm/environment/mock/value_store.go b/fvm/environment/mock/value_store.go index d82343cbb5b..8790ea2653f 100644 --- a/fvm/environment/mock/value_store.go +++ b/fvm/environment/mock/value_store.go @@ -18,7 +18,7 @@ func (_m *ValueStore) AllocateSlabIndex(owner []byte) (atree.SlabIndex, error) { ret := _m.Called(owner) if len(ret) == 0 { - panic("no return value specified for AllocateStorageIndex") + panic("no return value specified for AllocateSlabIndex") } var r0 atree.SlabIndex From 6bd2e01996777d10c3b61c61bbd652d63776af62 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastian=20M=C3=BCller?= Date: Wed, 12 Jun 2024 09:55:57 -0700 Subject: [PATCH 33/60] Update to Cadence v1.0.0-preview.34 --- go.mod | 4 ++-- go.sum | 8 ++++---- insecure/go.mod | 4 ++-- insecure/go.sum | 8 ++++---- integration/go.mod | 4 ++-- integration/go.sum | 8 ++++---- 6 files changed, 18 insertions(+), 18 deletions(-) diff --git a/go.mod b/go.mod index 6f22a64920c..b29c3361983 100644 --- a/go.mod +++ b/go.mod @@ -47,12 +47,12 @@ require ( github.com/multiformats/go-multiaddr-dns v0.3.1 github.com/multiformats/go-multihash v0.2.3 github.com/onflow/atree v0.7.0-rc.2 - github.com/onflow/cadence v1.0.0-preview.33 + github.com/onflow/cadence v1.0.0-preview.34 github.com/onflow/crypto v0.25.1 github.com/onflow/flow v0.3.4 github.com/onflow/flow-core-contracts/lib/go/contracts v1.1.0 github.com/onflow/flow-core-contracts/lib/go/templates v1.0.0 - github.com/onflow/flow-go-sdk v1.0.0-preview.35 + github.com/onflow/flow-go-sdk v1.0.0-preview.36 github.com/onflow/flow/protobuf/go/flow v0.4.4 github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 github.com/pierrec/lz4 v2.6.1+incompatible diff --git a/go.sum b/go.sum index 0883c06396a..c0ec089e15a 100644 --- a/go.sum +++ b/go.sum @@ -2171,8 +2171,8 @@ github.com/onflow/atree v0.7.0-rc.2/go.mod h1:xvP61FoOs95K7IYdIYRnNcYQGf4nbF/uuJ github.com/onflow/boxo v0.0.0-20240201202436-f2477b92f483 h1:LpiQhTAfM9CAmNVEs0n//cBBgCg+vJSiIxTHYUklZ84= github.com/onflow/boxo v0.0.0-20240201202436-f2477b92f483/go.mod h1:pIZgTWdm3k3pLF9Uq6MB8JEcW07UDwNJjlXW1HELW80= github.com/onflow/cadence v1.0.0-M3/go.mod h1:odXGZZ/wGNA5mwT8bC9v8u8EXACHllB2ABSZK65TGL8= -github.com/onflow/cadence v1.0.0-preview.33 h1:kqkU+9//PRsyL3SMokeK2mStarZVxiwrGypyiOX/On8= -github.com/onflow/cadence v1.0.0-preview.33/go.mod h1:jOwvPSSLTr9TvaKMs7KKiBYMmpdpNNAFxBsjMlrqVD0= +github.com/onflow/cadence v1.0.0-preview.34 h1:MJSli75W6LJVUqSx/tq4MQe64H1+EcQBD/sNgpOO4jE= +github.com/onflow/cadence v1.0.0-preview.34/go.mod h1:jOwvPSSLTr9TvaKMs7KKiBYMmpdpNNAFxBsjMlrqVD0= github.com/onflow/crypto v0.25.0/go.mod h1:C8FbaX0x8y+FxWjbkHy0Q4EASCDR9bSPWZqlpCLYyVI= github.com/onflow/crypto v0.25.1 h1:0txy2PKPMM873JbpxQNbJmuOJtD56bfs48RQfm0ts5A= github.com/onflow/crypto v0.25.1/go.mod h1:C8FbaX0x8y+FxWjbkHy0Q4EASCDR9bSPWZqlpCLYyVI= @@ -2187,8 +2187,8 @@ github.com/onflow/flow-ft/lib/go/contracts v1.0.0/go.mod h1:PwsL8fC81cjnUnTfmyL/ github.com/onflow/flow-ft/lib/go/templates v1.0.0 h1:6cMS/lUJJ17HjKBfMO/eh0GGvnpElPgBXx7h5aoWJhs= github.com/onflow/flow-ft/lib/go/templates v1.0.0/go.mod h1:uQ8XFqmMK2jxyBSVrmyuwdWjTEb+6zGjRYotfDJ5pAE= github.com/onflow/flow-go-sdk v1.0.0-M1/go.mod h1:TDW0MNuCs4SvqYRUzkbRnRmHQL1h4X8wURsCw9P9beo= -github.com/onflow/flow-go-sdk v1.0.0-preview.35 h1:2ptBhFYFGOaYghZTRbj51BbYqTZjkyEpXDyaWDYrHwA= -github.com/onflow/flow-go-sdk v1.0.0-preview.35/go.mod h1:/G8vtAekhvgynLYVDtd6OnhixoGTzzknmhYCJB2YWWU= +github.com/onflow/flow-go-sdk v1.0.0-preview.36 h1:3g72MjmZPEEVAbtDATbjwqKoNSB7yHLWswUHSAB5zwQ= +github.com/onflow/flow-go-sdk v1.0.0-preview.36/go.mod h1:mjkXIluC+kseYyd8Z1aTq73IiffAUeoY5fuX/C2Z+1w= github.com/onflow/flow-nft/lib/go/contracts v1.2.1 h1:woAAS5z651sDpi7ihAHll8NvRS9uFXIXkL6xR+bKFZY= github.com/onflow/flow-nft/lib/go/contracts v1.2.1/go.mod h1:2gpbza+uzs1k7x31hkpBPlggIRkI53Suo0n2AyA2HcE= github.com/onflow/flow-nft/lib/go/templates v1.2.0 h1:JSQyh9rg0RC+D1930BiRXN8lrtMs+ubVMK6aQPon6Yc= diff --git a/insecure/go.mod b/insecure/go.mod index d4f03241831..534c198e208 100644 --- a/insecure/go.mod +++ b/insecure/go.mod @@ -199,12 +199,12 @@ require ( github.com/multiformats/go-varint v0.0.7 // indirect github.com/olekukonko/tablewriter v0.0.5 // indirect github.com/onflow/atree v0.7.0-rc.2 // indirect - github.com/onflow/cadence v1.0.0-preview.33 // indirect + github.com/onflow/cadence v1.0.0-preview.34 // indirect github.com/onflow/flow-core-contracts/lib/go/contracts v1.1.0 // indirect github.com/onflow/flow-core-contracts/lib/go/templates v1.0.0 // indirect github.com/onflow/flow-ft/lib/go/contracts v1.0.0 // indirect github.com/onflow/flow-ft/lib/go/templates v1.0.0 // indirect - github.com/onflow/flow-go-sdk v1.0.0-preview.35 // indirect + github.com/onflow/flow-go-sdk v1.0.0-preview.36 // indirect github.com/onflow/flow-nft/lib/go/contracts v1.2.1 // indirect github.com/onflow/flow-nft/lib/go/templates v1.2.0 // indirect github.com/onflow/flow/protobuf/go/flow v0.4.4 // indirect diff --git a/insecure/go.sum b/insecure/go.sum index fec8cd1bfb7..f97bb10c543 100644 --- a/insecure/go.sum +++ b/insecure/go.sum @@ -2160,8 +2160,8 @@ github.com/onflow/atree v0.6.1-0.20230711151834-86040b30171f/go.mod h1:xvP61FoOs github.com/onflow/atree v0.7.0-rc.2 h1:mZmVrl/zPlfI44EjV3FdR2QwIqT8nz1sCONUBFcML/U= github.com/onflow/atree v0.7.0-rc.2/go.mod h1:xvP61FoOs95K7IYdIYRnNcYQGf4nbF/uuJ0tHf4DRuM= github.com/onflow/cadence v1.0.0-M3/go.mod h1:odXGZZ/wGNA5mwT8bC9v8u8EXACHllB2ABSZK65TGL8= -github.com/onflow/cadence v1.0.0-preview.33 h1:kqkU+9//PRsyL3SMokeK2mStarZVxiwrGypyiOX/On8= -github.com/onflow/cadence v1.0.0-preview.33/go.mod h1:jOwvPSSLTr9TvaKMs7KKiBYMmpdpNNAFxBsjMlrqVD0= +github.com/onflow/cadence v1.0.0-preview.34 h1:MJSli75W6LJVUqSx/tq4MQe64H1+EcQBD/sNgpOO4jE= +github.com/onflow/cadence v1.0.0-preview.34/go.mod h1:jOwvPSSLTr9TvaKMs7KKiBYMmpdpNNAFxBsjMlrqVD0= github.com/onflow/crypto v0.25.0/go.mod h1:C8FbaX0x8y+FxWjbkHy0Q4EASCDR9bSPWZqlpCLYyVI= github.com/onflow/crypto v0.25.1 h1:0txy2PKPMM873JbpxQNbJmuOJtD56bfs48RQfm0ts5A= github.com/onflow/crypto v0.25.1/go.mod h1:C8FbaX0x8y+FxWjbkHy0Q4EASCDR9bSPWZqlpCLYyVI= @@ -2174,8 +2174,8 @@ github.com/onflow/flow-ft/lib/go/contracts v1.0.0/go.mod h1:PwsL8fC81cjnUnTfmyL/ github.com/onflow/flow-ft/lib/go/templates v1.0.0 h1:6cMS/lUJJ17HjKBfMO/eh0GGvnpElPgBXx7h5aoWJhs= github.com/onflow/flow-ft/lib/go/templates v1.0.0/go.mod h1:uQ8XFqmMK2jxyBSVrmyuwdWjTEb+6zGjRYotfDJ5pAE= github.com/onflow/flow-go-sdk v1.0.0-M1/go.mod h1:TDW0MNuCs4SvqYRUzkbRnRmHQL1h4X8wURsCw9P9beo= -github.com/onflow/flow-go-sdk v1.0.0-preview.35 h1:2ptBhFYFGOaYghZTRbj51BbYqTZjkyEpXDyaWDYrHwA= -github.com/onflow/flow-go-sdk v1.0.0-preview.35/go.mod h1:/G8vtAekhvgynLYVDtd6OnhixoGTzzknmhYCJB2YWWU= +github.com/onflow/flow-go-sdk v1.0.0-preview.36 h1:3g72MjmZPEEVAbtDATbjwqKoNSB7yHLWswUHSAB5zwQ= +github.com/onflow/flow-go-sdk v1.0.0-preview.36/go.mod h1:mjkXIluC+kseYyd8Z1aTq73IiffAUeoY5fuX/C2Z+1w= github.com/onflow/flow-nft/lib/go/contracts v1.2.1 h1:woAAS5z651sDpi7ihAHll8NvRS9uFXIXkL6xR+bKFZY= github.com/onflow/flow-nft/lib/go/contracts v1.2.1/go.mod h1:2gpbza+uzs1k7x31hkpBPlggIRkI53Suo0n2AyA2HcE= github.com/onflow/flow-nft/lib/go/templates v1.2.0 h1:JSQyh9rg0RC+D1930BiRXN8lrtMs+ubVMK6aQPon6Yc= diff --git a/integration/go.mod b/integration/go.mod index 138d2e0374c..3a706343122 100644 --- a/integration/go.mod +++ b/integration/go.mod @@ -19,13 +19,13 @@ require ( github.com/ipfs/go-datastore v0.6.0 github.com/ipfs/go-ds-badger2 v0.1.3 github.com/libp2p/go-libp2p v0.32.2 - github.com/onflow/cadence v1.0.0-preview.33 + github.com/onflow/cadence v1.0.0-preview.34 github.com/onflow/crypto v0.25.1 github.com/onflow/flow-core-contracts/lib/go/contracts v1.1.0 github.com/onflow/flow-core-contracts/lib/go/templates v1.0.0 github.com/onflow/flow-emulator v1.0.0-preview.24 github.com/onflow/flow-go v0.35.5-0.20240517202625-55f862b45dfd - github.com/onflow/flow-go-sdk v1.0.0-preview.35 + github.com/onflow/flow-go-sdk v1.0.0-preview.36 github.com/onflow/flow-go/insecure v0.0.0-00010101000000-000000000000 github.com/onflow/flow/protobuf/go/flow v0.4.4 github.com/onflow/go-ethereum v1.13.4 diff --git a/integration/go.sum b/integration/go.sum index 310bad73ae6..b264c2122a4 100644 --- a/integration/go.sum +++ b/integration/go.sum @@ -2150,8 +2150,8 @@ github.com/onflow/atree v0.6.1-0.20230711151834-86040b30171f/go.mod h1:xvP61FoOs github.com/onflow/atree v0.7.0-rc.2 h1:mZmVrl/zPlfI44EjV3FdR2QwIqT8nz1sCONUBFcML/U= github.com/onflow/atree v0.7.0-rc.2/go.mod h1:xvP61FoOs95K7IYdIYRnNcYQGf4nbF/uuJ0tHf4DRuM= github.com/onflow/cadence v1.0.0-M3/go.mod h1:odXGZZ/wGNA5mwT8bC9v8u8EXACHllB2ABSZK65TGL8= -github.com/onflow/cadence v1.0.0-preview.33 h1:kqkU+9//PRsyL3SMokeK2mStarZVxiwrGypyiOX/On8= -github.com/onflow/cadence v1.0.0-preview.33/go.mod h1:jOwvPSSLTr9TvaKMs7KKiBYMmpdpNNAFxBsjMlrqVD0= +github.com/onflow/cadence v1.0.0-preview.34 h1:MJSli75W6LJVUqSx/tq4MQe64H1+EcQBD/sNgpOO4jE= +github.com/onflow/cadence v1.0.0-preview.34/go.mod h1:jOwvPSSLTr9TvaKMs7KKiBYMmpdpNNAFxBsjMlrqVD0= github.com/onflow/crypto v0.25.0/go.mod h1:C8FbaX0x8y+FxWjbkHy0Q4EASCDR9bSPWZqlpCLYyVI= github.com/onflow/crypto v0.25.1 h1:0txy2PKPMM873JbpxQNbJmuOJtD56bfs48RQfm0ts5A= github.com/onflow/crypto v0.25.1/go.mod h1:C8FbaX0x8y+FxWjbkHy0Q4EASCDR9bSPWZqlpCLYyVI= @@ -2166,8 +2166,8 @@ github.com/onflow/flow-ft/lib/go/contracts v1.0.0/go.mod h1:PwsL8fC81cjnUnTfmyL/ github.com/onflow/flow-ft/lib/go/templates v1.0.0 h1:6cMS/lUJJ17HjKBfMO/eh0GGvnpElPgBXx7h5aoWJhs= github.com/onflow/flow-ft/lib/go/templates v1.0.0/go.mod h1:uQ8XFqmMK2jxyBSVrmyuwdWjTEb+6zGjRYotfDJ5pAE= github.com/onflow/flow-go-sdk v1.0.0-M1/go.mod h1:TDW0MNuCs4SvqYRUzkbRnRmHQL1h4X8wURsCw9P9beo= -github.com/onflow/flow-go-sdk v1.0.0-preview.35 h1:2ptBhFYFGOaYghZTRbj51BbYqTZjkyEpXDyaWDYrHwA= -github.com/onflow/flow-go-sdk v1.0.0-preview.35/go.mod h1:/G8vtAekhvgynLYVDtd6OnhixoGTzzknmhYCJB2YWWU= +github.com/onflow/flow-go-sdk v1.0.0-preview.36 h1:3g72MjmZPEEVAbtDATbjwqKoNSB7yHLWswUHSAB5zwQ= +github.com/onflow/flow-go-sdk v1.0.0-preview.36/go.mod h1:mjkXIluC+kseYyd8Z1aTq73IiffAUeoY5fuX/C2Z+1w= github.com/onflow/flow-nft/lib/go/contracts v1.2.1 h1:woAAS5z651sDpi7ihAHll8NvRS9uFXIXkL6xR+bKFZY= github.com/onflow/flow-nft/lib/go/contracts v1.2.1/go.mod h1:2gpbza+uzs1k7x31hkpBPlggIRkI53Suo0n2AyA2HcE= github.com/onflow/flow-nft/lib/go/templates v1.2.0 h1:JSQyh9rg0RC+D1930BiRXN8lrtMs+ubVMK6aQPon6Yc= From 053e77ccfd828e4592a164c569c7bab2540e9555 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastian=20M=C3=BCller?= Date: Wed, 12 Jun 2024 10:27:37 -0700 Subject: [PATCH 34/60] adjust tests: add new visits, account for them in metrics --- .../cadence_values_migration_test.go | 64 +++++++++++++++++++ .../migration_matrics_collector_test.go | 4 +- 2 files changed, 66 insertions(+), 2 deletions(-) diff --git a/cmd/util/ledger/migrations/cadence_values_migration_test.go b/cmd/util/ledger/migrations/cadence_values_migration_test.go index f63d2fa99e5..882952b9362 100644 --- a/cmd/util/ledger/migrations/cadence_values_migration_test.go +++ b/cmd/util/ledger/migrations/cadence_values_migration_test.go @@ -457,6 +457,70 @@ func checkMigratedState( storageMapKey: interpreter.Uint64StorageMapKey(0x3), value: "StorageCapabilityController(borrowType: Type<&A.0ae53cb6e3f42a79.FlowToken.Vault>(), capabilityID: 3, target: /storage/flowTokenVault)", }, + { + storageKey: interpreter.StorageKey{ + Key: "path_cap", + Address: address, + }, + storageMapKey: interpreter.StringStorageMapKey("flowTokenVault"), + value: "3", + }, + { + storageKey: interpreter.StorageKey{ + Key: "path_cap", + Address: address, + }, + storageMapKey: interpreter.StringStorageMapKey("flowTokenVault"), + value: "1", + }, + { + storageKey: interpreter.StorageKey{ + Key: "path_cap", + Address: address, + }, + storageMapKey: interpreter.StringStorageMapKey("flowTokenVault"), + value: "nil", + }, + { + storageKey: interpreter.StorageKey{ + Key: "path_cap", + Address: address, + }, + storageMapKey: interpreter.StringStorageMapKey("flowTokenVault"), + value: "nil", + }, + { + storageKey: interpreter.StorageKey{ + Key: "path_cap", + Address: address, + }, + storageMapKey: interpreter.StringStorageMapKey("flowTokenVault"), + value: "{3: nil, 1: nil}", + }, + { + storageKey: interpreter.StorageKey{ + Key: "path_cap", + Address: address, + }, + storageMapKey: interpreter.StringStorageMapKey("r"), + value: "2", + }, + { + storageKey: interpreter.StorageKey{ + Key: "path_cap", + Address: address, + }, + storageMapKey: interpreter.StringStorageMapKey("r"), + value: "nil", + }, + { + storageKey: interpreter.StorageKey{ + Key: "path_cap", + Address: address, + }, + storageMapKey: interpreter.StringStorageMapKey("r"), + value: "{2: nil}", + }, }, visitMigration.visits, ) diff --git a/cmd/util/ledger/migrations/migration_matrics_collector_test.go b/cmd/util/ledger/migrations/migration_matrics_collector_test.go index 1bd70a82739..29afafb19ef 100644 --- a/cmd/util/ledger/migrations/migration_matrics_collector_test.go +++ b/cmd/util/ledger/migrations/migration_matrics_collector_test.go @@ -87,7 +87,7 @@ func TestMigrationMetricsCollection(t *testing.T) { require.Equal( t, Metrics{ - TotalValues: 752, + TotalValues: 789, TotalErrors: 6, ErrorsPerContract: map[string]int{ "A.01cf0e2f2f715450.Test": 6, @@ -187,7 +187,7 @@ func TestMigrationMetricsCollection(t *testing.T) { require.Equal( t, Metrics{ - TotalValues: 752, + TotalValues: 789, TotalErrors: 6, ErrorsPerContract: map[string]int{ "A.01cf0e2f2f715450.Test": 6, From b86ca1c87b0a63b0a41a9a8260bb5f4efc14c65b Mon Sep 17 00:00:00 2001 From: Peter Argue <89119817+peterargue@users.noreply.github.com> Date: Wed, 12 Jun 2024 10:41:09 -0700 Subject: [PATCH 35/60] [CI] Remove cruise control overrides in Access integration tests --- integration/tests/access/cohort3/grpc_state_stream_test.go | 1 - integration/tests/access/cohort3/grpc_streaming_blocks_test.go | 1 - 2 files changed, 2 deletions(-) diff --git a/integration/tests/access/cohort3/grpc_state_stream_test.go b/integration/tests/access/cohort3/grpc_state_stream_test.go index d4f77b9245d..2f2d883bb1e 100644 --- a/integration/tests/access/cohort3/grpc_state_stream_test.go +++ b/integration/tests/access/cohort3/grpc_state_stream_test.go @@ -114,7 +114,6 @@ func (s *GrpcStateStreamSuite) SetupTest() { testnet.AsGhost()) consensusConfigs := []func(config *testnet.NodeConfig){ - testnet.WithAdditionalFlag("--cruise-ctl-fallback-proposal-duration=400ms"), testnet.WithAdditionalFlag(fmt.Sprintf("--required-verification-seal-approvals=%d", 1)), testnet.WithAdditionalFlag(fmt.Sprintf("--required-construction-seal-approvals=%d", 1)), testnet.WithLogLevel(zerolog.FatalLevel), diff --git a/integration/tests/access/cohort3/grpc_streaming_blocks_test.go b/integration/tests/access/cohort3/grpc_streaming_blocks_test.go index 1cc6139676b..96c7655406a 100644 --- a/integration/tests/access/cohort3/grpc_streaming_blocks_test.go +++ b/integration/tests/access/cohort3/grpc_streaming_blocks_test.go @@ -74,7 +74,6 @@ func (s *GrpcBlocksStreamSuite) SetupTest() { ) consensusConfigs := []func(config *testnet.NodeConfig){ - testnet.WithAdditionalFlag("--cruise-ctl-fallback-proposal-duration=400ms"), testnet.WithAdditionalFlag(fmt.Sprintf("--required-verification-seal-approvals=%d", 1)), testnet.WithAdditionalFlag(fmt.Sprintf("--required-construction-seal-approvals=%d", 1)), testnet.WithLogLevel(zerolog.FatalLevel), From c9c1de2e8fe6cd4b8be3fd84d1d46f17e7d370f5 Mon Sep 17 00:00:00 2001 From: Josh Hannan Date: Wed, 12 Jun 2024 14:14:27 -0500 Subject: [PATCH 36/60] update core contracts versions --- engine/execution/state/bootstrap/bootstrap_test.go | 2 +- go.mod | 4 ++-- go.sum | 8 ++++---- insecure/go.mod | 4 ++-- insecure/go.sum | 8 ++++---- integration/go.mod | 4 ++-- integration/go.sum | 8 ++++---- utils/unittest/execution_state.go | 6 +++--- 8 files changed, 22 insertions(+), 22 deletions(-) diff --git a/engine/execution/state/bootstrap/bootstrap_test.go b/engine/execution/state/bootstrap/bootstrap_test.go index 4f88f874675..a02820c3919 100644 --- a/engine/execution/state/bootstrap/bootstrap_test.go +++ b/engine/execution/state/bootstrap/bootstrap_test.go @@ -53,7 +53,7 @@ func TestBootstrapLedger(t *testing.T) { } func TestBootstrapLedger_ZeroTokenSupply(t *testing.T) { - expectedStateCommitmentBytes, _ := hex.DecodeString("aff1aafa7a34803d7b545791e60b7ffbcae52be602cef514430170969652d050") + expectedStateCommitmentBytes, _ := hex.DecodeString("bcaf10db3920ea76a9b90d0f5b21d03cb2bc17087aa70554e4ebc01de6e166f1") expectedStateCommitment, err := flow.ToStateCommitment(expectedStateCommitmentBytes) require.NoError(t, err) diff --git a/go.mod b/go.mod index b29c3361983..4646272db38 100644 --- a/go.mod +++ b/go.mod @@ -50,9 +50,9 @@ require ( github.com/onflow/cadence v1.0.0-preview.34 github.com/onflow/crypto v0.25.1 github.com/onflow/flow v0.3.4 - github.com/onflow/flow-core-contracts/lib/go/contracts v1.1.0 - github.com/onflow/flow-core-contracts/lib/go/templates v1.0.0 github.com/onflow/flow-go-sdk v1.0.0-preview.36 + github.com/onflow/flow-core-contracts/lib/go/contracts v1.3.0 + github.com/onflow/flow-core-contracts/lib/go/templates v1.3.0 github.com/onflow/flow/protobuf/go/flow v0.4.4 github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 github.com/pierrec/lz4 v2.6.1+incompatible diff --git a/go.sum b/go.sum index c0ec089e15a..7beb1e4191c 100644 --- a/go.sum +++ b/go.sum @@ -2178,10 +2178,10 @@ github.com/onflow/crypto v0.25.1 h1:0txy2PKPMM873JbpxQNbJmuOJtD56bfs48RQfm0ts5A= github.com/onflow/crypto v0.25.1/go.mod h1:C8FbaX0x8y+FxWjbkHy0Q4EASCDR9bSPWZqlpCLYyVI= github.com/onflow/flow v0.3.4 h1:FXUWVdYB90f/rjNcY0Owo30gL790tiYff9Pb/sycXYE= github.com/onflow/flow v0.3.4/go.mod h1:lzyAYmbu1HfkZ9cfnL5/sjrrsnJiUU8fRL26CqLP7+c= -github.com/onflow/flow-core-contracts/lib/go/contracts v1.1.0 h1:AegPBm079X0qjneUYs+mRCpEUxSZ1lw5h4MbuXHlqn0= -github.com/onflow/flow-core-contracts/lib/go/contracts v1.1.0/go.mod h1:u/mkP/B+PbV33tEG3qfkhhBlydSvAKxfLZSfB4lsJHg= -github.com/onflow/flow-core-contracts/lib/go/templates v1.0.0 h1:za6bxPPW4JIsthhasUDTa1ruKjIO8DIhun9INQfj61Y= -github.com/onflow/flow-core-contracts/lib/go/templates v1.0.0/go.mod h1:NgbMOYnMh0GN48VsNKZuiwK7uyk38Wyo8jN9+C9QE30= +github.com/onflow/flow-core-contracts/lib/go/contracts v1.3.0 h1:cq3RfBr9TnTSnsGlUHMjMGZib24Horfb1XJqMpkN5ew= +github.com/onflow/flow-core-contracts/lib/go/contracts v1.3.0/go.mod h1:u/mkP/B+PbV33tEG3qfkhhBlydSvAKxfLZSfB4lsJHg= +github.com/onflow/flow-core-contracts/lib/go/templates v1.3.0 h1:aMFJdB2CW+Dzm+AJ5QN6J1yWh+a0l2RxHN2/TtLaXUo= +github.com/onflow/flow-core-contracts/lib/go/templates v1.3.0/go.mod h1:NgbMOYnMh0GN48VsNKZuiwK7uyk38Wyo8jN9+C9QE30= github.com/onflow/flow-ft/lib/go/contracts v1.0.0 h1:mToacZ5NWqtlWwk/7RgIl/jeKB/Sy/tIXdw90yKHcV0= github.com/onflow/flow-ft/lib/go/contracts v1.0.0/go.mod h1:PwsL8fC81cjnUnTfmyL/HOIyHnyaw/JA474Wfj2tl6A= github.com/onflow/flow-ft/lib/go/templates v1.0.0 h1:6cMS/lUJJ17HjKBfMO/eh0GGvnpElPgBXx7h5aoWJhs= diff --git a/insecure/go.mod b/insecure/go.mod index 534c198e208..653b5dddc0f 100644 --- a/insecure/go.mod +++ b/insecure/go.mod @@ -200,8 +200,8 @@ require ( github.com/olekukonko/tablewriter v0.0.5 // indirect github.com/onflow/atree v0.7.0-rc.2 // indirect github.com/onflow/cadence v1.0.0-preview.34 // indirect - github.com/onflow/flow-core-contracts/lib/go/contracts v1.1.0 // indirect - github.com/onflow/flow-core-contracts/lib/go/templates v1.0.0 // indirect + github.com/onflow/flow-core-contracts/lib/go/contracts v1.3.0 // indirect + github.com/onflow/flow-core-contracts/lib/go/templates v1.3.0 // indirect github.com/onflow/flow-ft/lib/go/contracts v1.0.0 // indirect github.com/onflow/flow-ft/lib/go/templates v1.0.0 // indirect github.com/onflow/flow-go-sdk v1.0.0-preview.36 // indirect diff --git a/insecure/go.sum b/insecure/go.sum index f97bb10c543..030ffa3f4cb 100644 --- a/insecure/go.sum +++ b/insecure/go.sum @@ -2165,10 +2165,10 @@ github.com/onflow/cadence v1.0.0-preview.34/go.mod h1:jOwvPSSLTr9TvaKMs7KKiBYMmp github.com/onflow/crypto v0.25.0/go.mod h1:C8FbaX0x8y+FxWjbkHy0Q4EASCDR9bSPWZqlpCLYyVI= github.com/onflow/crypto v0.25.1 h1:0txy2PKPMM873JbpxQNbJmuOJtD56bfs48RQfm0ts5A= github.com/onflow/crypto v0.25.1/go.mod h1:C8FbaX0x8y+FxWjbkHy0Q4EASCDR9bSPWZqlpCLYyVI= -github.com/onflow/flow-core-contracts/lib/go/contracts v1.1.0 h1:AegPBm079X0qjneUYs+mRCpEUxSZ1lw5h4MbuXHlqn0= -github.com/onflow/flow-core-contracts/lib/go/contracts v1.1.0/go.mod h1:u/mkP/B+PbV33tEG3qfkhhBlydSvAKxfLZSfB4lsJHg= -github.com/onflow/flow-core-contracts/lib/go/templates v1.0.0 h1:za6bxPPW4JIsthhasUDTa1ruKjIO8DIhun9INQfj61Y= -github.com/onflow/flow-core-contracts/lib/go/templates v1.0.0/go.mod h1:NgbMOYnMh0GN48VsNKZuiwK7uyk38Wyo8jN9+C9QE30= +github.com/onflow/flow-core-contracts/lib/go/contracts v1.3.0 h1:cq3RfBr9TnTSnsGlUHMjMGZib24Horfb1XJqMpkN5ew= +github.com/onflow/flow-core-contracts/lib/go/contracts v1.3.0/go.mod h1:u/mkP/B+PbV33tEG3qfkhhBlydSvAKxfLZSfB4lsJHg= +github.com/onflow/flow-core-contracts/lib/go/templates v1.3.0 h1:aMFJdB2CW+Dzm+AJ5QN6J1yWh+a0l2RxHN2/TtLaXUo= +github.com/onflow/flow-core-contracts/lib/go/templates v1.3.0/go.mod h1:NgbMOYnMh0GN48VsNKZuiwK7uyk38Wyo8jN9+C9QE30= github.com/onflow/flow-ft/lib/go/contracts v1.0.0 h1:mToacZ5NWqtlWwk/7RgIl/jeKB/Sy/tIXdw90yKHcV0= github.com/onflow/flow-ft/lib/go/contracts v1.0.0/go.mod h1:PwsL8fC81cjnUnTfmyL/HOIyHnyaw/JA474Wfj2tl6A= github.com/onflow/flow-ft/lib/go/templates v1.0.0 h1:6cMS/lUJJ17HjKBfMO/eh0GGvnpElPgBXx7h5aoWJhs= diff --git a/integration/go.mod b/integration/go.mod index 3a706343122..ce66485b5a2 100644 --- a/integration/go.mod +++ b/integration/go.mod @@ -21,8 +21,8 @@ require ( github.com/libp2p/go-libp2p v0.32.2 github.com/onflow/cadence v1.0.0-preview.34 github.com/onflow/crypto v0.25.1 - github.com/onflow/flow-core-contracts/lib/go/contracts v1.1.0 - github.com/onflow/flow-core-contracts/lib/go/templates v1.0.0 + github.com/onflow/flow-core-contracts/lib/go/contracts v1.3.0 + github.com/onflow/flow-core-contracts/lib/go/templates v1.3.0 github.com/onflow/flow-emulator v1.0.0-preview.24 github.com/onflow/flow-go v0.35.5-0.20240517202625-55f862b45dfd github.com/onflow/flow-go-sdk v1.0.0-preview.36 diff --git a/integration/go.sum b/integration/go.sum index b264c2122a4..821a5c27fd9 100644 --- a/integration/go.sum +++ b/integration/go.sum @@ -2155,10 +2155,10 @@ github.com/onflow/cadence v1.0.0-preview.34/go.mod h1:jOwvPSSLTr9TvaKMs7KKiBYMmp github.com/onflow/crypto v0.25.0/go.mod h1:C8FbaX0x8y+FxWjbkHy0Q4EASCDR9bSPWZqlpCLYyVI= github.com/onflow/crypto v0.25.1 h1:0txy2PKPMM873JbpxQNbJmuOJtD56bfs48RQfm0ts5A= github.com/onflow/crypto v0.25.1/go.mod h1:C8FbaX0x8y+FxWjbkHy0Q4EASCDR9bSPWZqlpCLYyVI= -github.com/onflow/flow-core-contracts/lib/go/contracts v1.1.0 h1:AegPBm079X0qjneUYs+mRCpEUxSZ1lw5h4MbuXHlqn0= -github.com/onflow/flow-core-contracts/lib/go/contracts v1.1.0/go.mod h1:u/mkP/B+PbV33tEG3qfkhhBlydSvAKxfLZSfB4lsJHg= -github.com/onflow/flow-core-contracts/lib/go/templates v1.0.0 h1:za6bxPPW4JIsthhasUDTa1ruKjIO8DIhun9INQfj61Y= -github.com/onflow/flow-core-contracts/lib/go/templates v1.0.0/go.mod h1:NgbMOYnMh0GN48VsNKZuiwK7uyk38Wyo8jN9+C9QE30= +github.com/onflow/flow-core-contracts/lib/go/contracts v1.3.0 h1:cq3RfBr9TnTSnsGlUHMjMGZib24Horfb1XJqMpkN5ew= +github.com/onflow/flow-core-contracts/lib/go/contracts v1.3.0/go.mod h1:u/mkP/B+PbV33tEG3qfkhhBlydSvAKxfLZSfB4lsJHg= +github.com/onflow/flow-core-contracts/lib/go/templates v1.3.0 h1:aMFJdB2CW+Dzm+AJ5QN6J1yWh+a0l2RxHN2/TtLaXUo= +github.com/onflow/flow-core-contracts/lib/go/templates v1.3.0/go.mod h1:NgbMOYnMh0GN48VsNKZuiwK7uyk38Wyo8jN9+C9QE30= github.com/onflow/flow-emulator v1.0.0-preview.24 h1:SonXMBeYxVwNn94M+OUmKIYScIMQG22wugh9n/tHY5k= github.com/onflow/flow-emulator v1.0.0-preview.24/go.mod h1:QprPouTWO3iv9VF/y4Ksltv2XIbzNMzjjr5zzq51i7Q= github.com/onflow/flow-ft/lib/go/contracts v1.0.0 h1:mToacZ5NWqtlWwk/7RgIl/jeKB/Sy/tIXdw90yKHcV0= diff --git a/utils/unittest/execution_state.go b/utils/unittest/execution_state.go index cb8dbd90786..28844644616 100644 --- a/utils/unittest/execution_state.go +++ b/utils/unittest/execution_state.go @@ -23,7 +23,7 @@ const ServiceAccountPrivateKeySignAlgo = crypto.ECDSAP256 const ServiceAccountPrivateKeyHashAlgo = hash.SHA2_256 // Pre-calculated state commitment with root account with the above private key -const GenesisStateCommitmentHex = "884400ce45738c071438df5926248e284446afc2528314bf734447d1cbc51bd9" +const GenesisStateCommitmentHex = "7bd7f9627241f803a2d31bb312ce409ef44b888a0d3d81dedf08357b1a652076" var GenesisStateCommitment flow.StateCommitment @@ -87,10 +87,10 @@ func genesisCommitHexByChainID(chainID flow.ChainID) string { return GenesisStateCommitmentHex } if chainID == flow.Testnet { - return "00b2d5c8f1d09e4578eb01accbdc86969122e9e3b00e21f27f7e341b509c47bd" + return "fc82b815c9cf75b8b217819f26d68b0da62f26c2a569fd1645cb0d9351e1e8bb" } if chainID == flow.Sandboxnet { return "e1c08b17f9e5896f03fe28dd37ca396c19b26628161506924fbf785834646ea1" } - return "a9f69641c6412877f12bd0078dd58911d92971b3fd47b2b0ccbc377a96a1589c" + return "4f43005cc29f9f033e81262c73f89080ce6e6591e757519daaa59ec16b8b8a56" } From e5632072b290ed7dc690ae52494da79e11b5a8ca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastian=20M=C3=BCller?= Date: Wed, 12 Jun 2024 13:42:27 -0700 Subject: [PATCH 37/60] go mod tidy --- go.mod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 4646272db38..4478289e694 100644 --- a/go.mod +++ b/go.mod @@ -50,9 +50,9 @@ require ( github.com/onflow/cadence v1.0.0-preview.34 github.com/onflow/crypto v0.25.1 github.com/onflow/flow v0.3.4 - github.com/onflow/flow-go-sdk v1.0.0-preview.36 github.com/onflow/flow-core-contracts/lib/go/contracts v1.3.0 github.com/onflow/flow-core-contracts/lib/go/templates v1.3.0 + github.com/onflow/flow-go-sdk v1.0.0-preview.36 github.com/onflow/flow/protobuf/go/flow v0.4.4 github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 github.com/pierrec/lz4 v2.6.1+incompatible From a48d27a824f8c9e959d18c59adc22591ef9c504f Mon Sep 17 00:00:00 2001 From: ramtinms Date: Wed, 12 Jun 2024 14:56:36 -0700 Subject: [PATCH 38/60] Add error message to the Tx executed event --- fvm/evm/stdlib/contract.cdc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/fvm/evm/stdlib/contract.cdc b/fvm/evm/stdlib/contract.cdc index 62947b9fa5c..2e28d3c1c7c 100644 --- a/fvm/evm/stdlib/contract.cdc +++ b/fvm/evm/stdlib/contract.cdc @@ -50,6 +50,8 @@ contract EVM { payload: String, // code indicating a specific validation (201-300) or execution (301-400) error errorCode: UInt16, + // a human-readable message about the error (if any) + errorMessage: String, // the amount of gas transaction used gasConsumed: UInt64, // if transaction was a deployment contains a newly deployed contract address From 0e5e254fa589c1a60e6807ec01fca8b08729e355 Mon Sep 17 00:00:00 2001 From: ramtinms Date: Wed, 12 Jun 2024 14:58:23 -0700 Subject: [PATCH 39/60] move error message closer to error code --- fvm/evm/types/events.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/fvm/evm/types/events.go b/fvm/evm/types/events.go index 842f9f553e6..cd801a17e4d 100644 --- a/fvm/evm/types/events.go +++ b/fvm/evm/types/events.go @@ -92,13 +92,13 @@ func (p *transactionEvent) ToCadence(location common.Location) (cadence.Event, e cadence.NewField("type", cadence.UInt8Type), cadence.NewField("payload", cadence.StringType), cadence.NewField("errorCode", cadence.UInt16Type), + cadence.NewField("errorMessage", cadence.StringType), cadence.NewField("gasConsumed", cadence.UInt64Type), cadence.NewField("contractAddress", cadence.StringType), cadence.NewField("logs", cadence.StringType), cadence.NewField("blockHeight", cadence.UInt64Type), // todo we can remove hash and just reference block by height (evm-gateway dependency) cadence.NewField("blockHash", cadence.StringType), - cadence.NewField("errorMessage", cadence.StringType), cadence.NewField("returnedData", cadence.StringType), }, nil, @@ -110,12 +110,12 @@ func (p *transactionEvent) ToCadence(location common.Location) (cadence.Event, e cadence.NewUInt8(p.Result.TxType), cadence.String(hex.EncodeToString(p.Payload)), cadence.NewUInt16(uint16(p.Result.ResultSummary().ErrorCode)), + cadence.String(errorMsg), cadence.NewUInt64(p.Result.GasConsumed), deployedAddress, cadence.String(hex.EncodeToString(encodedLogs)), cadence.NewUInt64(p.BlockHeight), cadence.String(p.BlockHash.String()), - cadence.String(errorMsg), cadence.String(hex.EncodeToString(p.Result.ReturnedData)), }).WithType(eventType), nil } From 341b455ede9355f5a8d7b6ef1015a702983e3317 Mon Sep 17 00:00:00 2001 From: ramtinms Date: Wed, 12 Jun 2024 18:35:51 -0700 Subject: [PATCH 40/60] add returned data --- fvm/evm/stdlib/contract.cdc | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/fvm/evm/stdlib/contract.cdc b/fvm/evm/stdlib/contract.cdc index 2e28d3c1c7c..96a9177a0d6 100644 --- a/fvm/evm/stdlib/contract.cdc +++ b/fvm/evm/stdlib/contract.cdc @@ -62,6 +62,13 @@ contract EVM { blockHeight: UInt64, // block hash in which transaction was included blockHash: String + /// captures the data that is returned from + /// the evm. For contract deployments + /// it returns the code deployed to + /// the address provided in the contractAddress field. + /// in case of revert, the smart contract custom error message + /// is also returned here (see EIP-140 for more details). + returnedData: ) access(all) From 6b328a1d42c2653edd938678b997f1d39e53dcae Mon Sep 17 00:00:00 2001 From: ramtinms Date: Wed, 12 Jun 2024 23:21:04 -0700 Subject: [PATCH 41/60] hot fix --- fvm/evm/stdlib/contract.cdc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fvm/evm/stdlib/contract.cdc b/fvm/evm/stdlib/contract.cdc index 96a9177a0d6..5a918772f1b 100644 --- a/fvm/evm/stdlib/contract.cdc +++ b/fvm/evm/stdlib/contract.cdc @@ -68,7 +68,7 @@ contract EVM { /// the address provided in the contractAddress field. /// in case of revert, the smart contract custom error message /// is also returned here (see EIP-140 for more details). - returnedData: + returnedData: [UInt8] ) access(all) From 11bdad354cdca6c0a75b1d23fd26d46896b5ec72 Mon Sep 17 00:00:00 2001 From: ramtinms Date: Wed, 12 Jun 2024 23:21:53 -0700 Subject: [PATCH 42/60] . --- fvm/evm/stdlib/contract.cdc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fvm/evm/stdlib/contract.cdc b/fvm/evm/stdlib/contract.cdc index 5a918772f1b..c15ee34a5b9 100644 --- a/fvm/evm/stdlib/contract.cdc +++ b/fvm/evm/stdlib/contract.cdc @@ -68,7 +68,7 @@ contract EVM { /// the address provided in the contractAddress field. /// in case of revert, the smart contract custom error message /// is also returned here (see EIP-140 for more details). - returnedData: [UInt8] + returnedData: String ) access(all) From 44fc4dc1b9227df3e49fdc7a51f677caa2182c58 Mon Sep 17 00:00:00 2001 From: ramtinms Date: Wed, 12 Jun 2024 23:22:15 -0700 Subject: [PATCH 43/60] update doc --- fvm/evm/stdlib/contract.cdc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fvm/evm/stdlib/contract.cdc b/fvm/evm/stdlib/contract.cdc index c15ee34a5b9..25f8a0c924a 100644 --- a/fvm/evm/stdlib/contract.cdc +++ b/fvm/evm/stdlib/contract.cdc @@ -62,7 +62,7 @@ contract EVM { blockHeight: UInt64, // block hash in which transaction was included blockHash: String - /// captures the data that is returned from + /// captures the hex encoded data that is returned from /// the evm. For contract deployments /// it returns the code deployed to /// the address provided in the contractAddress field. From 02feb0fabd65896fa931b0398e93e9fee8e07cc3 Mon Sep 17 00:00:00 2001 From: ramtinms Date: Wed, 12 Jun 2024 23:42:01 -0700 Subject: [PATCH 44/60] fix typo --- fvm/evm/stdlib/contract.cdc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fvm/evm/stdlib/contract.cdc b/fvm/evm/stdlib/contract.cdc index 25f8a0c924a..ededac7781d 100644 --- a/fvm/evm/stdlib/contract.cdc +++ b/fvm/evm/stdlib/contract.cdc @@ -61,7 +61,7 @@ contract EVM { // block height in which transaction was inclued blockHeight: UInt64, // block hash in which transaction was included - blockHash: String + blockHash: String, /// captures the hex encoded data that is returned from /// the evm. For contract deployments /// it returns the code deployed to From eb76ea497f3938b2ea8292082b641cf75e5c6c08 Mon Sep 17 00:00:00 2001 From: ramtinms Date: Thu, 13 Jun 2024 09:20:07 -0700 Subject: [PATCH 45/60] update state commitment --- engine/execution/state/bootstrap/bootstrap_test.go | 2 +- utils/unittest/execution_state.go | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/engine/execution/state/bootstrap/bootstrap_test.go b/engine/execution/state/bootstrap/bootstrap_test.go index a02820c3919..cd3edb6f2cb 100644 --- a/engine/execution/state/bootstrap/bootstrap_test.go +++ b/engine/execution/state/bootstrap/bootstrap_test.go @@ -53,7 +53,7 @@ func TestBootstrapLedger(t *testing.T) { } func TestBootstrapLedger_ZeroTokenSupply(t *testing.T) { - expectedStateCommitmentBytes, _ := hex.DecodeString("bcaf10db3920ea76a9b90d0f5b21d03cb2bc17087aa70554e4ebc01de6e166f1") + expectedStateCommitmentBytes, _ := hex.DecodeString("7cb3d30faaaab3cb402338023b3a068bc856fb788086b1212aa0f1950f24d854") expectedStateCommitment, err := flow.ToStateCommitment(expectedStateCommitmentBytes) require.NoError(t, err) diff --git a/utils/unittest/execution_state.go b/utils/unittest/execution_state.go index 28844644616..48c86d5b1ea 100644 --- a/utils/unittest/execution_state.go +++ b/utils/unittest/execution_state.go @@ -23,7 +23,7 @@ const ServiceAccountPrivateKeySignAlgo = crypto.ECDSAP256 const ServiceAccountPrivateKeyHashAlgo = hash.SHA2_256 // Pre-calculated state commitment with root account with the above private key -const GenesisStateCommitmentHex = "7bd7f9627241f803a2d31bb312ce409ef44b888a0d3d81dedf08357b1a652076" +const GenesisStateCommitmentHex = "ede86048a53464cdd3cef060e5e2e1603d91c2d2a0568417501af5ca9455f430" var GenesisStateCommitment flow.StateCommitment @@ -87,10 +87,10 @@ func genesisCommitHexByChainID(chainID flow.ChainID) string { return GenesisStateCommitmentHex } if chainID == flow.Testnet { - return "fc82b815c9cf75b8b217819f26d68b0da62f26c2a569fd1645cb0d9351e1e8bb" + return "ac9887d488bc90e33ce97478ca3f59c69795ed66061cf68d60d518be2fa0b658" } if chainID == flow.Sandboxnet { return "e1c08b17f9e5896f03fe28dd37ca396c19b26628161506924fbf785834646ea1" } - return "4f43005cc29f9f033e81262c73f89080ce6e6591e757519daaa59ec16b8b8a56" + return "eb6a5749ba1a4bd9ef5b3aa804b123e0d257f68735867e661b1a1418b26f9229" } From 292b9c176dd9090a8425a0788cbfea63fe48a219 Mon Sep 17 00:00:00 2001 From: Faye Amacker <33205765+fxamacker@users.noreply.github.com> Date: Fri, 14 Jun 2024 10:09:29 -0500 Subject: [PATCH 46/60] Optimize diff-states cmd to diff accts in parallel --- cmd/util/cmd/diff-states/cmd.go | 294 ++++++++++++++----- cmd/util/cmd/diff-states/diff_states_test.go | 21 +- 2 files changed, 232 insertions(+), 83 deletions(-) diff --git a/cmd/util/cmd/diff-states/cmd.go b/cmd/util/cmd/diff-states/cmd.go index b26a759b7a4..d58fbe2001e 100644 --- a/cmd/util/cmd/diff-states/cmd.go +++ b/cmd/util/cmd/diff-states/cmd.go @@ -2,6 +2,7 @@ package diff_states import ( "bytes" + "context" "encoding/hex" "encoding/json" "errors" @@ -17,6 +18,7 @@ import ( "github.com/onflow/flow-go/cmd/util/ledger/util/registers" "github.com/onflow/flow-go/ledger" "github.com/onflow/flow-go/model/flow" + moduleUtil "github.com/onflow/flow-go/module/util" ) var ( @@ -40,6 +42,13 @@ var Cmd = &cobra.Command{ const ReporterName = "state-diff" +type state uint8 + +const ( + oldState state = 1 + newState state = 2 +) + func init() { // Input 1 @@ -177,7 +186,10 @@ func run(*cobra.Command, []string) { } } - diff(registers1, registers2, chainID, rw) + err := diff(registers1, registers2, chainID, rw, flagNWorker) + if err != nil { + log.Warn().Err(err).Msgf("failed to diff registers") + } } func loadPayloads() (payloads1, payloads2 []*ledger.Payload) { @@ -266,108 +278,248 @@ func payloadsToRegisters(payloads1, payloads2 []*ledger.Payload) (registers1, re var accountsDiffer = errors.New("accounts differ") +func diffAccount( + owner string, + accountRegisters1 *registers.AccountRegisters, + accountRegisters2 *registers.AccountRegisters, + chainID flow.ChainID, + rw reporters.ReportWriter, +) (err error) { + + if accountRegisters1.Count() != accountRegisters2.Count() { + rw.Write(countDiff{ + Owner: owner, + State1: accountRegisters1.Count(), + State2: accountRegisters2.Count(), + }) + } + + err = accountRegisters1.ForEach(func(owner, key string, value1 []byte) error { + var value2 []byte + value2, err = accountRegisters2.Get(owner, key) + if err != nil { + return err + } + + if !bytes.Equal(value1, value2) { + + if flagRaw { + rw.Write(rawDiff{ + Owner: owner, + Key: key, + Value1: value1, + Value2: value2, + }) + } else { + // stop on first difference in accounts + return accountsDiffer + } + } + + return nil + }) + if err != nil { + if flagRaw || !errors.Is(err, accountsDiffer) { + return err + } + + address, err := common.BytesToAddress([]byte(owner)) + if err != nil { + return err + } + + migrations.NewCadenceValueDiffReporter( + address, + chainID, + rw, + true, + flagNWorker, + ).DiffStates( + accountRegisters1, + accountRegisters2, + migrations.AllStorageMapDomains, + ) + } + + return nil +} + func diff( registers1 *registers.ByAccount, registers2 *registers.ByAccount, chainID flow.ChainID, rw reporters.ReportWriter, -) { - log.Info().Msg("Diffing accounts") + nWorkers int, +) error { + log.Info().Msgf("Diffing %d accounts", registers1.AccountCount()) - err := registers1.ForEachAccount(func(accountRegisters1 *registers.AccountRegisters) (err error) { - owner := accountRegisters1.Owner() + if registers1.AccountCount() < nWorkers { + nWorkers = registers1.AccountCount() + } - if !registers2.HasAccountOwner(owner) { - rw.Write(accountMissing{ - Owner: owner, - State: 2, - }) + logAccount := moduleUtil.LogProgress( + log.Logger, + moduleUtil.DefaultLogProgressConfig( + "processing account group", + registers1.AccountCount(), + ), + ) - return nil - } + if nWorkers <= 1 { + foundAccountCountInRegisters2 := 0 - accountRegisters2 := registers2.AccountRegisters(owner) + _ = registers1.ForEachAccount(func(accountRegisters1 *registers.AccountRegisters) (err error) { + owner := accountRegisters1.Owner() - if accountRegisters1.Count() != accountRegisters2.Count() { - rw.Write(countDiff{ - Owner: owner, - State1: accountRegisters1.Count(), - State2: accountRegisters2.Count(), - }) - } + if !registers2.HasAccountOwner(owner) { + rw.Write(accountMissing{ + Owner: owner, + State: int(newState), + }) + + return nil + } - err = accountRegisters1.ForEach(func(owner, key string, value1 []byte) error { - var value2 []byte - value2, err = accountRegisters2.Get(owner, key) + foundAccountCountInRegisters2++ + + accountRegisters2 := registers2.AccountRegisters(owner) + + err = diffAccount( + owner, + accountRegisters1, + accountRegisters2, + chainID, + rw, + ) if err != nil { - return err + log.Warn().Err(err).Msgf("failed to diff account %x", []byte(owner)) } - if !bytes.Equal(value1, value2) { + logAccount(1) - if flagRaw { - rw.Write(rawDiff{ - Owner: owner, - Key: key, - Value1: value1, - Value2: value2, + return nil + }) + + if foundAccountCountInRegisters2 < registers2.AccountCount() { + _ = registers2.ForEachAccount(func(accountRegisters2 *registers.AccountRegisters) error { + owner := accountRegisters2.Owner() + if !registers1.HasAccountOwner(owner) { + rw.Write(accountMissing{ + Owner: owner, + State: int(oldState), }) - } else { - // stop on first difference in accounts - return accountsDiffer } - } + return nil + }) + } + + return nil + } + + type job struct { + owner string + accountRegisters1 *registers.AccountRegisters + accountRegisters2 *registers.AccountRegisters + } + + type result struct { + owner string + err error + } + + jobs := make(chan job, nWorkers) + results := make(chan result, nWorkers) + + g, ctx := errgroup.WithContext(context.Background()) + + // Launch goroutines to diff accounts + for i := 0; i < nWorkers; i++ { + g.Go(func() (err error) { + for job := range jobs { + err := diffAccount( + job.owner, + job.accountRegisters1, + job.accountRegisters2, + chainID, + rw, + ) + + select { + case results <- result{owner: job.owner, err: err}: + case <-ctx.Done(): + return ctx.Err() + } + } return nil }) - if err != nil { - if flagRaw || !errors.Is(err, accountsDiffer) { - return err - } + } - address, err := common.BytesToAddress([]byte(owner)) - if err != nil { - return err + // Launch goroutine to wait for workers and close result channel + go func() { + _ = g.Wait() + close(results) + }() + + // Launch goroutine to send account registers to jobs channel + go func() { + defer close(jobs) + + foundAccountCountInRegisters2 := 0 + + _ = registers1.ForEachAccount(func(accountRegisters1 *registers.AccountRegisters) (err error) { + owner := accountRegisters1.Owner() + if !registers2.HasAccountOwner(owner) { + rw.Write(accountMissing{ + Owner: owner, + State: int(newState), + }) + + return nil } - migrations.NewCadenceValueDiffReporter( - address, - chainID, - rw, - true, - flagNWorker, - ).DiffStates( - accountRegisters1, - accountRegisters2, - migrations.AllStorageMapDomains, - ) - } + foundAccountCountInRegisters2++ - return nil - }) - if err != nil { - log.Fatal().Err(err).Msg("failed to diff") - } + accountRegisters2 := registers2.AccountRegisters(owner) + + jobs <- job{ + owner: owner, + accountRegisters1: accountRegisters1, + accountRegisters2: accountRegisters2, + } - err = registers2.ForEachAccount(func(accountRegisters2 *registers.AccountRegisters) (err error) { - owner := accountRegisters2.Owner() + return nil + }) - if !registers1.HasAccountOwner(owner) { - rw.Write(accountMissing{ - Owner: owner, - State: 1, + if foundAccountCountInRegisters2 < registers2.AccountCount() { + _ = registers2.ForEachAccount(func(accountRegisters2 *registers.AccountRegisters) (err error) { + owner := accountRegisters2.Owner() + if !registers1.HasAccountOwner(owner) { + rw.Write(accountMissing{ + Owner: owner, + State: int(oldState), + }) + } + return nil }) - return nil } + }() - return nil - }) - if err != nil { - log.Fatal().Err(err).Msg("failed to diff") + // Gather results + for result := range results { + logAccount(1) + if result.err != nil { + log.Warn().Err(result.err).Msgf("failed to diff account %x", []byte(result.owner)) + } } - log.Info().Msg("Finished diffing accounts") + log.Info().Msgf("Finished diffing accounts, waiting for goroutines...") + + if err := g.Wait(); err != nil { + return err + } + return nil } type rawDiff struct { diff --git a/cmd/util/cmd/diff-states/diff_states_test.go b/cmd/util/cmd/diff-states/diff_states_test.go index 01c238ca3ab..fe6041dec5d 100644 --- a/cmd/util/cmd/diff-states/diff_states_test.go +++ b/cmd/util/cmd/diff-states/diff_states_test.go @@ -1,6 +1,7 @@ package diff_states import ( + "encoding/json" "io/fs" "path/filepath" "strings" @@ -82,18 +83,14 @@ func TestDiffStates(t *testing.T) { report, err := io.ReadFile(reportPath) require.NoError(t, err) - assert.JSONEq( - t, - ` - [ - {"kind":"raw-diff", "owner":"0100000000000000", "key":"62", "value1":"03", "value2":"05"}, - {"kind":"account-missing", "owner":"0200000000000000", "state":2}, - {"kind":"account-missing", "owner":"0300000000000000", "state":1}, - {"kind":"account-missing", "owner":"0400000000000000", "state":1} - ] - `, - string(report), - ) + var msgs []any + err = json.Unmarshal(report, &msgs) + require.NoError(t, err) + assert.Equal(t, 4, len(msgs)) + assert.Containsf(t, string(report), `{"kind":"raw-diff","owner":"0100000000000000","key":"62","value1":"03","value2":"05"}`, "diff report contains raw-diff") + assert.Containsf(t, string(report), `{"kind":"account-missing","owner":"0200000000000000","state":2}`, "diff report contains account-missing for 0200000000000000") + assert.Containsf(t, string(report), `{"kind":"account-missing","owner":"0300000000000000","state":1}`, "diff report contains account-missing for 0300000000000000") + assert.Containsf(t, string(report), `{"kind":"account-missing","owner":"0400000000000000","state":1}`, "diff report contains account-missing for 0400000000000000") }) } From 10708aa1d3437112d0f79b22f9d346da6d085da9 Mon Sep 17 00:00:00 2001 From: Faye Amacker <33205765+fxamacker@users.noreply.github.com> Date: Fri, 14 Jun 2024 10:28:32 -0500 Subject: [PATCH 47/60] Optimize diff-states cmd to diff domains in parallel --- .../ledger/migrations/cadence_value_diff.go | 145 ++++++++++++++---- 1 file changed, 114 insertions(+), 31 deletions(-) diff --git a/cmd/util/ledger/migrations/cadence_value_diff.go b/cmd/util/ledger/migrations/cadence_value_diff.go index a12dcd7126a..90d5172277d 100644 --- a/cmd/util/ledger/migrations/cadence_value_diff.go +++ b/cmd/util/ledger/migrations/cadence_value_diff.go @@ -3,8 +3,10 @@ package migrations import ( "fmt" + "github.com/onflow/cadence/runtime" "github.com/onflow/cadence/runtime/common" "github.com/onflow/cadence/runtime/interpreter" + "golang.org/x/sync/errgroup" "github.com/onflow/flow-go/cmd/util/ledger/reporters" "github.com/onflow/flow-go/cmd/util/ledger/util" @@ -72,6 +74,8 @@ type difference struct { NewValueStaticType string `json:",omitempty"` } +const minLargeAccountRegisterCount = 1_000 + type CadenceValueDiffReporter struct { address common.Address chainID flow.ChainID @@ -96,81 +100,130 @@ func NewCadenceValueDiffReporter( } } -func (dr *CadenceValueDiffReporter) newStorageRuntime( - registers registers.Registers, -) ( - *InterpreterMigrationRuntime, - error, -) { - // TODO: maybe make read-only again - runtimeInterface, err := NewInterpreterMigrationRuntime( - registers, - dr.chainID, - InterpreterMigrationRuntimeConfig{}, - ) - if err != nil { - return nil, err - } +func (dr *CadenceValueDiffReporter) DiffStates(oldRegs, newRegs registers.Registers, domains []string) { - return runtimeInterface, nil -} + oldStorage := newReadonlyStorage(oldRegs) -func (dr *CadenceValueDiffReporter) DiffStates(oldRegs, newRegs registers.Registers, domains []string) { + newStorage := newReadonlyStorage(newRegs) - // Create all the runtime components we need for comparing Cadence values. - oldRuntime, err := dr.newStorageRuntime(oldRegs) + var loadAtreeStorageGroup errgroup.Group + + loadAtreeStorageGroup.Go(func() (err error) { + return loadAtreeSlabsInStorage(oldStorage, oldRegs, dr.nWorkers) + }) + + err := loadAtreeSlabsInStorage(newStorage, newRegs, dr.nWorkers) if err != nil { dr.reportWriter.Write( diffError{ Address: dr.address.Hex(), Kind: diffErrorKindString[abortErrorKind], - Msg: fmt.Sprintf("failed to create runtime for old registers: %s", err), + Msg: fmt.Sprintf("failed to preload new atree registers: %s", err), }) return } - newRuntime, err := dr.newStorageRuntime(newRegs) - if err != nil { + // Wait for old registers to be loaded in storage. + if err := loadAtreeStorageGroup.Wait(); err != nil { dr.reportWriter.Write( diffError{ Address: dr.address.Hex(), Kind: diffErrorKindString[abortErrorKind], - Msg: fmt.Sprintf("failed to create runtime with new registers: %s", err), + Msg: fmt.Sprintf("failed to preload old atree registers: %s", err), }) return } - err = loadAtreeSlabsInStorage(oldRuntime.Storage, oldRegs, dr.nWorkers) + if oldRegs.Count() > minLargeAccountRegisterCount { + // Add concurrency to diff domains + var g errgroup.Group + + // NOTE: preload storage map in the same goroutine + for _, domain := range domains { + _ = oldStorage.GetStorageMap(dr.address, domain, false) + _ = newStorage.GetStorageMap(dr.address, domain, false) + } + + // Create goroutine to diff storage domain + g.Go(func() (err error) { + oldRuntime, err := newReadonlyStorageRuntimeWithStorage(oldStorage, oldRegs.Count()) + if err != nil { + return fmt.Errorf("failed to create runtime for old registers: %s", err) + } + + newRuntime, err := newReadonlyStorageRuntimeWithStorage(newStorage, newRegs.Count()) + if err != nil { + return fmt.Errorf("failed to create runtime for new registers: %s", err) + } + + dr.diffStorageDomain(oldRuntime, newRuntime, common.PathDomainStorage.Identifier()) + return nil + }) + + // Create goroutine to diff other domains + g.Go(func() (err error) { + oldRuntime, err := newReadonlyStorageRuntimeWithStorage(oldStorage, oldRegs.Count()) + if err != nil { + return fmt.Errorf("failed to create runtime for old registers: %s", err) + } + + newRuntime, err := newReadonlyStorageRuntimeWithStorage(newStorage, oldRegs.Count()) + if err != nil { + return fmt.Errorf("failed to create runtime for new registers: %s", err) + } + + for _, domain := range domains { + if domain != common.PathDomainStorage.Identifier() { + dr.diffStorageDomain(oldRuntime, newRuntime, domain) + } + } + return nil + }) + + err = g.Wait() + if err != nil { + dr.reportWriter.Write( + diffError{ + Address: dr.address.Hex(), + Kind: diffErrorKindString[abortErrorKind], + Msg: err.Error(), + }) + } + + return + } + + // Skip goroutine overhead for smaller accounts + oldRuntime, err := newReadonlyStorageRuntimeWithStorage(oldStorage, oldRegs.Count()) if err != nil { dr.reportWriter.Write( diffError{ Address: dr.address.Hex(), Kind: diffErrorKindString[abortErrorKind], - Msg: fmt.Sprintf("failed to preload old atree registers: %s", err), + Msg: fmt.Sprintf("failed to create runtime for old registers: %s", err), }) return } - err = loadAtreeSlabsInStorage(newRuntime.Storage, newRegs, dr.nWorkers) + newRuntime, err := newReadonlyStorageRuntimeWithStorage(newStorage, newRegs.Count()) if err != nil { dr.reportWriter.Write( diffError{ Address: dr.address.Hex(), Kind: diffErrorKindString[abortErrorKind], - Msg: fmt.Sprintf("failed to preload new atree registers: %s", err), + Msg: fmt.Sprintf("failed to create runtime with new registers: %s", err), }) return } - // Iterate through all domains and compare cadence values. for _, domain := range domains { dr.diffStorageDomain(oldRuntime, newRuntime, domain) } } func (dr *CadenceValueDiffReporter) diffStorageDomain( - oldRuntime *InterpreterMigrationRuntime, - newRuntime *InterpreterMigrationRuntime, + oldRuntime *readonlyStorageRuntime, + newRuntime *readonlyStorageRuntime, domain string, ) { defer func() { @@ -966,3 +1019,33 @@ func min(a, b int) int { } return b } + +func newReadonlyStorage(regs registers.Registers) *runtime.Storage { + ledger := ®isters.ReadOnlyLedger{Registers: regs} + return runtime.NewStorage(ledger, nil) +} + +type readonlyStorageRuntime struct { + Interpreter *interpreter.Interpreter + Storage *runtime.Storage + PayloadCount int +} + +func newReadonlyStorageRuntimeWithStorage(storage *runtime.Storage, payloadCount int) (*readonlyStorageRuntime, error) { + inter, err := interpreter.NewInterpreter( + nil, + nil, + &interpreter.Config{ + Storage: storage, + }, + ) + if err != nil { + return nil, err + } + + return &readonlyStorageRuntime{ + Interpreter: inter, + Storage: storage, + PayloadCount: payloadCount, + }, nil +} From 4de220468c4ab3e165d11cf3ea5e3f8a49536262 Mon Sep 17 00:00:00 2001 From: Faye Amacker <33205765+fxamacker@users.noreply.github.com> Date: Fri, 14 Jun 2024 10:47:16 -0500 Subject: [PATCH 48/60] Fix panic when comparing enum dict key in diff states Currently, dict key comparison uses nil *interpreter.Interpreter. This causes a panic if dict key is enum type and interpreter is needed to retrieve type info for comparison. This commit passes non-nil Interpreter for dict key comparisons. --- cmd/util/ledger/migrations/cadence_value_diff.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cmd/util/ledger/migrations/cadence_value_diff.go b/cmd/util/ledger/migrations/cadence_value_diff.go index a12dcd7126a..015e8c5ea9c 100644 --- a/cmd/util/ledger/migrations/cadence_value_diff.go +++ b/cmd/util/ledger/migrations/cadence_value_diff.go @@ -805,7 +805,7 @@ func (dr *CadenceValueDiffReporter) diffCadenceDictionaryValue( return true }) - onlyOldKeys, onlyNewKeys, sharedKeys := diffCadenceValues(oldKeys, newKeys) + onlyOldKeys, onlyNewKeys, sharedKeys := diffCadenceValues(vInterpreter, oldKeys, newKeys) // Log keys only present in old dict value if len(onlyOldKeys) > 0 { @@ -915,7 +915,7 @@ func diff[T comparable](old, new []T) (onlyOld, onlyNew, shared []T) { return } -func diffCadenceValues(old, new []interpreter.Value) (onlyOld, onlyNew, shared []interpreter.Value) { +func diffCadenceValues(oldInterpreter *interpreter.Interpreter, old, new []interpreter.Value) (onlyOld, onlyNew, shared []interpreter.Value) { onlyOld = make([]interpreter.Value, 0, len(old)) onlyNew = make([]interpreter.Value, 0, len(new)) shared = make([]interpreter.Value, 0, min(len(old), len(new))) @@ -929,7 +929,7 @@ func diffCadenceValues(old, new []interpreter.Value) (onlyOld, onlyNew, shared [ foundShared := false if ev, ok := o.(interpreter.EquatableValue); ok { - if ev.Equal(nil, interpreter.EmptyLocationRange, n) { + if ev.Equal(oldInterpreter, interpreter.EmptyLocationRange, n) { foundShared = true } } else { From a76e519cbe87b65f738132c4a52370e2449ea09c Mon Sep 17 00:00:00 2001 From: Faye Amacker <33205765+fxamacker@users.noreply.github.com> Date: Fri, 14 Jun 2024 15:14:36 -0500 Subject: [PATCH 49/60] Optimize diff-states to diff storage elems in parallel This adds additional parallelism to diff-states command to the concurrency added by PR #6097, which was limited to: - parallel diff of separate accounts - parallel diff of separate domains within large accounts Elements stored in storage domain can be deeply nested data, which takes time to compare. This commit compares elements in storage domain in parallel for large accounts. This requires creating Cadence interpreter in each worker goroutine for value comparison. Also added duration logging when diffing large accounts. --- cmd/util/cmd/diff-states/cmd.go | 2 +- .../ledger/migrations/cadence_value_diff.go | 157 ++++++++++++++++-- 2 files changed, 146 insertions(+), 13 deletions(-) diff --git a/cmd/util/cmd/diff-states/cmd.go b/cmd/util/cmd/diff-states/cmd.go index d58fbe2001e..ed1d13ef30a 100644 --- a/cmd/util/cmd/diff-states/cmd.go +++ b/cmd/util/cmd/diff-states/cmd.go @@ -333,7 +333,7 @@ func diffAccount( chainID, rw, true, - flagNWorker, + flagNWorker/2, ).DiffStates( accountRegisters1, accountRegisters2, diff --git a/cmd/util/ledger/migrations/cadence_value_diff.go b/cmd/util/ledger/migrations/cadence_value_diff.go index 90d5172277d..9eb892b127c 100644 --- a/cmd/util/ledger/migrations/cadence_value_diff.go +++ b/cmd/util/ledger/migrations/cadence_value_diff.go @@ -2,10 +2,12 @@ package migrations import ( "fmt" + "time" "github.com/onflow/cadence/runtime" "github.com/onflow/cadence/runtime/common" "github.com/onflow/cadence/runtime/interpreter" + "github.com/rs/zerolog/log" "golang.org/x/sync/errgroup" "github.com/onflow/flow-go/cmd/util/ledger/reporters" @@ -74,7 +76,7 @@ type difference struct { NewValueStaticType string `json:",omitempty"` } -const minLargeAccountRegisterCount = 1_000 +const minLargeAccountRegisterCount = 50_000 type CadenceValueDiffReporter struct { address common.Address @@ -156,7 +158,7 @@ func (dr *CadenceValueDiffReporter) DiffStates(oldRegs, newRegs registers.Regist return fmt.Errorf("failed to create runtime for new registers: %s", err) } - dr.diffStorageDomain(oldRuntime, newRuntime, common.PathDomainStorage.Identifier()) + dr.diffDomain(oldRuntime, newRuntime, common.PathDomainStorage.Identifier()) return nil }) @@ -174,7 +176,7 @@ func (dr *CadenceValueDiffReporter) DiffStates(oldRegs, newRegs registers.Regist for _, domain := range domains { if domain != common.PathDomainStorage.Identifier() { - dr.diffStorageDomain(oldRuntime, newRuntime, domain) + dr.diffDomain(oldRuntime, newRuntime, domain) } } return nil @@ -217,11 +219,11 @@ func (dr *CadenceValueDiffReporter) DiffStates(oldRegs, newRegs registers.Regist } for _, domain := range domains { - dr.diffStorageDomain(oldRuntime, newRuntime, domain) + dr.diffDomain(oldRuntime, newRuntime, domain) } } -func (dr *CadenceValueDiffReporter) diffStorageDomain( +func (dr *CadenceValueDiffReporter) diffDomain( oldRuntime *readonlyStorageRuntime, newRuntime *readonlyStorageRuntime, domain string, @@ -317,8 +319,11 @@ func (dr *CadenceValueDiffReporter) diffStorageDomain( }) } - // Compare elements present in both storage maps - for _, key := range sharedKeys { + if len(sharedKeys) == 0 { + return + } + + getValues := func(key any) (interpreter.Value, interpreter.Value, *util.Trace, bool) { trace := util.NewTrace(fmt.Sprintf("%s[%v]", domain, key)) @@ -350,17 +355,27 @@ func (dr *CadenceValueDiffReporter) diffStorageDomain( key, ), }) - continue + return nil, nil, nil, false } oldValue := oldStorageMap.ReadValue(nil, mapKey) newValue := newStorageMap.ReadValue(nil, mapKey) + return oldValue, newValue, trace, true + } + + diffValues := func( + oldInterpreter *interpreter.Interpreter, + oldValue interpreter.Value, + newInterpreter *interpreter.Interpreter, + newValue interpreter.Value, + trace *util.Trace, + ) { hasDifference := dr.diffValues( - oldRuntime.Interpreter, + oldInterpreter, oldValue, - newRuntime.Interpreter, + newInterpreter, newValue, domain, trace, @@ -377,13 +392,131 @@ func (dr *CadenceValueDiffReporter) diffStorageDomain( Trace: trace.String(), OldValue: oldValue.String(), NewValue: newValue.String(), - OldValueStaticType: oldValue.StaticType(oldRuntime.Interpreter).String(), - NewValueStaticType: newValue.StaticType(newRuntime.Interpreter).String(), + OldValueStaticType: oldValue.StaticType(oldInterpreter).String(), + NewValueStaticType: newValue.StaticType(newInterpreter).String(), }) } } + } + + // Skip goroutine overhead for non-storage domain and small accounts. + if domain != common.PathDomainStorage.Identifier() || + oldRuntime.PayloadCount < minLargeAccountRegisterCount || + len(sharedKeys) == 1 { + + for _, key := range sharedKeys { + oldValue, newValue, trace, canDiff := getValues(key) + if canDiff { + diffValues( + oldRuntime.Interpreter, + oldValue, + newRuntime.Interpreter, + newValue, + trace, + ) + } + } + return + } + + startTime := time.Now() + + log.Info().Msgf( + "Diffing %x storage domain containing %d elements (%d payloads) ...", + dr.address[:], + len(sharedKeys), + oldRuntime.PayloadCount, + ) + + // Diffing storage domain in large account + + type job struct { + oldValue interpreter.Value + newValue interpreter.Value + trace *util.Trace + } + nWorkers := dr.nWorkers + if len(sharedKeys) < nWorkers { + nWorkers = len(sharedKeys) } + + jobs := make(chan job, nWorkers) + + var g errgroup.Group + + for i := 0; i < nWorkers; i++ { + + g.Go(func() error { + oldInterpreter, err := interpreter.NewInterpreter( + nil, + nil, + &interpreter.Config{ + Storage: oldRuntime.Storage, + }, + ) + if err != nil { + dr.reportWriter.Write( + diffError{ + Address: dr.address.Hex(), + Kind: diffErrorKindString[abortErrorKind], + Msg: fmt.Sprintf("failed to create interpreter for old registers: %s", err), + }) + return nil + } + + newInterpreter, err := interpreter.NewInterpreter( + nil, + nil, + &interpreter.Config{ + Storage: newRuntime.Storage, + }, + ) + if err != nil { + dr.reportWriter.Write( + diffError{ + Address: dr.address.Hex(), + Kind: diffErrorKindString[abortErrorKind], + Msg: fmt.Sprintf("failed to create interpreter for new registers: %s", err), + }) + return nil + } + + for job := range jobs { + diffValues(oldInterpreter, job.oldValue, newInterpreter, job.newValue, job.trace) + } + + return nil + }) + } + + // Launch goroutine to send account registers to jobs channel + go func() { + defer close(jobs) + + for _, key := range sharedKeys { + oldValue, newValue, trace, canDiff := getValues(key) + if canDiff { + jobs <- job{ + oldValue: oldValue, + newValue: newValue, + trace: trace, + } + } + } + }() + + // Wait for workers + _ = g.Wait() + + log.Info(). + Msgf( + "Finished diffing %x storage domain containing %d elements (%d payloads) in %s", + dr.address[:], + len(sharedKeys), + oldRuntime.PayloadCount, + time.Since(startTime), + ) } func (dr *CadenceValueDiffReporter) diffValues( From a7a3c4bcdadc36bfb943d6ed6610146f9282a4b4 Mon Sep 17 00:00:00 2001 From: Faye Amacker <33205765+fxamacker@users.noreply.github.com> Date: Mon, 17 Jun 2024 07:20:47 -0500 Subject: [PATCH 50/60] Increase threshold of large account in diff-states This commit increases the minimum payload count threshold for large accounts to use extra goroutines. So only the largest accounts that would benefit most from extra goroutines use them. The minimum payload count was 50,000 and was increased to 1 million. --- cmd/util/ledger/migrations/cadence_value_diff.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/util/ledger/migrations/cadence_value_diff.go b/cmd/util/ledger/migrations/cadence_value_diff.go index 1a8ed656cb1..34c372ca603 100644 --- a/cmd/util/ledger/migrations/cadence_value_diff.go +++ b/cmd/util/ledger/migrations/cadence_value_diff.go @@ -76,7 +76,7 @@ type difference struct { NewValueStaticType string `json:",omitempty"` } -const minLargeAccountRegisterCount = 50_000 +const minLargeAccountRegisterCount = 1_000_000 type CadenceValueDiffReporter struct { address common.Address From bfb066da37e8189da78ac896d648731bc5c3b97a Mon Sep 17 00:00:00 2001 From: Faye Amacker <33205765+fxamacker@users.noreply.github.com> Date: Mon, 17 Jun 2024 07:53:31 -0500 Subject: [PATCH 51/60] Optimize diff-states by removing old bottleneck Currently, diffCadenceValues() is used in diffCadenceDictionaryValue() to compare dict keys in old and new dict values. The complexity is O(n^2) where n is number of keys. This commit removes explicit key comparisons in diffCadenceDictionaryValue() and detects not-found keys when retrieving elements for comparisons. The complexity is O(n) where n is number of keys. Speedup for 1 account using 10 goroutines (from diff-states logs on gcp): - before: 1 hour 9 minutes - after: 1 minute 18 seconds --- .../ledger/migrations/cadence_value_diff.go | 130 +++++++----------- 1 file changed, 52 insertions(+), 78 deletions(-) diff --git a/cmd/util/ledger/migrations/cadence_value_diff.go b/cmd/util/ledger/migrations/cadence_value_diff.go index 68fd2c2598b..b06765e1d61 100644 --- a/cmd/util/ledger/migrations/cadence_value_diff.go +++ b/cmd/util/ledger/migrations/cadence_value_diff.go @@ -858,9 +858,36 @@ func (dr *CadenceValueDiffReporter) diffCadenceDictionaryValue( return true }) - onlyOldKeys, onlyNewKeys, sharedKeys := diffCadenceValues(vInterpreter, oldKeys, newKeys) + onlyOldKeys := make([]interpreter.Value, 0, len(oldKeys)) + + // Compare elements in both dict values + + for _, key := range oldKeys { + valueTrace := trace.Append(fmt.Sprintf("[%v]", key)) + + oldValue, _ := v.Get(vInterpreter, interpreter.EmptyLocationRange, key) + + newValue, found := otherDictionary.Get(otherInterpreter, interpreter.EmptyLocationRange, key) + if !found { + onlyOldKeys = append(onlyOldKeys, key) + continue + } + + elementHasDifference := dr.diffValues( + vInterpreter, + oldValue, + otherInterpreter, + newValue, + domain, + valueTrace, + ) + if elementHasDifference { + hasDifference = true + } + } // Log keys only present in old dict value + if len(onlyOldKeys) > 0 { hasDifference = true @@ -878,42 +905,34 @@ func (dr *CadenceValueDiffReporter) diffCadenceDictionaryValue( }) } - // Log field names only present in new composite value - if len(onlyNewKeys) > 0 { - hasDifference = true - - dr.reportWriter.Write( - difference{ - Address: dr.address.Hex(), - Domain: domain, - Kind: diffKindString[cadenceValueDiffKind], - Trace: trace.String(), - Msg: fmt.Sprintf( - "new dict value has %d elements with keys %v, that are not present in old dict value", - len(onlyNewKeys), - onlyNewKeys, - ), - }) - } - - // Compare elements in both dict values - for _, key := range sharedKeys { - valueTrace := trace.Append(fmt.Sprintf("[%v]", key)) + // Log keys only present in new dict value - oldValue, _ := v.Get(vInterpreter, interpreter.EmptyLocationRange, key) + if len(oldKeys) != len(newKeys) || len(onlyOldKeys) > 0 { + onlyNewKeys := make([]interpreter.Value, 0, len(newKeys)) - newValue, _ := otherDictionary.Get(otherInterpreter, interpreter.EmptyLocationRange, key) + // find keys only present in new dict + for _, key := range newKeys { + found := v.ContainsKey(vInterpreter, interpreter.EmptyLocationRange, key) + if !found { + onlyNewKeys = append(onlyNewKeys, key) + } + } - elementHasDifference := dr.diffValues( - vInterpreter, - oldValue, - otherInterpreter, - newValue, - domain, - valueTrace, - ) - if elementHasDifference { + if len(onlyNewKeys) > 0 { hasDifference = true + + dr.reportWriter.Write( + difference{ + Address: dr.address.Hex(), + Domain: domain, + Kind: diffKindString[cadenceValueDiffKind], + Trace: trace.String(), + Msg: fmt.Sprintf( + "new dict value has %d elements with keys %v, that are not present in old dict value", + len(onlyNewKeys), + onlyNewKeys, + ), + }) } } @@ -968,51 +987,6 @@ func diff[T comparable](old, new []T) (onlyOld, onlyNew, shared []T) { return } -func diffCadenceValues(oldInterpreter *interpreter.Interpreter, old, new []interpreter.Value) (onlyOld, onlyNew, shared []interpreter.Value) { - onlyOld = make([]interpreter.Value, 0, len(old)) - onlyNew = make([]interpreter.Value, 0, len(new)) - shared = make([]interpreter.Value, 0, min(len(old), len(new))) - - sharedNew := make([]bool, len(new)) - - for _, o := range old { - found := false - - for i, n := range new { - foundShared := false - - if ev, ok := o.(interpreter.EquatableValue); ok { - if ev.Equal(oldInterpreter, interpreter.EmptyLocationRange, n) { - foundShared = true - } - } else { - if o == n { - foundShared = true - } - } - - if foundShared { - shared = append(shared, o) - found = true - sharedNew[i] = true - break - } - } - - if !found { - onlyOld = append(onlyOld, o) - } - } - - for i, shared := range sharedNew { - if !shared { - onlyNew = append(onlyNew, new[i]) - } - } - - return -} - func min(a, b int) int { if a <= b { return a From 27b974725b290cb34b85203da100c44b126dd79e Mon Sep 17 00:00:00 2001 From: Faye Amacker <33205765+fxamacker@users.noreply.github.com> Date: Mon, 17 Jun 2024 12:18:50 -0500 Subject: [PATCH 52/60] Check non-nil error (currently always nil) in diff-states This commit checks for error returned from a goroutine which currently does not return any error. This check is done as a precaution in case the goroutine is modified in the future to return error. --- cmd/util/ledger/migrations/cadence_value_diff.go | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/cmd/util/ledger/migrations/cadence_value_diff.go b/cmd/util/ledger/migrations/cadence_value_diff.go index 34c372ca603..706fa4f71a9 100644 --- a/cmd/util/ledger/migrations/cadence_value_diff.go +++ b/cmd/util/ledger/migrations/cadence_value_diff.go @@ -507,7 +507,15 @@ func (dr *CadenceValueDiffReporter) diffDomain( }() // Wait for workers - _ = g.Wait() + err := g.Wait() + if err != nil { + dr.reportWriter.Write( + diffError{ + Address: dr.address.Hex(), + Kind: diffErrorKindString[abortErrorKind], + Msg: fmt.Sprintf("failed to diff domain %s: %s", domain, err), + }) + } log.Info(). Msgf( From 114eb0066a31ff23c28f87671b418d7ea86961b9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastian=20M=C3=BCller?= Date: Tue, 18 Jun 2024 11:48:45 -0700 Subject: [PATCH 53/60] add a flag which forces value comparison --- cmd/util/cmd/diff-states/cmd.go | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/cmd/util/cmd/diff-states/cmd.go b/cmd/util/cmd/diff-states/cmd.go index ed1d13ef30a..414f7147d36 100644 --- a/cmd/util/cmd/diff-states/cmd.go +++ b/cmd/util/cmd/diff-states/cmd.go @@ -30,6 +30,7 @@ var ( flagStateCommitment1 string flagStateCommitment2 string flagRaw bool + flagAlwaysDiffValues bool flagNWorker int flagChain string ) @@ -113,6 +114,13 @@ func init() { "Raw or value", ) + Cmd.Flags().BoolVar( + &flagAlwaysDiffValues, + "always-diff-values", + false, + "always diff on value level. useful when trying to test iteration, by diffing same state.", + ) + Cmd.Flags().IntVar( &flagNWorker, "n-worker", @@ -294,6 +302,8 @@ func diffAccount( }) } + diffValues := flagAlwaysDiffValues + err = accountRegisters1.ForEach(func(owner, key string, value1 []byte) error { var value2 []byte value2, err = accountRegisters2.Get(owner, key) @@ -323,6 +333,10 @@ func diffAccount( return err } + diffValues = true + } + + if diffValues { address, err := common.BytesToAddress([]byte(owner)) if err != nil { return err From 0ad28ccd8d8eb2901af744f3705ef80a0db52b92 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastian=20M=C3=BCller?= Date: Tue, 18 Jun 2024 11:52:17 -0700 Subject: [PATCH 54/60] profile whole state extraction, not just migration a full profile is useful for e.g. profile-guided optimization --- cmd/util/cmd/execution-state-extract/cmd.go | 15 +++++++++++++++ .../execution_state_extract.go | 14 -------------- 2 files changed, 15 insertions(+), 14 deletions(-) diff --git a/cmd/util/cmd/execution-state-extract/cmd.go b/cmd/util/cmd/execution-state-extract/cmd.go index f9701e503c3..b8920226a4a 100644 --- a/cmd/util/cmd/execution-state-extract/cmd.go +++ b/cmd/util/cmd/execution-state-extract/cmd.go @@ -5,6 +5,7 @@ import ( "fmt" "os" "path" + "runtime/pprof" "strings" "github.com/rs/zerolog/log" @@ -178,6 +179,20 @@ func init() { } func run(*cobra.Command, []string) { + if flagCPUProfile != "" { + f, err := os.Create(flagCPUProfile) + if err != nil { + log.Fatal().Err(err).Msg("could not create CPU profile") + } + + err = pprof.StartCPUProfile(f) + if err != nil { + log.Fatal().Err(err).Msg("could not start CPU profile") + } + + defer pprof.StopCPUProfile() + } + var stateCommitment flow.StateCommitment if len(flagBlockHash) > 0 && len(flagStateCommitment) > 0 { diff --git a/cmd/util/cmd/execution-state-extract/execution_state_extract.go b/cmd/util/cmd/execution-state-extract/execution_state_extract.go index 319f5fc057e..f53b53167c3 100644 --- a/cmd/util/cmd/execution-state-extract/execution_state_extract.go +++ b/cmd/util/cmd/execution-state-extract/execution_state_extract.go @@ -6,7 +6,6 @@ import ( "fmt" "math" "os" - "runtime/pprof" syncAtomic "sync/atomic" "time" @@ -325,19 +324,6 @@ func newMigration( nWorker int, ) ledger.Migration { return func(payloads []*ledger.Payload) ([]*ledger.Payload, error) { - if flagCPUProfile != "" { - f, err := os.Create(flagCPUProfile) - if err != nil { - logger.Fatal().Err(err).Msg("could not create CPU profile") - } - - err = pprof.StartCPUProfile(f) - if err != nil { - logger.Fatal().Err(err).Msg("could not start CPU profile") - } - - defer pprof.StopCPUProfile() - } if len(migrations) == 0 { return payloads, nil From 10c04d7296eec94f7b0ac48a0c7e7af35c9d2899 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastian=20M=C3=BCller?= Date: Tue, 18 Jun 2024 11:53:59 -0700 Subject: [PATCH 55/60] expose encoding and decoding of contract names register --- fvm/environment/accounts.go | 43 +++++++++++++++++++++++++------------ 1 file changed, 29 insertions(+), 14 deletions(-) diff --git a/fvm/environment/accounts.go b/fvm/environment/accounts.go index 01041f19a3f..b22ee29cb50 100644 --- a/fvm/environment/accounts.go +++ b/fvm/environment/accounts.go @@ -431,6 +431,20 @@ func (a *StatefulAccounts) setContract( return nil } +func EncodeContractNames(contractNames contractNames) ([]byte, error) { + var buf bytes.Buffer + cborEncoder := cbor.NewEncoder(&buf) + err := cborEncoder.Encode(contractNames) + if err != nil { + return nil, errors.NewEncodingFailuref( + err, + "cannot encode contract names: %s", + contractNames, + ) + } + return buf.Bytes(), nil +} + func (a *StatefulAccounts) setContractNames( contractNames contractNames, address flow.Address, @@ -443,16 +457,11 @@ func (a *StatefulAccounts) setContractNames( if !ok { return errors.NewAccountNotFoundError(address) } - var buf bytes.Buffer - cborEncoder := cbor.NewEncoder(&buf) - err = cborEncoder.Encode(contractNames) + + newContractNames, err := EncodeContractNames(contractNames) if err != nil { - return errors.NewEncodingFailuref( - err, - "cannot encode contract names: %s", - contractNames) + return err } - newContractNames := buf.Bytes() id := flow.ContractNamesRegisterID(address) prevContractNames, err := a.GetValue(id) @@ -607,20 +616,26 @@ func (a *StatefulAccounts) getContractNames( error, ) { // TODO return fatal error if can't fetch - encContractNames, err := a.GetValue(flow.ContractNamesRegisterID(address)) + encodedContractNames, err := a.GetValue(flow.ContractNamesRegisterID(address)) if err != nil { return nil, fmt.Errorf("cannot get deployed contract names: %w", err) } + + return DecodeContractNames(encodedContractNames) +} + +func DecodeContractNames(encodedContractNames []byte) ([]string, error) { identifiers := make([]string, 0) - if len(encContractNames) > 0 { - buf := bytes.NewReader(encContractNames) + if len(encodedContractNames) > 0 { + buf := bytes.NewReader(encodedContractNames) cborDecoder := cbor.NewDecoder(buf) - err = cborDecoder.Decode(&identifiers) + err := cborDecoder.Decode(&identifiers) if err != nil { return nil, fmt.Errorf( "cannot decode deployed contract names %x: %w", - encContractNames, - err) + encodedContractNames, + err, + ) } } return identifiers, nil From 81309672270436f17088b95b01191393c214e51e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastian=20M=C3=BCller?= Date: Tue, 18 Jun 2024 11:54:31 -0700 Subject: [PATCH 56/60] get contract names from contract names register, avoid iteration over all registers --- .../cadence_values_migration_test.go | 10 ++++ .../migrations/contract_checking_migration.go | 53 ++++++++++++------- 2 files changed, 44 insertions(+), 19 deletions(-) diff --git a/cmd/util/ledger/migrations/cadence_values_migration_test.go b/cmd/util/ledger/migrations/cadence_values_migration_test.go index 882952b9362..cb65054c35d 100644 --- a/cmd/util/ledger/migrations/cadence_values_migration_test.go +++ b/cmd/util/ledger/migrations/cadence_values_migration_test.go @@ -1274,6 +1274,16 @@ func TestProgramParsingError(t *testing.T) { ) require.NoError(t, err) + encodedContractNames, err := environment.EncodeContractNames([]string{contractName}) + require.NoError(t, err) + + err = registersByAccount.Set( + string(testAddress[:]), + flow.ContractNamesKey, + encodedContractNames, + ) + require.NoError(t, err) + // Migrate // TODO: EVM contract is not deployed in snapshot yet, so can't update it diff --git a/cmd/util/ledger/migrations/contract_checking_migration.go b/cmd/util/ledger/migrations/contract_checking_migration.go index d891ed5c966..668c644f70c 100644 --- a/cmd/util/ledger/migrations/contract_checking_migration.go +++ b/cmd/util/ledger/migrations/contract_checking_migration.go @@ -12,6 +12,7 @@ import ( "github.com/onflow/flow-go/cmd/util/ledger/reporters" "github.com/onflow/flow-go/cmd/util/ledger/util/registers" + "github.com/onflow/flow-go/fvm/environment" "github.com/onflow/flow-go/model/flow" ) @@ -50,35 +51,49 @@ func NewContractCheckingMigration( } contracts := make([]contract, 0, contractCountEstimate) - err = registersByAccount.ForEach(func(owner string, key string, value []byte) error { + err = registersByAccount.ForEachAccount(func(accountRegisters *registers.AccountRegisters) error { + owner := accountRegisters.Owner() - // Skip payloads that are not contract code - contractName := flow.KeyContractName(key) - if contractName == "" { - return nil + encodedContractNames, err := accountRegisters.Get(owner, flow.ContractNamesKey) + if err != nil { + return err } - address := common.Address([]byte(owner)) - code := value - location := common.AddressLocation{ - Address: address, - Name: contractName, + contractNames, err := environment.DecodeContractNames(encodedContractNames) + if err != nil { + return err } - contracts = append( - contracts, - contract{ - location: location, - code: code, - }, - ) + for _, contractName := range contractNames { + + contractKey := flow.ContractKey(contractName) + + code, err := accountRegisters.Get(owner, contractKey) + if err != nil { + return err + } - contractsForPrettyPrinting[location] = code + address := common.Address([]byte(owner)) + location := common.AddressLocation{ + Address: address, + Name: contractName, + } + + contracts = append( + contracts, + contract{ + location: location, + code: code, + }, + ) + + contractsForPrettyPrinting[location] = code + } return nil }) if err != nil { - return fmt.Errorf("failed to iterate over registers: %w", err) + return fmt.Errorf("failed to get contracts of accounts: %w", err) } sort.Slice(contracts, func(i, j int) bool { From db5fd33dfe5afafa6da4cc94e8bbbf5307472a1f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastian=20M=C3=BCller?= Date: Tue, 18 Jun 2024 12:50:56 -0700 Subject: [PATCH 57/60] log gathering of contracts --- cmd/util/ledger/migrations/contract_checking_migration.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/cmd/util/ledger/migrations/contract_checking_migration.go b/cmd/util/ledger/migrations/contract_checking_migration.go index 668c644f70c..473c6679ed7 100644 --- a/cmd/util/ledger/migrations/contract_checking_migration.go +++ b/cmd/util/ledger/migrations/contract_checking_migration.go @@ -43,6 +43,8 @@ func NewContractCheckingMigration( // Gather all contracts + log.Info().Msg("Gathering contracts ...") + contractsForPrettyPrinting := make(map[common.Location][]byte, contractCountEstimate) type contract struct { @@ -102,6 +104,8 @@ func NewContractCheckingMigration( return a.location.ID() < b.location.ID() }) + log.Info().Msgf("Gathered all contracts (%d)", len(contracts)) + // Check all contracts for _, contract := range contracts { From f9c68391eb299688935a0ced918b6a0313ff75ee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastian=20M=C3=BCller?= Date: Tue, 18 Jun 2024 12:51:10 -0700 Subject: [PATCH 58/60] defer close of reporter --- cmd/util/ledger/migrations/contract_checking_migration.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/cmd/util/ledger/migrations/contract_checking_migration.go b/cmd/util/ledger/migrations/contract_checking_migration.go index 473c6679ed7..6683ee5569a 100644 --- a/cmd/util/ledger/migrations/contract_checking_migration.go +++ b/cmd/util/ledger/migrations/contract_checking_migration.go @@ -31,6 +31,7 @@ func NewContractCheckingMigration( return func(registersByAccount *registers.ByAccount) error { reporter := rwf.ReportWriter(contractCheckingReporterName) + defer reporter.Close() mr, err := NewInterpreterMigrationRuntime( registersByAccount, @@ -153,8 +154,6 @@ func NewContractCheckingMigration( } } - reporter.Close() - return nil } } From bc8a250186564a24e4e95d8d343ad77ecbf409a9 Mon Sep 17 00:00:00 2001 From: "Leo Zhang (zhangchiqing)" Date: Thu, 13 Jun 2024 14:55:56 -0700 Subject: [PATCH 59/60] log epoch height view --- network/p2p/cache/protocol_state_provider.go | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/network/p2p/cache/protocol_state_provider.go b/network/p2p/cache/protocol_state_provider.go index 6f7a4462b5b..cf782b46b5f 100644 --- a/network/p2p/cache/protocol_state_provider.go +++ b/network/p2p/cache/protocol_state_provider.go @@ -70,7 +70,10 @@ func NewProtocolStateIDCache( // and virtually latency free. However, we run data base queries and acquire locks here, // which is undesired. func (p *ProtocolStateIDCache) EpochTransition(newEpochCounter uint64, header *flow.Header) { - p.logger.Info().Uint64("newEpochCounter", newEpochCounter).Msg("epoch transition") + p.logger.Info().Uint64("newEpochCounter", newEpochCounter). + Uint64("height", header.Height). + Uint64("view", header.View). + Msg("epoch transition") p.update(header.ID()) } @@ -82,7 +85,10 @@ func (p *ProtocolStateIDCache) EpochTransition(newEpochCounter uint64, header *f // and virtually latency free. However, we run data base queries and acquire locks here, // which is undesired. func (p *ProtocolStateIDCache) EpochSetupPhaseStarted(currentEpochCounter uint64, header *flow.Header) { - p.logger.Info().Uint64("currentEpochCounter", currentEpochCounter).Msg("epoch setup phase started") + p.logger.Info().Uint64("currentEpochCounter", currentEpochCounter). + Uint64("height", header.Height). + Uint64("view", header.View). + Msg("epoch setup phase started") p.update(header.ID()) } @@ -94,7 +100,10 @@ func (p *ProtocolStateIDCache) EpochSetupPhaseStarted(currentEpochCounter uint64 // and virtually latency free. However, we run data base queries and acquire locks here, // which is undesired. func (p *ProtocolStateIDCache) EpochCommittedPhaseStarted(currentEpochCounter uint64, header *flow.Header) { - p.logger.Info().Uint64("currentEpochCounter", currentEpochCounter).Msg("epoch committed phase started") + p.logger.Info().Uint64("currentEpochCounter", currentEpochCounter). + Uint64("height", header.Height). + Uint64("view", header.View). + Msg("epoch committed phase started") p.update(header.ID()) } From 5b3dda96897aa4ccbf236bf834295d8d7cc7f871 Mon Sep 17 00:00:00 2001 From: "Leo Zhang (zhangchiqing)" Date: Tue, 18 Jun 2024 16:32:16 -0700 Subject: [PATCH 60/60] create protocol event logger --- cmd/scaffold.go | 5 +++ network/p2p/cache/protocol_state_provider.go | 12 ------ state/protocol/events/logger.go | 42 ++++++++++++++++++++ 3 files changed, 47 insertions(+), 12 deletions(-) create mode 100644 state/protocol/events/logger.go diff --git a/cmd/scaffold.go b/cmd/scaffold.go index 734e8371b92..5db4e75c395 100644 --- a/cmd/scaffold.go +++ b/cmd/scaffold.go @@ -471,6 +471,11 @@ func (fnb *FlowNodeBuilder) EnqueueNetworkInit() { peerManagerFilters) }) + fnb.Module("epoch transition logger", func(node *NodeConfig) error { + node.ProtocolEvents.AddConsumer(events.NewEventLogger(node.Logger)) + return nil + }) + fnb.Module("network underlay dependency", func(node *NodeConfig) error { fnb.networkUnderlayDependable = module.NewProxiedReadyDoneAware() fnb.PeerManagerDependencies.Add(fnb.networkUnderlayDependable) diff --git a/network/p2p/cache/protocol_state_provider.go b/network/p2p/cache/protocol_state_provider.go index cf782b46b5f..5d4d5fa06e1 100644 --- a/network/p2p/cache/protocol_state_provider.go +++ b/network/p2p/cache/protocol_state_provider.go @@ -70,10 +70,6 @@ func NewProtocolStateIDCache( // and virtually latency free. However, we run data base queries and acquire locks here, // which is undesired. func (p *ProtocolStateIDCache) EpochTransition(newEpochCounter uint64, header *flow.Header) { - p.logger.Info().Uint64("newEpochCounter", newEpochCounter). - Uint64("height", header.Height). - Uint64("view", header.View). - Msg("epoch transition") p.update(header.ID()) } @@ -85,10 +81,6 @@ func (p *ProtocolStateIDCache) EpochTransition(newEpochCounter uint64, header *f // and virtually latency free. However, we run data base queries and acquire locks here, // which is undesired. func (p *ProtocolStateIDCache) EpochSetupPhaseStarted(currentEpochCounter uint64, header *flow.Header) { - p.logger.Info().Uint64("currentEpochCounter", currentEpochCounter). - Uint64("height", header.Height). - Uint64("view", header.View). - Msg("epoch setup phase started") p.update(header.ID()) } @@ -100,10 +92,6 @@ func (p *ProtocolStateIDCache) EpochSetupPhaseStarted(currentEpochCounter uint64 // and virtually latency free. However, we run data base queries and acquire locks here, // which is undesired. func (p *ProtocolStateIDCache) EpochCommittedPhaseStarted(currentEpochCounter uint64, header *flow.Header) { - p.logger.Info().Uint64("currentEpochCounter", currentEpochCounter). - Uint64("height", header.Height). - Uint64("view", header.View). - Msg("epoch committed phase started") p.update(header.ID()) } diff --git a/state/protocol/events/logger.go b/state/protocol/events/logger.go new file mode 100644 index 00000000000..942ba00e480 --- /dev/null +++ b/state/protocol/events/logger.go @@ -0,0 +1,42 @@ +package events + +import ( + "github.com/rs/zerolog" + + "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/state/protocol" +) + +type EventLogger struct { + Noop // satisfy protocol events consumer interface + logger zerolog.Logger +} + +var _ protocol.Consumer = (*EventLogger)(nil) + +func NewEventLogger(logger zerolog.Logger) *EventLogger { + return &EventLogger{ + logger: logger.With().Str("module", "protocol_events_logger").Logger(), + } +} + +func (p EventLogger) EpochTransition(newEpochCounter uint64, header *flow.Header) { + p.logger.Info().Uint64("newEpochCounter", newEpochCounter). + Uint64("height", header.Height). + Uint64("view", header.View). + Msg("epoch transition") +} + +func (p EventLogger) EpochSetupPhaseStarted(currentEpochCounter uint64, header *flow.Header) { + p.logger.Info().Uint64("currentEpochCounter", currentEpochCounter). + Uint64("height", header.Height). + Uint64("view", header.View). + Msg("epoch setup phase started") +} + +func (p EventLogger) EpochCommittedPhaseStarted(currentEpochCounter uint64, header *flow.Header) { + p.logger.Info().Uint64("currentEpochCounter", currentEpochCounter). + Uint64("height", header.Height). + Uint64("view", header.View). + Msg("epoch committed phase started") +}