From 494bbe64143ec845a772d8e6fef9d3d2bd8f724f Mon Sep 17 00:00:00 2001 From: Faye Amacker <33205765+fxamacker@users.noreply.github.com> Date: Wed, 15 Oct 2025 08:52:37 -0500 Subject: [PATCH] Remove unused storage func FixLoadedBrokenReferences This commit removes PersistentSlabStorage.FixLoadedBrokenReferences() which is no longer needed. It was added for FixSlabsWithBrokenReferencesMigration for Crescendo spork to fix testnet data (not mainnet). Since the migration using it was removed in flow-go #8036, this function is longer needed in atree. --- storage.go | 228 ------ storage_test.go | 1886 ----------------------------------------------- 2 files changed, 2114 deletions(-) diff --git a/storage.go b/storage.go index 7684f457..69955944 100644 --- a/storage.go +++ b/storage.go @@ -20,7 +20,6 @@ package atree import ( "encoding/binary" - "errors" "fmt" "sort" "strings" @@ -1013,233 +1012,6 @@ func (s *PersistentSlabStorage) DeltasSizeWithoutTempAddresses() uint64 { return size } -// FixLoadedBrokenReferences traverses loaded slabs and fixes broken references in maps. -// A broken reference is a SlabID referencing a non-existent slab. -// To fix a map containing broken references, this function replaces broken map with -// empty map having the same SlabID and also removes all slabs in the old map. -// Limitations: -// - only fix broken references in map -// - only traverse loaded slabs in deltas and cache -// NOTE: The intended use case is to enable migration programs in onflow/flow-go to -// fix broken references. As of April 2024, only 10 registers in testnet (not mainnet) -// were found to have broken references and they seem to have resulted from a bug -// that was fixed 2 years ago by https://github.com/onflow/cadence/pull/1565. -func (s *PersistentSlabStorage) FixLoadedBrokenReferences(needToFix func(old Value) bool) ( - fixedSlabIDs map[SlabID][]SlabID, // key: root slab ID, value: slab IDs containing broken refs - skippedSlabIDs map[SlabID][]SlabID, // key: root slab ID, value: slab IDs containing broken refs - err error, -) { - - // parentOf is used to find root slab from non-root slab. - // Broken reference can be in non-root slab, and we need SlabID of root slab - // to replace broken map by creating an empty new map with same SlabID. - parentOf := make(map[SlabID]SlabID) - - getRootSlabID := func(id SlabID) SlabID { - for { - parentID, ok := parentOf[id] - if ok { - id = parentID - } else { - return id - } - } - } - - hasBrokenReferenceInSlab := func(id SlabID, slab Slab) bool { - if slab == nil { - return false - } - - switch slab.(type) { - case *ArrayMetaDataSlab, *MapMetaDataSlab: // metadata slabs - var foundBrokenRef bool - - for _, childStorable := range slab.ChildStorables() { - - if slabIDStorable, ok := childStorable.(SlabIDStorable); ok { - - childID := SlabID(slabIDStorable) - - // Track parent-child relationship of root slabs and non-root slabs. - parentOf[childID] = id - - if !s.existIfLoaded(childID) { - foundBrokenRef = true - } - - // Continue with remaining child storables to track parent-child relationship. - } - } - - return foundBrokenRef - - default: // data slabs - childStorables := slab.ChildStorables() - - for len(childStorables) > 0 { - - var nextChildStorables []Storable - - for _, childStorable := range childStorables { - - if slabIDStorable, ok := childStorable.(SlabIDStorable); ok { - - if !s.existIfLoaded(SlabID(slabIDStorable)) { - return true - } - - continue - } - - // Append child storables of this childStorable to - // handle nested SlabIDStorable, such as Cadence SomeValue. - nextChildStorables = append( - nextChildStorables, - childStorable.ChildStorables()..., - ) - } - - childStorables = nextChildStorables - } - - return false - } - } - - var brokenSlabIDs []SlabID - - // Iterate delta slabs. - for id, slab := range s.deltas { - if hasBrokenReferenceInSlab(id, slab) { - brokenSlabIDs = append(brokenSlabIDs, id) - } - } - - // Iterate cache slabs. - for id, slab := range s.cache { - if _, ok := s.deltas[id]; ok { - continue - } - if hasBrokenReferenceInSlab(id, slab) { - brokenSlabIDs = append(brokenSlabIDs, id) - } - } - - if len(brokenSlabIDs) == 0 { - return nil, nil, nil - } - - rootSlabIDsWithBrokenData := make(map[SlabID][]SlabID) - var errs []error - - // Find SlabIDs of root slab for slabs containing broken references. - for _, id := range brokenSlabIDs { - rootID := getRootSlabID(id) - if rootID == SlabIDUndefined { - errs = append(errs, fmt.Errorf("failed to get root slab id for slab %s", id)) - continue - } - rootSlabIDsWithBrokenData[rootID] = append(rootSlabIDsWithBrokenData[rootID], id) - } - - for rootSlabID, brokenSlabIDs := range rootSlabIDsWithBrokenData { - rootSlab := s.RetrieveIfLoaded(rootSlabID) - if rootSlab == nil { - errs = append(errs, fmt.Errorf("failed to retrieve loaded root slab %s", rootSlabID)) - continue - } - - switch rootSlab := rootSlab.(type) { - case MapSlab: - value, err := rootSlab.StoredValue(s) - if err != nil { - errs = append(errs, fmt.Errorf("failed to convert slab %s into value", rootSlab.SlabID())) - continue - } - - if needToFix(value) { - err := s.fixBrokenReferencesInMap(rootSlab) - if err != nil { - errs = append(errs, err) - continue - } - } else { - if skippedSlabIDs == nil { - skippedSlabIDs = make(map[SlabID][]SlabID) - } - skippedSlabIDs[rootSlabID] = brokenSlabIDs - } - - default: - // IMPORTANT: Only handle map slabs for now. DO NOT silently fix currently unknown problems. - errs = append(errs, fmt.Errorf("failed to fix broken references in non-map slab %s (%T)", rootSlab.SlabID(), rootSlab)) - } - } - - for id := range skippedSlabIDs { - delete(rootSlabIDsWithBrokenData, id) - } - - return rootSlabIDsWithBrokenData, skippedSlabIDs, errors.Join(errs...) -} - -// fixBrokenReferencesInMap replaces replaces broken map with empty map -// having the same SlabID and also removes all slabs in the old map. -func (s *PersistentSlabStorage) fixBrokenReferencesInMap(old MapSlab) error { - id := old.SlabID() - - oldExtraData := old.ExtraData() - - // Create an empty map with the same StorgeID, type, and seed as the old map. - newMap := &MapDataSlab{ - header: MapSlabHeader{ - slabID: id, - size: mapRootDataSlabPrefixSize + hkeyElementsPrefixSize, - }, - extraData: &MapExtraData{ - TypeInfo: oldExtraData.TypeInfo, - Seed: oldExtraData.Seed, - }, - elements: newHkeyElements(0), - } - - // Store new empty map with the same SlabID. - err := s.Store(id, newMap) - if err != nil { - return err - } - - // Remove all slabs and references in old map. - references, _, err := s.getAllChildReferences(old) - if err != nil { - return err - } - - for _, childID := range references { - err = s.Remove(childID) - if err != nil { - return err - } - } - - return nil -} - -func (s *PersistentSlabStorage) existIfLoaded(id SlabID) bool { - // Check deltas. - if slab, ok := s.deltas[id]; ok { - return slab != nil - } - - // Check read cache. - if slab, ok := s.cache[id]; ok { - return slab != nil - } - - return false -} - // GetAllChildReferences returns child references of given slab (all levels), // including nested container and theirs child references. func (s *PersistentSlabStorage) GetAllChildReferences(id SlabID) ( diff --git a/storage_test.go b/storage_test.go index 7403e0f2..efae0982 100644 --- a/storage_test.go +++ b/storage_test.go @@ -1416,1892 +1416,6 @@ func (s slowStorable) Encode(encoder *atree.Encoder) error { return s.Uint8Value.Encode(encoder) } -func TestFixLoadedBrokenReferences(t *testing.T) { - address := atree.Address{1, 2, 3, 4, 5, 6, 7, 8} - - t.Run("healthy", func(t *testing.T) { - - // Create a health storage with arrays and maps - mapMetaDataRootID := atree.NewSlabID(address, atree.SlabIndex{0, 0, 0, 0, 0, 0, 0, 1}) - mapDataNonRootID1 := atree.NewSlabID(address, atree.SlabIndex{0, 0, 0, 0, 0, 0, 0, 2}) - mapDataNonRootID2 := atree.NewSlabID(address, atree.SlabIndex{0, 0, 0, 0, 0, 0, 0, 3}) - nestedArrayID := atree.NewSlabID(address, atree.SlabIndex{0, 0, 0, 0, 0, 0, 0, 4}) - - emptyMapDataRootID := atree.NewSlabID(address, atree.SlabIndex{0, 0, 0, 0, 0, 0, 0, 5}) - - mapDataRootID := atree.NewSlabID(address, atree.SlabIndex{0, 0, 0, 0, 0, 0, 0, 6}) - - emptyArrayDataRootID := atree.NewSlabID(address, atree.SlabIndex{0, 0, 0, 0, 0, 0, 0, 7}) - - arrayDataRootID := atree.NewSlabID(address, atree.SlabIndex{0, 0, 0, 0, 0, 0, 0, 8}) - - arrayMetaDataRootID := atree.NewSlabID(address, atree.SlabIndex{0, 0, 0, 0, 0, 0, 0, 9}) - arrayDataNonRootID1 := atree.NewSlabID(address, atree.SlabIndex{0, 0, 0, 0, 0, 0, 0, 10}) - arrayDataNonRootID2 := atree.NewSlabID(address, atree.SlabIndex{0, 0, 0, 0, 0, 0, 0, 11}) - nestedArrayID2 := atree.NewSlabID(address, atree.SlabIndex{0, 0, 0, 0, 0, 0, 0, 12}) - - rootIDs := []atree.SlabID{ - mapMetaDataRootID, - emptyMapDataRootID, - mapDataRootID, - emptyArrayDataRootID, - arrayDataRootID, - arrayMetaDataRootID, - } - - data := map[atree.SlabID][]byte{ - // root map metadata slab - // metadata slab - mapMetaDataRootID: { - // extra data - // version - 0x00, - // flag: root + map meta - 0x89, - // extra data (CBOR encoded array of 3 elements) - 0x83, - // type info: "map" - 0x18, 0x2A, - // count: 8 - 0x08, - // seed - 0x1b, 0x52, 0xa8, 0x78, 0x3, 0x85, 0x2c, 0xaa, 0x49, - - // version - 0x00, - // flag: root + meta - 0x89, - // child header count - 0x00, 0x02, - // child header 1 (storage id, first key, size) - 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x01, 0x02, - // child header 2 - 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, - 0x00, 0x00, 0x00, 0xfe, - }, - - // data slab - mapDataNonRootID1: { - // version - 0x00, - // flag: map data - 0x08, - // next storage id - 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, - - // the following encoded data is valid CBOR - - // elements (array of 3 elements) - 0x83, - - // level: 0 - 0x00, - - // hkeys (byte string of length 8 * 4) - 0x5b, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x20, - // hkey: 0 - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - // hkey: 1 - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, - // hkey: 2 - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, - // hkey: 3 - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, - - // elements (array of 4 elements) - // each element is encoded as CBOR array of 2 elements (key, value) - 0x9b, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, - // element: [aaaaaaaaaaaaaaaaaaaaaa:aaaaaaaaaaaaaaaaaaaaaa] - 0x82, - 0x76, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, - 0x76, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, - // element: [bbbbbbbbbbbbbbbbbbbbbb:bbbbbbbbbbbbbbbbbbbbbb] - 0x82, - 0x76, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, - 0x76, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, - // element: [cccccccccccccccccccccc:cccccccccccccccccccccc] - 0x82, - 0x76, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, - 0x76, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, - // element: [dddddddddddddddddddddd:dddddddddddddddddddddd] - 0x82, - 0x76, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, - 0x76, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, - }, - - // data slab - mapDataNonRootID2: { - // version - 0x00, - // flag: has pointer + map data - 0x48, - // next storage id - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - - // the following encoded data is valid CBOR - - // elements (array of 3 elements) - 0x83, - - // level: 0 - 0x00, - - // hkeys (byte string of length 8 * 4) - 0x5b, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x20, - // hkey: 4 - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, - // hkey: 5 - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x05, - // hkey: 6 - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x06, - // hkey: 7 - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x07, - - // elements (array of 4 elements) - // each element is encoded as CBOR array of 2 elements (key, value) - 0x9b, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, - // element: [eeeeeeeeeeeeeeeeeeeeee:eeeeeeeeeeeeeeeeeeeeee] - 0x82, - 0x76, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, - 0x76, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, - // element: [ffffffffffffffffffffff:ffffffffffffffffffffff] - 0x82, - 0x76, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, - 0x76, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, - // element: [gggggggggggggggggggggg:gggggggggggggggggggggg] - 0x82, - 0x76, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, - 0x76, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, - // element: [hhhhhhhhhhhhhhhhhhhhhh:atree.SlabID(1,2,3,4,5,6,7,8,0,0,0,0,0,0,0,4)] - 0x82, - 0x76, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, - 0xd8, 0xff, 0x50, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, - }, - - // array data slab - nestedArrayID: { - // extra data - // version - 0x00, - // flag: root + array data - 0x80, - // extra data (CBOR encoded array of 1 elements) - 0x81, - // type info - 0x18, 0x2b, - - // version - 0x00, - // flag: root + array data - 0x80, - // CBOR encoded array head (fixed size 3 byte) - 0x99, 0x00, 0x01, - // CBOR encoded array elements - 0xd8, 0xa4, 0x00, - }, - - // empty map - emptyMapDataRootID: { - // extra data - // version - 0x00, - // flag: root + map data - 0x88, - // extra data (CBOR encoded array of 3 elements) - 0x83, - // type info - 0x18, 0x2a, - // count: 0 - 0x00, - // seed - 0x1b, 0x52, 0xa8, 0x78, 0x3, 0x85, 0x2c, 0xaa, 0x49, - - // version - 0x00, - // flag: root + map data - 0x88, - - // the following encoded data is valid CBOR - - // elements (array of 3 elements) - 0x83, - - // level: 0 - 0x00, - - // hkeys (byte string of length 8 * 1) - 0x5b, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - - // elements (array of 0 elements) - // each element is encoded as CBOR array of 2 elements (key, value) - 0x9b, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - }, - - // root map data slab - mapDataRootID: { - // extra data - // version - 0x00, - // flag: root + map data - 0x88, - // extra data (CBOR encoded array of 3 elements) - 0x83, - // type info - 0x18, 0x2a, - // count: 1 - 0x01, - // seed - 0x1b, 0x52, 0xa8, 0x78, 0x3, 0x85, 0x2c, 0xaa, 0x49, - - // version - 0x00, - // flag: root + map data - 0x88, - - // the following encoded data is valid CBOR - - // elements (array of 3 elements) - 0x83, - - // level: 0 - 0x00, - - // hkeys (byte string of length 8 * 1) - 0x5b, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x08, - // hkey: 0 - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - - // elements (array of 1 elements) - // each element is encoded as CBOR array of 2 elements (key, value) - 0x9b, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, - // element: [uint64(0):uint64(0)] - 0x82, 0xd8, 0xa4, 0x00, 0xd8, 0xa4, 0x00, - }, - - // empty array - emptyArrayDataRootID: { - // extra data - // version - 0x00, - // extra data flag - 0x80, - // array of extra data - 0x81, - // type info - 0x18, 0x2a, - - // version - 0x00, - // array data slab flag - 0x80, - // CBOR encoded array head (fixed size 3 byte) - 0x99, 0x00, 0x00, - }, - - // root array data slab - arrayDataRootID: { - // extra data - // version - 0x00, - // extra data flag - 0x80, - // array of extra data - 0x81, - // type info - 0x18, 0x2a, - - // version - 0x00, - // array data slab flag - 0x80, - // CBOR encoded array head (fixed size 3 byte) - 0x99, 0x00, 0x01, - // CBOR encoded array elements - 0xd8, 0xa4, 0x00, - }, - - // root array metadata slab - // (metadata slab) headers: [{id:2 size:228 count:9} {id:3 size:270 count:11} ] - arrayMetaDataRootID: { - // extra data - // version - 0x00, - // extra data flag - 0x81, - // array of extra data - 0x81, - // type info - 0x18, 0x2a, - - // version - 0x00, - // array meta data slab flag - 0x81, - // child header count - 0x00, 0x02, - // child header 1 (storage id, count, size) - 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0a, - 0x00, 0x00, 0x00, 0x09, - 0x00, 0x00, 0x00, 0xe4, - // child header 2 - 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0b, - 0x00, 0x00, 0x00, 0x0b, - 0x00, 0x00, 0x01, 0x0e, - }, - - // (data slab) next: 3, data: [aaaaaaaaaaaaaaaaaaaaaa ... aaaaaaaaaaaaaaaaaaaaaa] - arrayDataNonRootID1: { - // version - 0x00, - // array data slab flag - 0x00, - // next storage id - 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, - // CBOR encoded array head (fixed size 3 byte) - 0x99, 0x00, 0x09, - // CBOR encoded array elements - 0x76, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, - 0x76, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, - 0x76, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, - 0x76, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, - 0x76, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, - 0x76, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, - 0x76, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, - 0x76, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, - 0x76, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, - }, - - // (data slab) next: 0, data: [aaaaaaaaaaaaaaaaaaaaaa ... atree.SlabID(...)] - arrayDataNonRootID2: { - // version - 0x00, - // array data slab flag - 0x40, - // next storage id - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - // CBOR encoded array head (fixed size 3 byte) - 0x99, 0x00, 0x0b, - // CBOR encoded array elements - 0x76, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, - 0x76, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, - 0x76, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, - 0x76, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, - 0x76, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, - 0x76, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, - 0x76, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, - 0x76, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, - 0x76, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, - 0x76, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, - 0xd8, 0xff, 0x50, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0c, - }, - - // (data slab) next: 0, data: [0] - nestedArrayID2: { - // extra data - // version - 0x00, - // extra data flag - 0x80, - // array of extra data - 0x81, - // type info - 0x18, 0x2b, - - // version - 0x00, - // array data slab flag - 0x80, - // CBOR encoded array head (fixed size 3 byte) - 0x99, 0x00, 0x01, - // CBOR encoded array elements - 0xd8, 0xa4, 0x00, - }, - } - - storage := newTestPersistentStorageWithData(t, data) - - // Load data in storage - for id := range data { - _, found, err := storage.Retrieve(id) - require.NoError(t, err) - require.True(t, found) - } - - // Check health before fixing broken reference - rootIDSet, err := atree.CheckStorageHealth(storage, -1) - require.NoError(t, err) - require.Equal(t, len(rootIDs), len(rootIDSet)) - - for _, rootID := range rootIDs { - _, found := rootIDSet[rootID] - require.True(t, found) - } - - var fixedRootIDs map[atree.SlabID][]atree.SlabID - var skippedRootIDs map[atree.SlabID][]atree.SlabID - - // Don't fix any broken references - fixedRootIDs, skippedRootIDs, err = storage.FixLoadedBrokenReferences(func(_ atree.Value) bool { - return false - }) - require.NoError(t, err) - require.Equal(t, 0, len(fixedRootIDs)) - require.Equal(t, 0, len(skippedRootIDs)) - - // No data is modified because no fix happened - require.Equal(t, 0, GetDeltasCount(storage)) - - // Fix broken reference - fixedRootIDs, skippedRootIDs, err = storage.FixLoadedBrokenReferences(func(_ atree.Value) bool { - return true - }) - require.NoError(t, err) - require.Equal(t, 0, len(fixedRootIDs)) - require.Equal(t, 0, len(skippedRootIDs)) - - // No data is modified during fixing broken reference - require.Equal(t, 0, GetDeltasCount(storage)) - - // Check health after fixing broken reference - rootIDSet, err = atree.CheckStorageHealth(storage, -1) - require.NoError(t, err) - require.Equal(t, len(rootIDs), len(rootIDSet)) - - }) - - t.Run("broken root map data slab", func(t *testing.T) { - - rootID := atree.NewSlabID(address, atree.SlabIndex{0, 0, 0, 0, 0, 0, 0, 1}) - - brokenRefs := map[atree.SlabID][]atree.SlabID{ - rootID: {rootID}, - } - - data := map[atree.SlabID][]byte{ - rootID: { - // extra data - // version - 0x00, - // flag: root + map data - 0x88, - // extra data (CBOR encoded array of 3 elements) - 0x83, - // type info - 0x18, 0x2a, - // count: 1 - 0x01, - // seed - 0x1b, 0x52, 0xa8, 0x78, 0x3, 0x85, 0x2c, 0xaa, 0x49, - - // version - 0x00, - // flag: root + map data - 0x88, - - // the following encoded data is valid CBOR - - // elements (array of 3 elements) - 0x83, - - // level: 0 - 0x00, - - // hkeys (byte string of length 8 * 1) - 0x5b, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x08, - // hkey: 0 - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - - // elements (array of 1 elements) - // each element is encoded as CBOR array of 2 elements (key, value) - 0x9b, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, - // element: [atree.SlabID(0x0.1):uint64(0)] - 0x82, - 0xd8, 0xff, 0x50, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, - 0xd8, 0xa4, 0x00, - }, - } - - fixedData := map[atree.SlabID][]byte{ - rootID: { - // version - 0x10, - // flag: root + map data - 0x88, - - // extra data - // CBOR encoded array of 3 elements - 0x83, - // type info - 0x18, 0x2a, - // count: 0 - 0x00, - // seed - 0x1b, 0x52, 0xa8, 0x78, 0x3, 0x85, 0x2c, 0xaa, 0x49, - - // the following encoded data is valid CBOR - - // elements (array of 3 elements) - 0x83, - - // level: 0 - 0x00, - - // hkeys (byte string of length 8 * 1) - 0x59, 0x00, 0x00, - - // elements (array of 0 elements) - // each element is encoded as CBOR array of 2 elements (key, value) - 0x99, 0x00, 0x00, - }, - } - - storage := newTestPersistentStorageWithData(t, data) - - // Load data in storage - for id := range data { - _, found, err := storage.Retrieve(id) - require.NoError(t, err) - require.True(t, found) - } - - // Check health before fixing broken reference - _, err := atree.CheckStorageHealth(storage, -1) - require.ErrorContains(t, err, "slab (0x0.1) not found: slab not found during slab iteration") - - var fixedRootIDs map[atree.SlabID][]atree.SlabID - var skippedRootIDs map[atree.SlabID][]atree.SlabID - - // Don't fix any broken references - fixedRootIDs, skippedRootIDs, err = storage.FixLoadedBrokenReferences(func(_ atree.Value) bool { - return false - }) - require.NoError(t, err) - require.Equal(t, 0, len(fixedRootIDs)) - require.Equal(t, len(brokenRefs), len(skippedRootIDs)) - - for rootID, slabIDsWithBrokenRef := range brokenRefs { - require.ElementsMatch(t, slabIDsWithBrokenRef, skippedRootIDs[rootID]) - } - - // No data is modified because no fix happened - require.Equal(t, 0, GetDeltasCount(storage)) - - // Fix broken references - fixedRootIDs, skippedRootIDs, err = storage.FixLoadedBrokenReferences(func(_ atree.Value) bool { - return true - }) - require.NoError(t, err) - require.Equal(t, len(brokenRefs), len(fixedRootIDs)) - - for rootID, slabIDsWithBrokenRef := range brokenRefs { - require.ElementsMatch(t, slabIDsWithBrokenRef, fixedRootIDs[rootID]) - } - - require.Equal(t, 0, len(skippedRootIDs)) - require.Equal(t, 1, GetDeltasCount(storage)) - - // Check health after fixing broken reference - rootIDs, err := atree.CheckStorageHealth(storage, -1) - require.NoError(t, err) - require.Equal(t, 1, len(rootIDs)) - - _, ok := rootIDs[rootID] - require.True(t, ok) - - // Save data in storage - err = storage.FastCommit(runtime.NumCPU()) - require.NoError(t, err) - require.Equal(t, 0, GetDeltasCount(storage)) - - // Check encoded data - baseStorage := atree.GetBaseStorage(storage).(*testutils.InMemBaseStorage) - require.Equal(t, 1, baseStorage.SegmentCounts()) - - savedData, found, err := baseStorage.Retrieve(rootID) - require.NoError(t, err) - require.True(t, found) - require.Equal(t, fixedData[rootID], savedData) - }) - - t.Run("broken nested storable in root map data slab", func(t *testing.T) { - - rootID := atree.NewSlabID(address, atree.SlabIndex{0, 0, 0, 0, 0, 0, 0, 1}) - - brokenRefs := map[atree.SlabID][]atree.SlabID{ - rootID: {rootID}, - } - - data := map[atree.SlabID][]byte{ - rootID: { - // extra data - // version - 0x00, - // flag: root + map data - 0x88, - // extra data (CBOR encoded array of 3 elements) - 0x83, - // type info - 0x18, 0x2a, - // count: 1 - 0x01, - // seed - 0x1b, 0x52, 0xa8, 0x78, 0x3, 0x85, 0x2c, 0xaa, 0x49, - - // version - 0x00, - // flag: root + map data - 0x88, - - // the following encoded data is valid CBOR - - // elements (array of 3 elements) - 0x83, - - // level: 0 - 0x00, - - // hkeys (byte string of length 8 * 1) - 0x5b, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x08, - // hkey: 0 - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - - // elements (array of 1 elements) - // each element is encoded as CBOR array of 2 elements (key, value) - 0x9b, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, - // element: [uint64(0):SomeValue(atree.SlabID(0x0.1))] - 0x82, - 0xd8, 0xa4, 0x00, - 0xd8, testutils.CBORTagSomeValue, 0xd8, 0xff, 0x50, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, - }, - } - - fixedData := map[atree.SlabID][]byte{ - rootID: { - // version - 0x10, - // flag: root + map data - 0x88, - - // extra data - // CBOR encoded array of 3 elements - 0x83, - // type info - 0x18, 0x2a, - // count: 0 - 0x00, - // seed - 0x1b, 0x52, 0xa8, 0x78, 0x3, 0x85, 0x2c, 0xaa, 0x49, - - // the following encoded data is valid CBOR - - // elements (array of 3 elements) - 0x83, - - // level: 0 - 0x00, - - // hkeys (byte string of length 8 * 1) - 0x59, 0x00, 0x00, - - // elements (array of 0 elements) - // each element is encoded as CBOR array of 2 elements (key, value) - 0x99, 0x00, 0x00, - }, - } - - storage := newTestPersistentStorageWithData(t, data) - - // Load data in storage - for id := range data { - _, found, err := storage.Retrieve(id) - require.NoError(t, err) - require.True(t, found) - } - - // Check health before fixing broken reference - _, err := atree.CheckStorageHealth(storage, -1) - require.ErrorContains(t, err, "slab (0x0.1) not found: slab not found during slab iteration") - - var fixedRootIDs map[atree.SlabID][]atree.SlabID - var skippedRootIDs map[atree.SlabID][]atree.SlabID - - // Don't fix any broken references - fixedRootIDs, skippedRootIDs, err = storage.FixLoadedBrokenReferences(func(_ atree.Value) bool { - return false - }) - require.NoError(t, err) - require.Equal(t, 0, len(fixedRootIDs)) - require.Equal(t, len(brokenRefs), len(skippedRootIDs)) - - for rootID, slabIDsWithBrokenRef := range brokenRefs { - require.ElementsMatch(t, slabIDsWithBrokenRef, skippedRootIDs[rootID]) - } - - // No data is modified because no fix happened - require.Equal(t, 0, GetDeltasCount(storage)) - - // Fix broken references - fixedRootIDs, skippedRootIDs, err = storage.FixLoadedBrokenReferences(func(_ atree.Value) bool { - return true - }) - require.NoError(t, err) - require.Equal(t, len(brokenRefs), len(fixedRootIDs)) - - for rootID, slabIDsWithBrokenRef := range brokenRefs { - require.ElementsMatch(t, slabIDsWithBrokenRef, fixedRootIDs[rootID]) - } - - require.Equal(t, 0, len(skippedRootIDs)) - require.Equal(t, 1, GetDeltasCount(storage)) - - // Check health after fixing broken reference - rootIDs, err := atree.CheckStorageHealth(storage, -1) - require.NoError(t, err) - require.Equal(t, 1, len(rootIDs)) - - _, ok := rootIDs[rootID] - require.True(t, ok) - - // Save data in storage - err = storage.FastCommit(runtime.NumCPU()) - require.NoError(t, err) - require.Equal(t, 0, GetDeltasCount(storage)) - - // Check encoded data - baseStorage := atree.GetBaseStorage(storage).(*testutils.InMemBaseStorage) - require.Equal(t, 1, baseStorage.SegmentCounts()) - - savedData, found, err := baseStorage.Retrieve(rootID) - require.NoError(t, err) - require.True(t, found) - require.Equal(t, fixedData[rootID], savedData) - }) - - t.Run("broken non-root map data slab", func(t *testing.T) { - rootID := atree.NewSlabID(address, atree.SlabIndex{0, 0, 0, 0, 0, 0, 0, 1}) - nonRootDataID1 := atree.NewSlabID(address, atree.SlabIndex{0, 0, 0, 0, 0, 0, 0, 2}) - nonRootDataID2 := atree.NewSlabID(address, atree.SlabIndex{0, 0, 0, 0, 0, 0, 0, 3}) - - brokenRefs := map[atree.SlabID][]atree.SlabID{ - rootID: {nonRootDataID2}, - } - - // Expected serialized slab data with storage id - data := map[atree.SlabID][]byte{ - - // metadata slab - rootID: { - // extra data - // version - 0x00, - // flag: root + map meta - 0x89, - // extra data (CBOR encoded array of 3 elements) - 0x83, - // type info: "map" - 0x18, 0x2A, - // count: 8 - 0x08, - // seed - 0x1b, 0x52, 0xa8, 0x78, 0x3, 0x85, 0x2c, 0xaa, 0x49, - - // version - 0x00, - // flag: root + meta - 0x89, - // child header count - 0x00, 0x02, - // child header 1 (storage id, first key, size) - 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x01, 0x02, - // child header 2 - 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, - 0x00, 0x00, 0x00, 0xfe, - }, - - // data slab - nonRootDataID1: { - // version - 0x00, - // flag: map data - 0x08, - // next storage id - 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, - - // the following encoded data is valid CBOR - - // elements (array of 3 elements) - 0x83, - - // level: 0 - 0x00, - - // hkeys (byte string of length 8 * 4) - 0x5b, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x20, - // hkey: 0 - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - // hkey: 1 - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, - // hkey: 2 - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, - // hkey: 3 - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, - - // elements (array of 4 elements) - // each element is encoded as CBOR array of 2 elements (key, value) - 0x9b, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, - // element: [aaaaaaaaaaaaaaaaaaaaaa:aaaaaaaaaaaaaaaaaaaaaa] - 0x82, - 0x76, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, - 0x76, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, - // element: [bbbbbbbbbbbbbbbbbbbbbb:bbbbbbbbbbbbbbbbbbbbbb] - 0x82, - 0x76, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, - 0x76, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, - // element: [cccccccccccccccccccccc:cccccccccccccccccccccc] - 0x82, - 0x76, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, - 0x76, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, - // element: [dddddddddddddddddddddd:dddddddddddddddddddddd] - 0x82, - 0x76, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, - 0x76, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, - }, - - // data slab - nonRootDataID2: { - // version - 0x00, - // flag: has pointer + map data - 0x48, - // next storage id - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - - // the following encoded data is valid CBOR - - // elements (array of 3 elements) - 0x83, - - // level: 0 - 0x00, - - // hkeys (byte string of length 8 * 4) - 0x5b, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x20, - // hkey: 4 - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, - // hkey: 5 - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x05, - // hkey: 6 - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x06, - // hkey: 7 - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x07, - - // elements (array of 4 elements) - // each element is encoded as CBOR array of 2 elements (key, value) - 0x9b, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, - // element: [eeeeeeeeeeeeeeeeeeeeee:eeeeeeeeeeeeeeeeeeeeee] - 0x82, - 0x76, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, - 0x76, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, - // element: [ffffffffffffffffffffff:ffffffffffffffffffffff] - 0x82, - 0x76, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, - 0x76, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, - // element: [gggggggggggggggggggggg:gggggggggggggggggggggg] - 0x82, - 0x76, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, - 0x76, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, - // element: [hhhhhhhhhhhhhhhhhhhhhh:atree.SlabID(0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1)] - 0x82, - 0x76, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, - 0xd8, 0xff, 0x50, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, - }, - } - - fixedData := map[atree.SlabID][]byte{ - rootID: { - // version - 0x10, - // flag: root + map data - 0x88, - - // extra data - // CBOR encoded array of 3 elements - 0x83, - // type info - 0x18, 0x2a, - // count: 0 - 0x00, - // seed - 0x1b, 0x52, 0xa8, 0x78, 0x3, 0x85, 0x2c, 0xaa, 0x49, - - // the following encoded data is valid CBOR - - // elements (array of 3 elements) - 0x83, - - // level: 0 - 0x00, - - // hkeys (byte string of length 8 * 1) - 0x59, 0x00, 0x00, - - // elements (array of 0 elements) - // each element is encoded as CBOR array of 2 elements (key, value) - 0x99, 0x00, 0x00, - }, - } - - storage := newTestPersistentStorageWithData(t, data) - - // Load data in storage - for id := range data { - _, found, err := storage.Retrieve(id) - require.NoError(t, err) - require.True(t, found) - } - - // Check health before fixing broken reference - _, err := atree.CheckStorageHealth(storage, -1) - require.ErrorContains(t, err, "slab (0x0.1) not found: slab not found during slab iteration") - - var fixedRootIDs map[atree.SlabID][]atree.SlabID - var skippedRootIDs map[atree.SlabID][]atree.SlabID - - // Don't fix any broken references - fixedRootIDs, skippedRootIDs, err = storage.FixLoadedBrokenReferences(func(_ atree.Value) bool { - return false - }) - require.NoError(t, err) - require.Equal(t, 0, len(fixedRootIDs)) - require.Equal(t, len(brokenRefs), len(skippedRootIDs)) - - for rootID, slabIDsWithBrokenRef := range brokenRefs { - require.ElementsMatch(t, slabIDsWithBrokenRef, skippedRootIDs[rootID]) - } - - // No data is modified because no fix happened - require.Equal(t, 0, GetDeltasCount(storage)) - - // Fix broken reference - fixedRootIDs, skippedRootIDs, err = storage.FixLoadedBrokenReferences(func(_ atree.Value) bool { - return true - }) - require.NoError(t, err) - require.Equal(t, 1, len(fixedRootIDs)) - - for rootID, slabIDsWithBrokenRef := range brokenRefs { - require.ElementsMatch(t, slabIDsWithBrokenRef, fixedRootIDs[rootID]) - } - - require.Equal(t, 0, len(skippedRootIDs)) - require.Equal(t, 3, GetDeltasCount(storage)) - - // Check health after fixing broken reference - rootIDs, err := atree.CheckStorageHealth(storage, -1) - require.NoError(t, err) - require.Equal(t, 1, len(rootIDs)) - - _, ok := rootIDs[rootID] - require.True(t, ok) - - // Save data in storage - err = storage.FastCommit(runtime.NumCPU()) - require.NoError(t, err) - require.Equal(t, 0, GetDeltasCount(storage)) - - // Check encoded data - baseStorage := atree.GetBaseStorage(storage).(*testutils.InMemBaseStorage) - require.Equal(t, 1, baseStorage.SegmentCounts()) - - savedData, found, err := baseStorage.Retrieve(rootID) - require.NoError(t, err) - require.True(t, found) - require.Equal(t, fixedData[rootID], savedData) - }) - - t.Run("multiple data slabs with broken reference in the same map", func(t *testing.T) { - rootID := atree.NewSlabID(address, atree.SlabIndex{0, 0, 0, 0, 0, 0, 0, 1}) - nonRootDataID1 := atree.NewSlabID(address, atree.SlabIndex{0, 0, 0, 0, 0, 0, 0, 2}) - nonRootDataID2 := atree.NewSlabID(address, atree.SlabIndex{0, 0, 0, 0, 0, 0, 0, 3}) - - brokenRefs := map[atree.SlabID][]atree.SlabID{ - rootID: {nonRootDataID1, nonRootDataID2}, - } - - data := map[atree.SlabID][]byte{ - - // metadata slab - rootID: { - // extra data - // version - 0x00, - // flag: root + map meta - 0x89, - // extra data (CBOR encoded array of 3 elements) - 0x83, - // type info: "map" - 0x18, 0x2A, - // count: 8 - 0x08, - // seed - 0x1b, 0x52, 0xa8, 0x78, 0x3, 0x85, 0x2c, 0xaa, 0x49, - - // version - 0x00, - // flag: root + meta - 0x89, - // child header count - 0x00, 0x02, - // child header 1 (storage id, first key, size) - 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x01, 0x02, - // child header 2 - 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, - 0x00, 0x00, 0x00, 0xfe, - }, - - // data slab - nonRootDataID1: { - // version - 0x00, - // flag: map data - 0x08, - // next storage id - 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, - - // the following encoded data is valid CBOR - - // elements (array of 3 elements) - 0x83, - - // level: 0 - 0x00, - - // hkeys (byte string of length 8 * 4) - 0x5b, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x20, - // hkey: 0 - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - // hkey: 1 - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, - // hkey: 2 - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, - // hkey: 3 - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, - - // elements (array of 4 elements) - // each element is encoded as CBOR array of 2 elements (key, value) - 0x9b, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, - // element: [aaaaaaaaaaaaaaaaaaaaaa:aaaaaaaaaaaaaaaaaaaaaa] - 0x82, - 0x76, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, - 0x76, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, - // element: [bbbbbbbbbbbbbbbbbbbbbb:bbbbbbbbbbbbbbbbbbbbbb] - 0x82, - 0x76, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, - 0x76, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, - // element: [cccccccccccccccccccccc:cccccccccccccccccccccc] - 0x82, - 0x76, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, - 0x76, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, - // element: [dddddddddddddddddddddd:atree.SlabID(0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1)] - 0x82, - 0x76, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, - 0xd8, 0xff, 0x50, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, - }, - - // data slab - nonRootDataID2: { - // version - 0x00, - // flag: has pointer + map data - 0x48, - // next storage id - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - - // the following encoded data is valid CBOR - - // elements (array of 3 elements) - 0x83, - - // level: 0 - 0x00, - - // hkeys (byte string of length 8 * 4) - 0x5b, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x20, - // hkey: 4 - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, - // hkey: 5 - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x05, - // hkey: 6 - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x06, - // hkey: 7 - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x07, - - // elements (array of 4 elements) - // each element is encoded as CBOR array of 2 elements (key, value) - 0x9b, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, - // element: [eeeeeeeeeeeeeeeeeeeeee:eeeeeeeeeeeeeeeeeeeeee] - 0x82, - 0x76, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, - 0x76, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, - // element: [ffffffffffffffffffffff:ffffffffffffffffffffff] - 0x82, - 0x76, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, - 0x76, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, - // element: [gggggggggggggggggggggg:gggggggggggggggggggggg] - 0x82, - 0x76, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, - 0x76, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, - // element: [hhhhhhhhhhhhhhhhhhhhhh:atree.SlabID(0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2)] - 0x82, - 0x76, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, - 0xd8, 0xff, 0x50, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, - }, - } - - fixedData := map[atree.SlabID][]byte{ - rootID: { - // version - 0x10, - // flag: root + map data - 0x88, - - // extra data - // CBOR encoded array of 3 elements - 0x83, - // type info - 0x18, 0x2a, - // count: 0 - 0x00, - // seed - 0x1b, 0x52, 0xa8, 0x78, 0x3, 0x85, 0x2c, 0xaa, 0x49, - - // the following encoded data is valid CBOR - - // elements (array of 3 elements) - 0x83, - - // level: 0 - 0x00, - - // hkeys (byte string of length 8 * 1) - 0x59, 0x00, 0x00, - - // elements (array of 0 elements) - // each element is encoded as CBOR array of 2 elements (key, value) - 0x99, 0x00, 0x00, - }, - } - - storage := newTestPersistentStorageWithData(t, data) - - // Load data in storage - for id := range data { - _, found, err := storage.Retrieve(id) - require.NoError(t, err) - require.True(t, found) - } - - // Check health before fixing broken reference - _, err := atree.CheckStorageHealth(storage, -1) - require.ErrorContains(t, err, "slab not found during slab iteration") - - var fixedRootIDs map[atree.SlabID][]atree.SlabID - var skippedRootIDs map[atree.SlabID][]atree.SlabID - - // Don't fix any broken references - fixedRootIDs, skippedRootIDs, err = storage.FixLoadedBrokenReferences(func(_ atree.Value) bool { - return false - }) - require.NoError(t, err) - require.Equal(t, 0, len(fixedRootIDs)) - require.Equal(t, len(brokenRefs), len(skippedRootIDs)) - - for rootID, slabIDsWithBrokenRef := range brokenRefs { - require.ElementsMatch(t, slabIDsWithBrokenRef, skippedRootIDs[rootID]) - } - - // No data is modified because no fix happened - require.Equal(t, 0, GetDeltasCount(storage)) - - // Fix broken reference - fixedRootIDs, skippedRootIDs, err = storage.FixLoadedBrokenReferences(func(_ atree.Value) bool { - return true - }) - require.NoError(t, err) - require.Equal(t, len(brokenRefs), len(fixedRootIDs)) - - for rootID, slabIDsWithBrokenRef := range brokenRefs { - require.ElementsMatch(t, slabIDsWithBrokenRef, fixedRootIDs[rootID]) - } - - require.Equal(t, 0, len(skippedRootIDs)) - require.Equal(t, 3, GetDeltasCount(storage)) - - // Check health after fixing broken reference - rootIDs, err := atree.CheckStorageHealth(storage, -1) - require.NoError(t, err) - require.Equal(t, 1, len(rootIDs)) - - _, ok := rootIDs[rootID] - require.True(t, ok) - - // Save data in storage - err = storage.FastCommit(runtime.NumCPU()) - require.NoError(t, err) - require.Equal(t, 0, GetDeltasCount(storage)) - - // Check encoded data - baseStorage := atree.GetBaseStorage(storage).(*testutils.InMemBaseStorage) - require.Equal(t, 1, baseStorage.SegmentCounts()) - - savedData, found, err := baseStorage.Retrieve(rootID) - require.NoError(t, err) - require.True(t, found) - require.Equal(t, fixedData[rootID], savedData) - }) - - t.Run("broken reference in nested container", func(t *testing.T) { - parentContainerRootID := atree.NewSlabID(address, atree.SlabIndex{0, 0, 0, 0, 0, 0, 0, 1}) - nonRootDataID1 := atree.NewSlabID(address, atree.SlabIndex{0, 0, 0, 0, 0, 0, 0, 2}) - nonRootDataID2 := atree.NewSlabID(address, atree.SlabIndex{0, 0, 0, 0, 0, 0, 0, 3}) - nestedContainerRootID := atree.NewSlabID(address, atree.SlabIndex{0, 0, 0, 0, 0, 0, 0, 4}) - - brokenRefs := map[atree.SlabID][]atree.SlabID{ - nestedContainerRootID: {nestedContainerRootID}, - } - - data := map[atree.SlabID][]byte{ - - // metadata slab - parentContainerRootID: { - // extra data - // version - 0x00, - // flag: root + map meta - 0x89, - // extra data (CBOR encoded array of 3 elements) - 0x83, - // type info: "map" - 0x18, 0x2A, - // count: 8 - 0x08, - // seed - 0x1b, 0x52, 0xa8, 0x78, 0x3, 0x85, 0x2c, 0xaa, 0x49, - - // version - 0x00, - // flag: root + meta - 0x89, - // child header count - 0x00, 0x02, - // child header 1 (storage id, first key, size) - 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x01, 0x02, - // child header 2 - 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, - 0x00, 0x00, 0x00, 0xfe, - }, - - // data slab - nonRootDataID1: { - // version - 0x00, - // flag: map data - 0x08, - // next storage id - 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, - - // the following encoded data is valid CBOR - - // elements (array of 3 elements) - 0x83, - - // level: 0 - 0x00, - - // hkeys (byte string of length 8 * 4) - 0x5b, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x20, - // hkey: 0 - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - // hkey: 1 - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, - // hkey: 2 - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, - // hkey: 3 - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, - - // elements (array of 4 elements) - // each element is encoded as CBOR array of 2 elements (key, value) - 0x9b, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, - // element: [aaaaaaaaaaaaaaaaaaaaaa:aaaaaaaaaaaaaaaaaaaaaa] - 0x82, - 0x76, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, - 0x76, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, - // element: [bbbbbbbbbbbbbbbbbbbbbb:bbbbbbbbbbbbbbbbbbbbbb] - 0x82, - 0x76, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, - 0x76, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, - // element: [cccccccccccccccccccccc:cccccccccccccccccccccc] - 0x82, - 0x76, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, - 0x76, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, - // element: [dddddddddddddddddddddd:dddddddddddddddddddddd] - 0x82, - 0x76, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, - 0x76, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, - }, - - // data slab - nonRootDataID2: { - // version - 0x00, - // flag: has pointer + map data - 0x48, - // next storage id - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - - // the following encoded data is valid CBOR - - // elements (array of 3 elements) - 0x83, - - // level: 0 - 0x00, - - // hkeys (byte string of length 8 * 4) - 0x5b, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x20, - // hkey: 4 - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, - // hkey: 5 - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x05, - // hkey: 6 - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x06, - // hkey: 7 - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x07, - - // elements (array of 4 elements) - // each element is encoded as CBOR array of 2 elements (key, value) - 0x9b, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, - // element: [eeeeeeeeeeeeeeeeeeeeee:eeeeeeeeeeeeeeeeeeeeee] - 0x82, - 0x76, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, - 0x76, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, - // element: [ffffffffffffffffffffff:ffffffffffffffffffffff] - 0x82, - 0x76, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, - 0x76, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, - // element: [gggggggggggggggggggggg:gggggggggggggggggggggg] - 0x82, - 0x76, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, - 0x76, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, - // element: [hhhhhhhhhhhhhhhhhhhhhh:atree.SlabID(1,2,3,4,5,6,7,8,0,0,0,0,0,0,0,4)] - 0x82, - 0x76, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, - 0xd8, 0xff, 0x50, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, - }, - - // map data slab - nestedContainerRootID: { - // extra data - // version - 0x00, - // flag: root + map data - 0x88, - // extra data (CBOR encoded array of 3 elements) - 0x83, - // type info - 0x18, 0x2a, - // count: 1 - 0x01, - // seed - 0x1b, 0x52, 0xa8, 0x78, 0x3, 0x85, 0x2c, 0xaa, 0x49, - - // version - 0x00, - // flag: root + map data - 0x88, - - // the following encoded data is valid CBOR - - // elements (array of 3 elements) - 0x83, - - // level: 0 - 0x00, - - // hkeys (byte string of length 8 * 1) - 0x5b, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x08, - // hkey: 0 - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - - // elements (array of 1 elements) - // each element is encoded as CBOR array of 2 elements (key, value) - 0x9b, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, - // element: [uint64(0):atree.SlabID(0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1)] - 0x82, - 0xd8, 0xa4, 0x00, - 0xd8, 0xff, 0x50, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, - }, - } - - fixedData := map[atree.SlabID][]byte{ - // map data slab - nestedContainerRootID: { - // version - 0x10, - // flag: root + map data - 0x88, - - // extra data - // CBOR encoded array of 3 elements - 0x83, - // type info - 0x18, 0x2a, - // count: 0 - 0x00, - // seed - 0x1b, 0x52, 0xa8, 0x78, 0x3, 0x85, 0x2c, 0xaa, 0x49, - - // the following encoded data is valid CBOR - - // elements (array of 3 elements) - 0x83, - - // level: 0 - 0x00, - - // hkeys (byte string of length 8 * 1) - 0x59, 0x00, 0x00, - - // elements (array of 0 elements) - // each element is encoded as CBOR array of 2 elements (key, value) - 0x99, 0x00, 0x00, - }, - } - - storage := newTestPersistentStorageWithData(t, data) - - // Load data in storage - for id := range data { - _, found, err := storage.Retrieve(id) - require.NoError(t, err) - require.True(t, found) - } - - // Check health before fixing broken reference - _, err := atree.CheckStorageHealth(storage, -1) - require.ErrorContains(t, err, "slab (0x0.1) not found: slab not found during slab iteration") - - var fixedRootIDs map[atree.SlabID][]atree.SlabID - var skippedRootIDs map[atree.SlabID][]atree.SlabID - - // Don't fix any broken references - fixedRootIDs, skippedRootIDs, err = storage.FixLoadedBrokenReferences(func(_ atree.Value) bool { - return false - }) - require.NoError(t, err) - require.Equal(t, 0, len(fixedRootIDs)) - require.Equal(t, len(brokenRefs), len(skippedRootIDs)) - - for rootID, slabIDsWithBrokenRef := range brokenRefs { - require.ElementsMatch(t, slabIDsWithBrokenRef, skippedRootIDs[rootID]) - } - - // No data is modified because no fix happened - require.Equal(t, 0, GetDeltasCount(storage)) - - // Fix broken reference - fixedRootIDs, skippedRootIDs, err = storage.FixLoadedBrokenReferences(func(_ atree.Value) bool { - return true - }) - require.NoError(t, err) - require.Equal(t, len(brokenRefs), len(fixedRootIDs)) - - for rootID, slabIDsWithBrokenRef := range brokenRefs { - require.ElementsMatch(t, slabIDsWithBrokenRef, fixedRootIDs[rootID]) - } - - require.Equal(t, 0, len(skippedRootIDs)) - require.Equal(t, 1, GetDeltasCount(storage)) - - // Check health after fixing broken reference - rootIDs, err := atree.CheckStorageHealth(storage, -1) - require.NoError(t, err) - require.Equal(t, 1, len(rootIDs)) - - _, ok := rootIDs[parentContainerRootID] - require.True(t, ok) - - // Save data in storage - err = storage.FastCommit(runtime.NumCPU()) - require.NoError(t, err) - require.Equal(t, 0, GetDeltasCount(storage)) - - // Check encoded data - baseStorage := atree.GetBaseStorage(storage).(*testutils.InMemBaseStorage) - require.Equal(t, 4, baseStorage.SegmentCounts()) - - savedData, found, err := baseStorage.Retrieve(nestedContainerRootID) - require.NoError(t, err) - require.True(t, found) - require.Equal(t, fixedData[nestedContainerRootID], savedData) - }) - - t.Run("selectively fix maps", func(t *testing.T) { - rootID1 := atree.NewSlabID(address, atree.SlabIndex{0, 0, 0, 0, 0, 0, 0, 1}) - nonRootDataID1 := atree.NewSlabID(address, atree.SlabIndex{0, 0, 0, 0, 0, 0, 0, 2}) - nonRootDataID2 := atree.NewSlabID(address, atree.SlabIndex{0, 0, 0, 0, 0, 0, 0, 3}) // containing broken ref - - rootID2 := atree.NewSlabID(address, atree.SlabIndex{0, 0, 0, 0, 0, 0, 0, 4}) // containing broken ref - - rootIDs := []atree.SlabID{rootID1, rootID2} - - brokenRefs := map[atree.SlabID][]atree.SlabID{ - rootID1: {nonRootDataID2}, - rootID2: {rootID2}, - } - - data := map[atree.SlabID][]byte{ - // metadata slab - rootID1: { - // extra data - // version - 0x00, - // flag: root + map meta - 0x89, - // extra data (CBOR encoded array of 3 elements) - 0x83, - // type info: "map" - 0x18, 0x2A, - // count: 8 - 0x08, - // seed - 0x1b, 0x52, 0xa8, 0x78, 0x3, 0x85, 0x2c, 0xaa, 0x49, - - // version - 0x00, - // flag: root + meta - 0x89, - // child header count - 0x00, 0x02, - // child header 1 (storage id, first key, size) - 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x01, 0x02, - // child header 2 - 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, - 0x00, 0x00, 0x00, 0xfe, - }, - - // data slab - nonRootDataID1: { - // version - 0x00, - // flag: map data - 0x08, - // next storage id - 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, - - // the following encoded data is valid CBOR - - // elements (array of 3 elements) - 0x83, - - // level: 0 - 0x00, - - // hkeys (byte string of length 8 * 4) - 0x5b, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x20, - // hkey: 0 - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - // hkey: 1 - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, - // hkey: 2 - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, - // hkey: 3 - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, - - // elements (array of 4 elements) - // each element is encoded as CBOR array of 2 elements (key, value) - 0x9b, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, - // element: [aaaaaaaaaaaaaaaaaaaaaa:aaaaaaaaaaaaaaaaaaaaaa] - 0x82, - 0x76, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, - 0x76, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, - // element: [bbbbbbbbbbbbbbbbbbbbbb:bbbbbbbbbbbbbbbbbbbbbb] - 0x82, - 0x76, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, - 0x76, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, - // element: [cccccccccccccccccccccc:cccccccccccccccccccccc] - 0x82, - 0x76, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, - 0x76, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, - // element: [dddddddddddddddddddddd:dddddddddddddddddddddd] - 0x82, - 0x76, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, - 0x76, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, - }, - - // data slab - nonRootDataID2: { - // version - 0x00, - // flag: has pointer + map data - 0x48, - // next storage id - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - - // the following encoded data is valid CBOR - - // elements (array of 3 elements) - 0x83, - - // level: 0 - 0x00, - - // hkeys (byte string of length 8 * 4) - 0x5b, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x20, - // hkey: 4 - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, - // hkey: 5 - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x05, - // hkey: 6 - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x06, - // hkey: 7 - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x07, - - // elements (array of 4 elements) - // each element is encoded as CBOR array of 2 elements (key, value) - 0x9b, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, - // element: [eeeeeeeeeeeeeeeeeeeeee:eeeeeeeeeeeeeeeeeeeeee] - 0x82, - 0x76, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, - 0x76, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, - // element: [ffffffffffffffffffffff:ffffffffffffffffffffff] - 0x82, - 0x76, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, - 0x76, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, - // element: [gggggggggggggggggggggg:gggggggggggggggggggggg] - 0x82, - 0x76, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, - 0x76, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, - // element: [hhhhhhhhhhhhhhhhhhhhhh:atree.SlabID(0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1)] - 0x82, - 0x76, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, - 0xd8, 0xff, 0x50, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, - }, - - // map data slab - rootID2: { - // extra data - // version - 0x00, - // flag: root + map data - 0x88, - // extra data (CBOR encoded array of 3 elements) - 0x83, - // type info - 0x18, 0x2a, - // count: 1 - 0x01, - // seed - 0x1b, 0x52, 0xa8, 0x78, 0x3, 0x85, 0x2c, 0xaa, 0x4a, - - // version - 0x00, - // flag: root + map data - 0x88, - - // the following encoded data is valid CBOR - - // elements (array of 3 elements) - 0x83, - - // level: 0 - 0x00, - - // hkeys (byte string of length 8 * 1) - 0x5b, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x08, - // hkey: 0 - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - - // elements (array of 1 elements) - // each element is encoded as CBOR array of 2 elements (key, value) - 0x9b, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, - // element: [uint64(0):atree.SlabID(0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1)] - 0x82, - 0xd8, 0xa4, 0x00, - 0xd8, 0xff, 0x50, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, - }, - } - - fixedData := map[atree.SlabID][]byte{ - rootID1: { - // version - 0x10, - // flag: root + map data - 0x88, - - // extra data - // CBOR encoded array of 3 elements - 0x83, - // type info - 0x18, 0x2a, - // count: 0 - 0x00, - // seed - 0x1b, 0x52, 0xa8, 0x78, 0x3, 0x85, 0x2c, 0xaa, 0x49, - - // the following encoded data is valid CBOR - - // elements (array of 3 elements) - 0x83, - - // level: 0 - 0x00, - - // hkeys (byte string of length 8 * 1) - 0x59, 0x00, 0x00, - - // elements (array of 0 elements) - // each element is encoded as CBOR array of 2 elements (key, value) - 0x99, 0x00, 0x00, - }, - - rootID2: { - // version - 0x10, - // flag: root + map data - 0x88, - - // extra data - // CBOR encoded array of 3 elements - 0x83, - // type info - 0x18, 0x2a, - // count: 0 - 0x00, - // seed - 0x1b, 0x52, 0xa8, 0x78, 0x3, 0x85, 0x2c, 0xaa, 0x4a, - - // the following encoded data is valid CBOR - - // elements (array of 3 elements) - 0x83, - - // level: 0 - 0x00, - - // hkeys (byte string of length 8 * 1) - 0x59, 0x00, 0x00, - - // elements (array of 0 elements) - // each element is encoded as CBOR array of 2 elements (key, value) - 0x99, 0x00, 0x00, - }, - } - - storage := newTestPersistentStorageWithData(t, data) - - // Load data in storage - for id := range data { - _, found, err := storage.Retrieve(id) - require.NoError(t, err) - require.True(t, found) - } - - // Check health before fixing broken reference - _, err := atree.CheckStorageHealth(storage, -1) - require.ErrorContains(t, err, "slab not found during slab iteration") - - var fixedRootIDs map[atree.SlabID][]atree.SlabID - var skippedRootIDs map[atree.SlabID][]atree.SlabID - - // Don't fix any broken references - fixedRootIDs, skippedRootIDs, err = storage.FixLoadedBrokenReferences(func(_ atree.Value) bool { - return false - }) - require.NoError(t, err) - require.Equal(t, 0, len(fixedRootIDs)) - require.Equal(t, len(brokenRefs), len(skippedRootIDs)) - - for rootID, slabIDsWithBrokenRef := range brokenRefs { - require.ElementsMatch(t, slabIDsWithBrokenRef, skippedRootIDs[rootID]) - } - - // No data is modified because no fix happened - require.Equal(t, 0, GetDeltasCount(storage)) - - // Only fix one map with broken reference - fixedRootIDs, skippedRootIDs, err = storage.FixLoadedBrokenReferences(func(v atree.Value) bool { - m, ok := v.(*atree.OrderedMap) - require.True(t, ok) - return rootID1 == m.SlabID() - }) - require.NoError(t, err) - require.Equal(t, 1, len(fixedRootIDs)) - require.Equal(t, brokenRefs[rootID1], fixedRootIDs[rootID1]) - require.Equal(t, 1, len(skippedRootIDs)) - require.Equal(t, brokenRefs[rootID2], skippedRootIDs[rootID2]) - require.Equal(t, 3, GetDeltasCount(storage)) - - // Save data in storage - err = storage.FastCommit(runtime.NumCPU()) - require.NoError(t, err) - require.Equal(t, 0, GetDeltasCount(storage)) - - // Check health after only fixing one map with broken reference - _, err = atree.CheckStorageHealth(storage, -1) - require.ErrorContains(t, err, "slab not found during slab iteration") - - // Fix remaining map with broken reference - fixedRootIDs, skippedRootIDs, err = storage.FixLoadedBrokenReferences(func(atree.Value) bool { - return true - }) - require.NoError(t, err) - require.Equal(t, 1, len(fixedRootIDs)) - require.Equal(t, brokenRefs[rootID2], fixedRootIDs[rootID2]) - require.Equal(t, 0, len(skippedRootIDs)) - require.Equal(t, 1, GetDeltasCount(storage)) - - // Check health after fixing remaining maps with broken reference - returnedRootIDs, err := atree.CheckStorageHealth(storage, -1) - require.NoError(t, err) - require.Equal(t, len(rootIDs), len(returnedRootIDs)) - - // Save data in storage - err = storage.FastCommit(runtime.NumCPU()) - require.NoError(t, err) - require.Equal(t, 0, GetDeltasCount(storage)) - - // Check encoded data - baseStorage := atree.GetBaseStorage(storage).(*testutils.InMemBaseStorage) - require.Equal(t, 2, baseStorage.SegmentCounts()) - - savedData, found, err := baseStorage.Retrieve(rootID1) - require.NoError(t, err) - require.True(t, found) - require.Equal(t, fixedData[rootID1], savedData) - - savedData, found, err = baseStorage.Retrieve(rootID2) - require.NoError(t, err) - require.True(t, found) - require.Equal(t, fixedData[rootID2], savedData) - }) -} - func TestGetAllChildReferencesFromArray(t *testing.T) { address := atree.Address{1, 2, 3, 4, 5, 6, 7, 8}