diff --git a/storage.go b/storage.go index 7684f457..69955944 100644 --- a/storage.go +++ b/storage.go @@ -20,7 +20,6 @@ package atree import ( "encoding/binary" - "errors" "fmt" "sort" "strings" @@ -1013,233 +1012,6 @@ func (s *PersistentSlabStorage) DeltasSizeWithoutTempAddresses() uint64 { return size } -// FixLoadedBrokenReferences traverses loaded slabs and fixes broken references in maps. -// A broken reference is a SlabID referencing a non-existent slab. -// To fix a map containing broken references, this function replaces broken map with -// empty map having the same SlabID and also removes all slabs in the old map. -// Limitations: -// - only fix broken references in map -// - only traverse loaded slabs in deltas and cache -// NOTE: The intended use case is to enable migration programs in onflow/flow-go to -// fix broken references. As of April 2024, only 10 registers in testnet (not mainnet) -// were found to have broken references and they seem to have resulted from a bug -// that was fixed 2 years ago by https://github.com/onflow/cadence/pull/1565. -func (s *PersistentSlabStorage) FixLoadedBrokenReferences(needToFix func(old Value) bool) ( - fixedSlabIDs map[SlabID][]SlabID, // key: root slab ID, value: slab IDs containing broken refs - skippedSlabIDs map[SlabID][]SlabID, // key: root slab ID, value: slab IDs containing broken refs - err error, -) { - - // parentOf is used to find root slab from non-root slab. - // Broken reference can be in non-root slab, and we need SlabID of root slab - // to replace broken map by creating an empty new map with same SlabID. - parentOf := make(map[SlabID]SlabID) - - getRootSlabID := func(id SlabID) SlabID { - for { - parentID, ok := parentOf[id] - if ok { - id = parentID - } else { - return id - } - } - } - - hasBrokenReferenceInSlab := func(id SlabID, slab Slab) bool { - if slab == nil { - return false - } - - switch slab.(type) { - case *ArrayMetaDataSlab, *MapMetaDataSlab: // metadata slabs - var foundBrokenRef bool - - for _, childStorable := range slab.ChildStorables() { - - if slabIDStorable, ok := childStorable.(SlabIDStorable); ok { - - childID := SlabID(slabIDStorable) - - // Track parent-child relationship of root slabs and non-root slabs. - parentOf[childID] = id - - if !s.existIfLoaded(childID) { - foundBrokenRef = true - } - - // Continue with remaining child storables to track parent-child relationship. - } - } - - return foundBrokenRef - - default: // data slabs - childStorables := slab.ChildStorables() - - for len(childStorables) > 0 { - - var nextChildStorables []Storable - - for _, childStorable := range childStorables { - - if slabIDStorable, ok := childStorable.(SlabIDStorable); ok { - - if !s.existIfLoaded(SlabID(slabIDStorable)) { - return true - } - - continue - } - - // Append child storables of this childStorable to - // handle nested SlabIDStorable, such as Cadence SomeValue. - nextChildStorables = append( - nextChildStorables, - childStorable.ChildStorables()..., - ) - } - - childStorables = nextChildStorables - } - - return false - } - } - - var brokenSlabIDs []SlabID - - // Iterate delta slabs. - for id, slab := range s.deltas { - if hasBrokenReferenceInSlab(id, slab) { - brokenSlabIDs = append(brokenSlabIDs, id) - } - } - - // Iterate cache slabs. - for id, slab := range s.cache { - if _, ok := s.deltas[id]; ok { - continue - } - if hasBrokenReferenceInSlab(id, slab) { - brokenSlabIDs = append(brokenSlabIDs, id) - } - } - - if len(brokenSlabIDs) == 0 { - return nil, nil, nil - } - - rootSlabIDsWithBrokenData := make(map[SlabID][]SlabID) - var errs []error - - // Find SlabIDs of root slab for slabs containing broken references. - for _, id := range brokenSlabIDs { - rootID := getRootSlabID(id) - if rootID == SlabIDUndefined { - errs = append(errs, fmt.Errorf("failed to get root slab id for slab %s", id)) - continue - } - rootSlabIDsWithBrokenData[rootID] = append(rootSlabIDsWithBrokenData[rootID], id) - } - - for rootSlabID, brokenSlabIDs := range rootSlabIDsWithBrokenData { - rootSlab := s.RetrieveIfLoaded(rootSlabID) - if rootSlab == nil { - errs = append(errs, fmt.Errorf("failed to retrieve loaded root slab %s", rootSlabID)) - continue - } - - switch rootSlab := rootSlab.(type) { - case MapSlab: - value, err := rootSlab.StoredValue(s) - if err != nil { - errs = append(errs, fmt.Errorf("failed to convert slab %s into value", rootSlab.SlabID())) - continue - } - - if needToFix(value) { - err := s.fixBrokenReferencesInMap(rootSlab) - if err != nil { - errs = append(errs, err) - continue - } - } else { - if skippedSlabIDs == nil { - skippedSlabIDs = make(map[SlabID][]SlabID) - } - skippedSlabIDs[rootSlabID] = brokenSlabIDs - } - - default: - // IMPORTANT: Only handle map slabs for now. DO NOT silently fix currently unknown problems. - errs = append(errs, fmt.Errorf("failed to fix broken references in non-map slab %s (%T)", rootSlab.SlabID(), rootSlab)) - } - } - - for id := range skippedSlabIDs { - delete(rootSlabIDsWithBrokenData, id) - } - - return rootSlabIDsWithBrokenData, skippedSlabIDs, errors.Join(errs...) -} - -// fixBrokenReferencesInMap replaces replaces broken map with empty map -// having the same SlabID and also removes all slabs in the old map. -func (s *PersistentSlabStorage) fixBrokenReferencesInMap(old MapSlab) error { - id := old.SlabID() - - oldExtraData := old.ExtraData() - - // Create an empty map with the same StorgeID, type, and seed as the old map. - newMap := &MapDataSlab{ - header: MapSlabHeader{ - slabID: id, - size: mapRootDataSlabPrefixSize + hkeyElementsPrefixSize, - }, - extraData: &MapExtraData{ - TypeInfo: oldExtraData.TypeInfo, - Seed: oldExtraData.Seed, - }, - elements: newHkeyElements(0), - } - - // Store new empty map with the same SlabID. - err := s.Store(id, newMap) - if err != nil { - return err - } - - // Remove all slabs and references in old map. - references, _, err := s.getAllChildReferences(old) - if err != nil { - return err - } - - for _, childID := range references { - err = s.Remove(childID) - if err != nil { - return err - } - } - - return nil -} - -func (s *PersistentSlabStorage) existIfLoaded(id SlabID) bool { - // Check deltas. - if slab, ok := s.deltas[id]; ok { - return slab != nil - } - - // Check read cache. - if slab, ok := s.cache[id]; ok { - return slab != nil - } - - return false -} - // GetAllChildReferences returns child references of given slab (all levels), // including nested container and theirs child references. func (s *PersistentSlabStorage) GetAllChildReferences(id SlabID) ( diff --git a/storage_test.go b/storage_test.go index 7403e0f2..efae0982 100644 --- a/storage_test.go +++ b/storage_test.go @@ -1416,1892 +1416,6 @@ func (s slowStorable) Encode(encoder *atree.Encoder) error { return s.Uint8Value.Encode(encoder) } -func TestFixLoadedBrokenReferences(t *testing.T) { - address := atree.Address{1, 2, 3, 4, 5, 6, 7, 8} - - t.Run("healthy", func(t *testing.T) { - - // Create a health storage with arrays and maps - mapMetaDataRootID := atree.NewSlabID(address, atree.SlabIndex{0, 0, 0, 0, 0, 0, 0, 1}) - mapDataNonRootID1 := atree.NewSlabID(address, atree.SlabIndex{0, 0, 0, 0, 0, 0, 0, 2}) - mapDataNonRootID2 := atree.NewSlabID(address, atree.SlabIndex{0, 0, 0, 0, 0, 0, 0, 3}) - nestedArrayID := atree.NewSlabID(address, atree.SlabIndex{0, 0, 0, 0, 0, 0, 0, 4}) - - emptyMapDataRootID := atree.NewSlabID(address, atree.SlabIndex{0, 0, 0, 0, 0, 0, 0, 5}) - - mapDataRootID := atree.NewSlabID(address, atree.SlabIndex{0, 0, 0, 0, 0, 0, 0, 6}) - - emptyArrayDataRootID := atree.NewSlabID(address, atree.SlabIndex{0, 0, 0, 0, 0, 0, 0, 7}) - - arrayDataRootID := atree.NewSlabID(address, atree.SlabIndex{0, 0, 0, 0, 0, 0, 0, 8}) - - arrayMetaDataRootID := atree.NewSlabID(address, atree.SlabIndex{0, 0, 0, 0, 0, 0, 0, 9}) - arrayDataNonRootID1 := atree.NewSlabID(address, atree.SlabIndex{0, 0, 0, 0, 0, 0, 0, 10}) - arrayDataNonRootID2 := atree.NewSlabID(address, atree.SlabIndex{0, 0, 0, 0, 0, 0, 0, 11}) - nestedArrayID2 := atree.NewSlabID(address, atree.SlabIndex{0, 0, 0, 0, 0, 0, 0, 12}) - - rootIDs := []atree.SlabID{ - mapMetaDataRootID, - emptyMapDataRootID, - mapDataRootID, - emptyArrayDataRootID, - arrayDataRootID, - arrayMetaDataRootID, - } - - data := map[atree.SlabID][]byte{ - // root map metadata slab - // metadata slab - mapMetaDataRootID: { - // extra data - // version - 0x00, - // flag: root + map meta - 0x89, - // extra data (CBOR encoded array of 3 elements) - 0x83, - // type info: "map" - 0x18, 0x2A, - // count: 8 - 0x08, - // seed - 0x1b, 0x52, 0xa8, 0x78, 0x3, 0x85, 0x2c, 0xaa, 0x49, - - // version - 0x00, - // flag: root + meta - 0x89, - // child header count - 0x00, 0x02, - // child header 1 (storage id, first key, size) - 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x01, 0x02, - // child header 2 - 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, - 0x00, 0x00, 0x00, 0xfe, - }, - - // data slab - mapDataNonRootID1: { - // version - 0x00, - // flag: map data - 0x08, - // next storage id - 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, - - // the following encoded data is valid CBOR - - // elements (array of 3 elements) - 0x83, - - // level: 0 - 0x00, - - // hkeys (byte string of length 8 * 4) - 0x5b, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x20, - // hkey: 0 - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - // hkey: 1 - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, - // hkey: 2 - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, - // hkey: 3 - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, - - // elements (array of 4 elements) - // each element is encoded as CBOR array of 2 elements (key, value) - 0x9b, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, - // element: [aaaaaaaaaaaaaaaaaaaaaa:aaaaaaaaaaaaaaaaaaaaaa] - 0x82, - 0x76, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, - 0x76, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, - // element: [bbbbbbbbbbbbbbbbbbbbbb:bbbbbbbbbbbbbbbbbbbbbb] - 0x82, - 0x76, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, - 0x76, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, - // element: [cccccccccccccccccccccc:cccccccccccccccccccccc] - 0x82, - 0x76, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, - 0x76, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, - // element: [dddddddddddddddddddddd:dddddddddddddddddddddd] - 0x82, - 0x76, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, - 0x76, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, - }, - - // data slab - mapDataNonRootID2: { - // version - 0x00, - // flag: has pointer + map data - 0x48, - // next storage id - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - - // the following encoded data is valid CBOR - - // elements (array of 3 elements) - 0x83, - - // level: 0 - 0x00, - - // hkeys (byte string of length 8 * 4) - 0x5b, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x20, - // hkey: 4 - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, - // hkey: 5 - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x05, - // hkey: 6 - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x06, - // hkey: 7 - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x07, - - // elements (array of 4 elements) - // each element is encoded as CBOR array of 2 elements (key, value) - 0x9b, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, - // element: [eeeeeeeeeeeeeeeeeeeeee:eeeeeeeeeeeeeeeeeeeeee] - 0x82, - 0x76, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, - 0x76, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, - // element: [ffffffffffffffffffffff:ffffffffffffffffffffff] - 0x82, - 0x76, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, - 0x76, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, - // element: [gggggggggggggggggggggg:gggggggggggggggggggggg] - 0x82, - 0x76, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, - 0x76, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, - // element: [hhhhhhhhhhhhhhhhhhhhhh:atree.SlabID(1,2,3,4,5,6,7,8,0,0,0,0,0,0,0,4)] - 0x82, - 0x76, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, - 0xd8, 0xff, 0x50, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, - }, - - // array data slab - nestedArrayID: { - // extra data - // version - 0x00, - // flag: root + array data - 0x80, - // extra data (CBOR encoded array of 1 elements) - 0x81, - // type info - 0x18, 0x2b, - - // version - 0x00, - // flag: root + array data - 0x80, - // CBOR encoded array head (fixed size 3 byte) - 0x99, 0x00, 0x01, - // CBOR encoded array elements - 0xd8, 0xa4, 0x00, - }, - - // empty map - emptyMapDataRootID: { - // extra data - // version - 0x00, - // flag: root + map data - 0x88, - // extra data (CBOR encoded array of 3 elements) - 0x83, - // type info - 0x18, 0x2a, - // count: 0 - 0x00, - // seed - 0x1b, 0x52, 0xa8, 0x78, 0x3, 0x85, 0x2c, 0xaa, 0x49, - - // version - 0x00, - // flag: root + map data - 0x88, - - // the following encoded data is valid CBOR - - // elements (array of 3 elements) - 0x83, - - // level: 0 - 0x00, - - // hkeys (byte string of length 8 * 1) - 0x5b, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - - // elements (array of 0 elements) - // each element is encoded as CBOR array of 2 elements (key, value) - 0x9b, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - }, - - // root map data slab - mapDataRootID: { - // extra data - // version - 0x00, - // flag: root + map data - 0x88, - // extra data (CBOR encoded array of 3 elements) - 0x83, - // type info - 0x18, 0x2a, - // count: 1 - 0x01, - // seed - 0x1b, 0x52, 0xa8, 0x78, 0x3, 0x85, 0x2c, 0xaa, 0x49, - - // version - 0x00, - // flag: root + map data - 0x88, - - // the following encoded data is valid CBOR - - // elements (array of 3 elements) - 0x83, - - // level: 0 - 0x00, - - // hkeys (byte string of length 8 * 1) - 0x5b, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x08, - // hkey: 0 - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - - // elements (array of 1 elements) - // each element is encoded as CBOR array of 2 elements (key, value) - 0x9b, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, - // element: [uint64(0):uint64(0)] - 0x82, 0xd8, 0xa4, 0x00, 0xd8, 0xa4, 0x00, - }, - - // empty array - emptyArrayDataRootID: { - // extra data - // version - 0x00, - // extra data flag - 0x80, - // array of extra data - 0x81, - // type info - 0x18, 0x2a, - - // version - 0x00, - // array data slab flag - 0x80, - // CBOR encoded array head (fixed size 3 byte) - 0x99, 0x00, 0x00, - }, - - // root array data slab - arrayDataRootID: { - // extra data - // version - 0x00, - // extra data flag - 0x80, - // array of extra data - 0x81, - // type info - 0x18, 0x2a, - - // version - 0x00, - // array data slab flag - 0x80, - // CBOR encoded array head (fixed size 3 byte) - 0x99, 0x00, 0x01, - // CBOR encoded array elements - 0xd8, 0xa4, 0x00, - }, - - // root array metadata slab - // (metadata slab) headers: [{id:2 size:228 count:9} {id:3 size:270 count:11} ] - arrayMetaDataRootID: { - // extra data - // version - 0x00, - // extra data flag - 0x81, - // array of extra data - 0x81, - // type info - 0x18, 0x2a, - - // version - 0x00, - // array meta data slab flag - 0x81, - // child header count - 0x00, 0x02, - // child header 1 (storage id, count, size) - 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0a, - 0x00, 0x00, 0x00, 0x09, - 0x00, 0x00, 0x00, 0xe4, - // child header 2 - 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0b, - 0x00, 0x00, 0x00, 0x0b, - 0x00, 0x00, 0x01, 0x0e, - }, - - // (data slab) next: 3, data: [aaaaaaaaaaaaaaaaaaaaaa ... aaaaaaaaaaaaaaaaaaaaaa] - arrayDataNonRootID1: { - // version - 0x00, - // array data slab flag - 0x00, - // next storage id - 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, - // CBOR encoded array head (fixed size 3 byte) - 0x99, 0x00, 0x09, - // CBOR encoded array elements - 0x76, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, - 0x76, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, - 0x76, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, - 0x76, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, - 0x76, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, - 0x76, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, - 0x76, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, - 0x76, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, - 0x76, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, - }, - - // (data slab) next: 0, data: [aaaaaaaaaaaaaaaaaaaaaa ... atree.SlabID(...)] - arrayDataNonRootID2: { - // version - 0x00, - // array data slab flag - 0x40, - // next storage id - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - // CBOR encoded array head (fixed size 3 byte) - 0x99, 0x00, 0x0b, - // CBOR encoded array elements - 0x76, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, - 0x76, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, - 0x76, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, - 0x76, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, - 0x76, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, - 0x76, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, - 0x76, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, - 0x76, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, - 0x76, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, - 0x76, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, - 0xd8, 0xff, 0x50, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0c, - }, - - // (data slab) next: 0, data: [0] - nestedArrayID2: { - // extra data - // version - 0x00, - // extra data flag - 0x80, - // array of extra data - 0x81, - // type info - 0x18, 0x2b, - - // version - 0x00, - // array data slab flag - 0x80, - // CBOR encoded array head (fixed size 3 byte) - 0x99, 0x00, 0x01, - // CBOR encoded array elements - 0xd8, 0xa4, 0x00, - }, - } - - storage := newTestPersistentStorageWithData(t, data) - - // Load data in storage - for id := range data { - _, found, err := storage.Retrieve(id) - require.NoError(t, err) - require.True(t, found) - } - - // Check health before fixing broken reference - rootIDSet, err := atree.CheckStorageHealth(storage, -1) - require.NoError(t, err) - require.Equal(t, len(rootIDs), len(rootIDSet)) - - for _, rootID := range rootIDs { - _, found := rootIDSet[rootID] - require.True(t, found) - } - - var fixedRootIDs map[atree.SlabID][]atree.SlabID - var skippedRootIDs map[atree.SlabID][]atree.SlabID - - // Don't fix any broken references - fixedRootIDs, skippedRootIDs, err = storage.FixLoadedBrokenReferences(func(_ atree.Value) bool { - return false - }) - require.NoError(t, err) - require.Equal(t, 0, len(fixedRootIDs)) - require.Equal(t, 0, len(skippedRootIDs)) - - // No data is modified because no fix happened - require.Equal(t, 0, GetDeltasCount(storage)) - - // Fix broken reference - fixedRootIDs, skippedRootIDs, err = storage.FixLoadedBrokenReferences(func(_ atree.Value) bool { - return true - }) - require.NoError(t, err) - require.Equal(t, 0, len(fixedRootIDs)) - require.Equal(t, 0, len(skippedRootIDs)) - - // No data is modified during fixing broken reference - require.Equal(t, 0, GetDeltasCount(storage)) - - // Check health after fixing broken reference - rootIDSet, err = atree.CheckStorageHealth(storage, -1) - require.NoError(t, err) - require.Equal(t, len(rootIDs), len(rootIDSet)) - - }) - - t.Run("broken root map data slab", func(t *testing.T) { - - rootID := atree.NewSlabID(address, atree.SlabIndex{0, 0, 0, 0, 0, 0, 0, 1}) - - brokenRefs := map[atree.SlabID][]atree.SlabID{ - rootID: {rootID}, - } - - data := map[atree.SlabID][]byte{ - rootID: { - // extra data - // version - 0x00, - // flag: root + map data - 0x88, - // extra data (CBOR encoded array of 3 elements) - 0x83, - // type info - 0x18, 0x2a, - // count: 1 - 0x01, - // seed - 0x1b, 0x52, 0xa8, 0x78, 0x3, 0x85, 0x2c, 0xaa, 0x49, - - // version - 0x00, - // flag: root + map data - 0x88, - - // the following encoded data is valid CBOR - - // elements (array of 3 elements) - 0x83, - - // level: 0 - 0x00, - - // hkeys (byte string of length 8 * 1) - 0x5b, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x08, - // hkey: 0 - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - - // elements (array of 1 elements) - // each element is encoded as CBOR array of 2 elements (key, value) - 0x9b, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, - // element: [atree.SlabID(0x0.1):uint64(0)] - 0x82, - 0xd8, 0xff, 0x50, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, - 0xd8, 0xa4, 0x00, - }, - } - - fixedData := map[atree.SlabID][]byte{ - rootID: { - // version - 0x10, - // flag: root + map data - 0x88, - - // extra data - // CBOR encoded array of 3 elements - 0x83, - // type info - 0x18, 0x2a, - // count: 0 - 0x00, - // seed - 0x1b, 0x52, 0xa8, 0x78, 0x3, 0x85, 0x2c, 0xaa, 0x49, - - // the following encoded data is valid CBOR - - // elements (array of 3 elements) - 0x83, - - // level: 0 - 0x00, - - // hkeys (byte string of length 8 * 1) - 0x59, 0x00, 0x00, - - // elements (array of 0 elements) - // each element is encoded as CBOR array of 2 elements (key, value) - 0x99, 0x00, 0x00, - }, - } - - storage := newTestPersistentStorageWithData(t, data) - - // Load data in storage - for id := range data { - _, found, err := storage.Retrieve(id) - require.NoError(t, err) - require.True(t, found) - } - - // Check health before fixing broken reference - _, err := atree.CheckStorageHealth(storage, -1) - require.ErrorContains(t, err, "slab (0x0.1) not found: slab not found during slab iteration") - - var fixedRootIDs map[atree.SlabID][]atree.SlabID - var skippedRootIDs map[atree.SlabID][]atree.SlabID - - // Don't fix any broken references - fixedRootIDs, skippedRootIDs, err = storage.FixLoadedBrokenReferences(func(_ atree.Value) bool { - return false - }) - require.NoError(t, err) - require.Equal(t, 0, len(fixedRootIDs)) - require.Equal(t, len(brokenRefs), len(skippedRootIDs)) - - for rootID, slabIDsWithBrokenRef := range brokenRefs { - require.ElementsMatch(t, slabIDsWithBrokenRef, skippedRootIDs[rootID]) - } - - // No data is modified because no fix happened - require.Equal(t, 0, GetDeltasCount(storage)) - - // Fix broken references - fixedRootIDs, skippedRootIDs, err = storage.FixLoadedBrokenReferences(func(_ atree.Value) bool { - return true - }) - require.NoError(t, err) - require.Equal(t, len(brokenRefs), len(fixedRootIDs)) - - for rootID, slabIDsWithBrokenRef := range brokenRefs { - require.ElementsMatch(t, slabIDsWithBrokenRef, fixedRootIDs[rootID]) - } - - require.Equal(t, 0, len(skippedRootIDs)) - require.Equal(t, 1, GetDeltasCount(storage)) - - // Check health after fixing broken reference - rootIDs, err := atree.CheckStorageHealth(storage, -1) - require.NoError(t, err) - require.Equal(t, 1, len(rootIDs)) - - _, ok := rootIDs[rootID] - require.True(t, ok) - - // Save data in storage - err = storage.FastCommit(runtime.NumCPU()) - require.NoError(t, err) - require.Equal(t, 0, GetDeltasCount(storage)) - - // Check encoded data - baseStorage := atree.GetBaseStorage(storage).(*testutils.InMemBaseStorage) - require.Equal(t, 1, baseStorage.SegmentCounts()) - - savedData, found, err := baseStorage.Retrieve(rootID) - require.NoError(t, err) - require.True(t, found) - require.Equal(t, fixedData[rootID], savedData) - }) - - t.Run("broken nested storable in root map data slab", func(t *testing.T) { - - rootID := atree.NewSlabID(address, atree.SlabIndex{0, 0, 0, 0, 0, 0, 0, 1}) - - brokenRefs := map[atree.SlabID][]atree.SlabID{ - rootID: {rootID}, - } - - data := map[atree.SlabID][]byte{ - rootID: { - // extra data - // version - 0x00, - // flag: root + map data - 0x88, - // extra data (CBOR encoded array of 3 elements) - 0x83, - // type info - 0x18, 0x2a, - // count: 1 - 0x01, - // seed - 0x1b, 0x52, 0xa8, 0x78, 0x3, 0x85, 0x2c, 0xaa, 0x49, - - // version - 0x00, - // flag: root + map data - 0x88, - - // the following encoded data is valid CBOR - - // elements (array of 3 elements) - 0x83, - - // level: 0 - 0x00, - - // hkeys (byte string of length 8 * 1) - 0x5b, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x08, - // hkey: 0 - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - - // elements (array of 1 elements) - // each element is encoded as CBOR array of 2 elements (key, value) - 0x9b, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, - // element: [uint64(0):SomeValue(atree.SlabID(0x0.1))] - 0x82, - 0xd8, 0xa4, 0x00, - 0xd8, testutils.CBORTagSomeValue, 0xd8, 0xff, 0x50, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, - }, - } - - fixedData := map[atree.SlabID][]byte{ - rootID: { - // version - 0x10, - // flag: root + map data - 0x88, - - // extra data - // CBOR encoded array of 3 elements - 0x83, - // type info - 0x18, 0x2a, - // count: 0 - 0x00, - // seed - 0x1b, 0x52, 0xa8, 0x78, 0x3, 0x85, 0x2c, 0xaa, 0x49, - - // the following encoded data is valid CBOR - - // elements (array of 3 elements) - 0x83, - - // level: 0 - 0x00, - - // hkeys (byte string of length 8 * 1) - 0x59, 0x00, 0x00, - - // elements (array of 0 elements) - // each element is encoded as CBOR array of 2 elements (key, value) - 0x99, 0x00, 0x00, - }, - } - - storage := newTestPersistentStorageWithData(t, data) - - // Load data in storage - for id := range data { - _, found, err := storage.Retrieve(id) - require.NoError(t, err) - require.True(t, found) - } - - // Check health before fixing broken reference - _, err := atree.CheckStorageHealth(storage, -1) - require.ErrorContains(t, err, "slab (0x0.1) not found: slab not found during slab iteration") - - var fixedRootIDs map[atree.SlabID][]atree.SlabID - var skippedRootIDs map[atree.SlabID][]atree.SlabID - - // Don't fix any broken references - fixedRootIDs, skippedRootIDs, err = storage.FixLoadedBrokenReferences(func(_ atree.Value) bool { - return false - }) - require.NoError(t, err) - require.Equal(t, 0, len(fixedRootIDs)) - require.Equal(t, len(brokenRefs), len(skippedRootIDs)) - - for rootID, slabIDsWithBrokenRef := range brokenRefs { - require.ElementsMatch(t, slabIDsWithBrokenRef, skippedRootIDs[rootID]) - } - - // No data is modified because no fix happened - require.Equal(t, 0, GetDeltasCount(storage)) - - // Fix broken references - fixedRootIDs, skippedRootIDs, err = storage.FixLoadedBrokenReferences(func(_ atree.Value) bool { - return true - }) - require.NoError(t, err) - require.Equal(t, len(brokenRefs), len(fixedRootIDs)) - - for rootID, slabIDsWithBrokenRef := range brokenRefs { - require.ElementsMatch(t, slabIDsWithBrokenRef, fixedRootIDs[rootID]) - } - - require.Equal(t, 0, len(skippedRootIDs)) - require.Equal(t, 1, GetDeltasCount(storage)) - - // Check health after fixing broken reference - rootIDs, err := atree.CheckStorageHealth(storage, -1) - require.NoError(t, err) - require.Equal(t, 1, len(rootIDs)) - - _, ok := rootIDs[rootID] - require.True(t, ok) - - // Save data in storage - err = storage.FastCommit(runtime.NumCPU()) - require.NoError(t, err) - require.Equal(t, 0, GetDeltasCount(storage)) - - // Check encoded data - baseStorage := atree.GetBaseStorage(storage).(*testutils.InMemBaseStorage) - require.Equal(t, 1, baseStorage.SegmentCounts()) - - savedData, found, err := baseStorage.Retrieve(rootID) - require.NoError(t, err) - require.True(t, found) - require.Equal(t, fixedData[rootID], savedData) - }) - - t.Run("broken non-root map data slab", func(t *testing.T) { - rootID := atree.NewSlabID(address, atree.SlabIndex{0, 0, 0, 0, 0, 0, 0, 1}) - nonRootDataID1 := atree.NewSlabID(address, atree.SlabIndex{0, 0, 0, 0, 0, 0, 0, 2}) - nonRootDataID2 := atree.NewSlabID(address, atree.SlabIndex{0, 0, 0, 0, 0, 0, 0, 3}) - - brokenRefs := map[atree.SlabID][]atree.SlabID{ - rootID: {nonRootDataID2}, - } - - // Expected serialized slab data with storage id - data := map[atree.SlabID][]byte{ - - // metadata slab - rootID: { - // extra data - // version - 0x00, - // flag: root + map meta - 0x89, - // extra data (CBOR encoded array of 3 elements) - 0x83, - // type info: "map" - 0x18, 0x2A, - // count: 8 - 0x08, - // seed - 0x1b, 0x52, 0xa8, 0x78, 0x3, 0x85, 0x2c, 0xaa, 0x49, - - // version - 0x00, - // flag: root + meta - 0x89, - // child header count - 0x00, 0x02, - // child header 1 (storage id, first key, size) - 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x01, 0x02, - // child header 2 - 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, - 0x00, 0x00, 0x00, 0xfe, - }, - - // data slab - nonRootDataID1: { - // version - 0x00, - // flag: map data - 0x08, - // next storage id - 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, - - // the following encoded data is valid CBOR - - // elements (array of 3 elements) - 0x83, - - // level: 0 - 0x00, - - // hkeys (byte string of length 8 * 4) - 0x5b, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x20, - // hkey: 0 - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - // hkey: 1 - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, - // hkey: 2 - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, - // hkey: 3 - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, - - // elements (array of 4 elements) - // each element is encoded as CBOR array of 2 elements (key, value) - 0x9b, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, - // element: [aaaaaaaaaaaaaaaaaaaaaa:aaaaaaaaaaaaaaaaaaaaaa] - 0x82, - 0x76, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, - 0x76, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, - // element: [bbbbbbbbbbbbbbbbbbbbbb:bbbbbbbbbbbbbbbbbbbbbb] - 0x82, - 0x76, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, - 0x76, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, - // element: [cccccccccccccccccccccc:cccccccccccccccccccccc] - 0x82, - 0x76, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, - 0x76, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, - // element: [dddddddddddddddddddddd:dddddddddddddddddddddd] - 0x82, - 0x76, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, - 0x76, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, - }, - - // data slab - nonRootDataID2: { - // version - 0x00, - // flag: has pointer + map data - 0x48, - // next storage id - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - - // the following encoded data is valid CBOR - - // elements (array of 3 elements) - 0x83, - - // level: 0 - 0x00, - - // hkeys (byte string of length 8 * 4) - 0x5b, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x20, - // hkey: 4 - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, - // hkey: 5 - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x05, - // hkey: 6 - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x06, - // hkey: 7 - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x07, - - // elements (array of 4 elements) - // each element is encoded as CBOR array of 2 elements (key, value) - 0x9b, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, - // element: [eeeeeeeeeeeeeeeeeeeeee:eeeeeeeeeeeeeeeeeeeeee] - 0x82, - 0x76, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, - 0x76, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, - // element: [ffffffffffffffffffffff:ffffffffffffffffffffff] - 0x82, - 0x76, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, - 0x76, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, - // element: [gggggggggggggggggggggg:gggggggggggggggggggggg] - 0x82, - 0x76, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, - 0x76, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, - // element: [hhhhhhhhhhhhhhhhhhhhhh:atree.SlabID(0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1)] - 0x82, - 0x76, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, - 0xd8, 0xff, 0x50, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, - }, - } - - fixedData := map[atree.SlabID][]byte{ - rootID: { - // version - 0x10, - // flag: root + map data - 0x88, - - // extra data - // CBOR encoded array of 3 elements - 0x83, - // type info - 0x18, 0x2a, - // count: 0 - 0x00, - // seed - 0x1b, 0x52, 0xa8, 0x78, 0x3, 0x85, 0x2c, 0xaa, 0x49, - - // the following encoded data is valid CBOR - - // elements (array of 3 elements) - 0x83, - - // level: 0 - 0x00, - - // hkeys (byte string of length 8 * 1) - 0x59, 0x00, 0x00, - - // elements (array of 0 elements) - // each element is encoded as CBOR array of 2 elements (key, value) - 0x99, 0x00, 0x00, - }, - } - - storage := newTestPersistentStorageWithData(t, data) - - // Load data in storage - for id := range data { - _, found, err := storage.Retrieve(id) - require.NoError(t, err) - require.True(t, found) - } - - // Check health before fixing broken reference - _, err := atree.CheckStorageHealth(storage, -1) - require.ErrorContains(t, err, "slab (0x0.1) not found: slab not found during slab iteration") - - var fixedRootIDs map[atree.SlabID][]atree.SlabID - var skippedRootIDs map[atree.SlabID][]atree.SlabID - - // Don't fix any broken references - fixedRootIDs, skippedRootIDs, err = storage.FixLoadedBrokenReferences(func(_ atree.Value) bool { - return false - }) - require.NoError(t, err) - require.Equal(t, 0, len(fixedRootIDs)) - require.Equal(t, len(brokenRefs), len(skippedRootIDs)) - - for rootID, slabIDsWithBrokenRef := range brokenRefs { - require.ElementsMatch(t, slabIDsWithBrokenRef, skippedRootIDs[rootID]) - } - - // No data is modified because no fix happened - require.Equal(t, 0, GetDeltasCount(storage)) - - // Fix broken reference - fixedRootIDs, skippedRootIDs, err = storage.FixLoadedBrokenReferences(func(_ atree.Value) bool { - return true - }) - require.NoError(t, err) - require.Equal(t, 1, len(fixedRootIDs)) - - for rootID, slabIDsWithBrokenRef := range brokenRefs { - require.ElementsMatch(t, slabIDsWithBrokenRef, fixedRootIDs[rootID]) - } - - require.Equal(t, 0, len(skippedRootIDs)) - require.Equal(t, 3, GetDeltasCount(storage)) - - // Check health after fixing broken reference - rootIDs, err := atree.CheckStorageHealth(storage, -1) - require.NoError(t, err) - require.Equal(t, 1, len(rootIDs)) - - _, ok := rootIDs[rootID] - require.True(t, ok) - - // Save data in storage - err = storage.FastCommit(runtime.NumCPU()) - require.NoError(t, err) - require.Equal(t, 0, GetDeltasCount(storage)) - - // Check encoded data - baseStorage := atree.GetBaseStorage(storage).(*testutils.InMemBaseStorage) - require.Equal(t, 1, baseStorage.SegmentCounts()) - - savedData, found, err := baseStorage.Retrieve(rootID) - require.NoError(t, err) - require.True(t, found) - require.Equal(t, fixedData[rootID], savedData) - }) - - t.Run("multiple data slabs with broken reference in the same map", func(t *testing.T) { - rootID := atree.NewSlabID(address, atree.SlabIndex{0, 0, 0, 0, 0, 0, 0, 1}) - nonRootDataID1 := atree.NewSlabID(address, atree.SlabIndex{0, 0, 0, 0, 0, 0, 0, 2}) - nonRootDataID2 := atree.NewSlabID(address, atree.SlabIndex{0, 0, 0, 0, 0, 0, 0, 3}) - - brokenRefs := map[atree.SlabID][]atree.SlabID{ - rootID: {nonRootDataID1, nonRootDataID2}, - } - - data := map[atree.SlabID][]byte{ - - // metadata slab - rootID: { - // extra data - // version - 0x00, - // flag: root + map meta - 0x89, - // extra data (CBOR encoded array of 3 elements) - 0x83, - // type info: "map" - 0x18, 0x2A, - // count: 8 - 0x08, - // seed - 0x1b, 0x52, 0xa8, 0x78, 0x3, 0x85, 0x2c, 0xaa, 0x49, - - // version - 0x00, - // flag: root + meta - 0x89, - // child header count - 0x00, 0x02, - // child header 1 (storage id, first key, size) - 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x01, 0x02, - // child header 2 - 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, - 0x00, 0x00, 0x00, 0xfe, - }, - - // data slab - nonRootDataID1: { - // version - 0x00, - // flag: map data - 0x08, - // next storage id - 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, - - // the following encoded data is valid CBOR - - // elements (array of 3 elements) - 0x83, - - // level: 0 - 0x00, - - // hkeys (byte string of length 8 * 4) - 0x5b, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x20, - // hkey: 0 - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - // hkey: 1 - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, - // hkey: 2 - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, - // hkey: 3 - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, - - // elements (array of 4 elements) - // each element is encoded as CBOR array of 2 elements (key, value) - 0x9b, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, - // element: [aaaaaaaaaaaaaaaaaaaaaa:aaaaaaaaaaaaaaaaaaaaaa] - 0x82, - 0x76, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, - 0x76, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, - // element: [bbbbbbbbbbbbbbbbbbbbbb:bbbbbbbbbbbbbbbbbbbbbb] - 0x82, - 0x76, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, - 0x76, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, - // element: [cccccccccccccccccccccc:cccccccccccccccccccccc] - 0x82, - 0x76, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, - 0x76, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, - // element: [dddddddddddddddddddddd:atree.SlabID(0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1)] - 0x82, - 0x76, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, - 0xd8, 0xff, 0x50, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, - }, - - // data slab - nonRootDataID2: { - // version - 0x00, - // flag: has pointer + map data - 0x48, - // next storage id - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - - // the following encoded data is valid CBOR - - // elements (array of 3 elements) - 0x83, - - // level: 0 - 0x00, - - // hkeys (byte string of length 8 * 4) - 0x5b, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x20, - // hkey: 4 - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, - // hkey: 5 - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x05, - // hkey: 6 - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x06, - // hkey: 7 - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x07, - - // elements (array of 4 elements) - // each element is encoded as CBOR array of 2 elements (key, value) - 0x9b, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, - // element: [eeeeeeeeeeeeeeeeeeeeee:eeeeeeeeeeeeeeeeeeeeee] - 0x82, - 0x76, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, - 0x76, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, - // element: [ffffffffffffffffffffff:ffffffffffffffffffffff] - 0x82, - 0x76, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, - 0x76, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, - // element: [gggggggggggggggggggggg:gggggggggggggggggggggg] - 0x82, - 0x76, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, - 0x76, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, - // element: [hhhhhhhhhhhhhhhhhhhhhh:atree.SlabID(0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2)] - 0x82, - 0x76, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, - 0xd8, 0xff, 0x50, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, - }, - } - - fixedData := map[atree.SlabID][]byte{ - rootID: { - // version - 0x10, - // flag: root + map data - 0x88, - - // extra data - // CBOR encoded array of 3 elements - 0x83, - // type info - 0x18, 0x2a, - // count: 0 - 0x00, - // seed - 0x1b, 0x52, 0xa8, 0x78, 0x3, 0x85, 0x2c, 0xaa, 0x49, - - // the following encoded data is valid CBOR - - // elements (array of 3 elements) - 0x83, - - // level: 0 - 0x00, - - // hkeys (byte string of length 8 * 1) - 0x59, 0x00, 0x00, - - // elements (array of 0 elements) - // each element is encoded as CBOR array of 2 elements (key, value) - 0x99, 0x00, 0x00, - }, - } - - storage := newTestPersistentStorageWithData(t, data) - - // Load data in storage - for id := range data { - _, found, err := storage.Retrieve(id) - require.NoError(t, err) - require.True(t, found) - } - - // Check health before fixing broken reference - _, err := atree.CheckStorageHealth(storage, -1) - require.ErrorContains(t, err, "slab not found during slab iteration") - - var fixedRootIDs map[atree.SlabID][]atree.SlabID - var skippedRootIDs map[atree.SlabID][]atree.SlabID - - // Don't fix any broken references - fixedRootIDs, skippedRootIDs, err = storage.FixLoadedBrokenReferences(func(_ atree.Value) bool { - return false - }) - require.NoError(t, err) - require.Equal(t, 0, len(fixedRootIDs)) - require.Equal(t, len(brokenRefs), len(skippedRootIDs)) - - for rootID, slabIDsWithBrokenRef := range brokenRefs { - require.ElementsMatch(t, slabIDsWithBrokenRef, skippedRootIDs[rootID]) - } - - // No data is modified because no fix happened - require.Equal(t, 0, GetDeltasCount(storage)) - - // Fix broken reference - fixedRootIDs, skippedRootIDs, err = storage.FixLoadedBrokenReferences(func(_ atree.Value) bool { - return true - }) - require.NoError(t, err) - require.Equal(t, len(brokenRefs), len(fixedRootIDs)) - - for rootID, slabIDsWithBrokenRef := range brokenRefs { - require.ElementsMatch(t, slabIDsWithBrokenRef, fixedRootIDs[rootID]) - } - - require.Equal(t, 0, len(skippedRootIDs)) - require.Equal(t, 3, GetDeltasCount(storage)) - - // Check health after fixing broken reference - rootIDs, err := atree.CheckStorageHealth(storage, -1) - require.NoError(t, err) - require.Equal(t, 1, len(rootIDs)) - - _, ok := rootIDs[rootID] - require.True(t, ok) - - // Save data in storage - err = storage.FastCommit(runtime.NumCPU()) - require.NoError(t, err) - require.Equal(t, 0, GetDeltasCount(storage)) - - // Check encoded data - baseStorage := atree.GetBaseStorage(storage).(*testutils.InMemBaseStorage) - require.Equal(t, 1, baseStorage.SegmentCounts()) - - savedData, found, err := baseStorage.Retrieve(rootID) - require.NoError(t, err) - require.True(t, found) - require.Equal(t, fixedData[rootID], savedData) - }) - - t.Run("broken reference in nested container", func(t *testing.T) { - parentContainerRootID := atree.NewSlabID(address, atree.SlabIndex{0, 0, 0, 0, 0, 0, 0, 1}) - nonRootDataID1 := atree.NewSlabID(address, atree.SlabIndex{0, 0, 0, 0, 0, 0, 0, 2}) - nonRootDataID2 := atree.NewSlabID(address, atree.SlabIndex{0, 0, 0, 0, 0, 0, 0, 3}) - nestedContainerRootID := atree.NewSlabID(address, atree.SlabIndex{0, 0, 0, 0, 0, 0, 0, 4}) - - brokenRefs := map[atree.SlabID][]atree.SlabID{ - nestedContainerRootID: {nestedContainerRootID}, - } - - data := map[atree.SlabID][]byte{ - - // metadata slab - parentContainerRootID: { - // extra data - // version - 0x00, - // flag: root + map meta - 0x89, - // extra data (CBOR encoded array of 3 elements) - 0x83, - // type info: "map" - 0x18, 0x2A, - // count: 8 - 0x08, - // seed - 0x1b, 0x52, 0xa8, 0x78, 0x3, 0x85, 0x2c, 0xaa, 0x49, - - // version - 0x00, - // flag: root + meta - 0x89, - // child header count - 0x00, 0x02, - // child header 1 (storage id, first key, size) - 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x01, 0x02, - // child header 2 - 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, - 0x00, 0x00, 0x00, 0xfe, - }, - - // data slab - nonRootDataID1: { - // version - 0x00, - // flag: map data - 0x08, - // next storage id - 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, - - // the following encoded data is valid CBOR - - // elements (array of 3 elements) - 0x83, - - // level: 0 - 0x00, - - // hkeys (byte string of length 8 * 4) - 0x5b, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x20, - // hkey: 0 - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - // hkey: 1 - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, - // hkey: 2 - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, - // hkey: 3 - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, - - // elements (array of 4 elements) - // each element is encoded as CBOR array of 2 elements (key, value) - 0x9b, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, - // element: [aaaaaaaaaaaaaaaaaaaaaa:aaaaaaaaaaaaaaaaaaaaaa] - 0x82, - 0x76, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, - 0x76, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, - // element: [bbbbbbbbbbbbbbbbbbbbbb:bbbbbbbbbbbbbbbbbbbbbb] - 0x82, - 0x76, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, - 0x76, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, - // element: [cccccccccccccccccccccc:cccccccccccccccccccccc] - 0x82, - 0x76, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, - 0x76, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, - // element: [dddddddddddddddddddddd:dddddddddddddddddddddd] - 0x82, - 0x76, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, - 0x76, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, - }, - - // data slab - nonRootDataID2: { - // version - 0x00, - // flag: has pointer + map data - 0x48, - // next storage id - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - - // the following encoded data is valid CBOR - - // elements (array of 3 elements) - 0x83, - - // level: 0 - 0x00, - - // hkeys (byte string of length 8 * 4) - 0x5b, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x20, - // hkey: 4 - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, - // hkey: 5 - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x05, - // hkey: 6 - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x06, - // hkey: 7 - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x07, - - // elements (array of 4 elements) - // each element is encoded as CBOR array of 2 elements (key, value) - 0x9b, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, - // element: [eeeeeeeeeeeeeeeeeeeeee:eeeeeeeeeeeeeeeeeeeeee] - 0x82, - 0x76, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, - 0x76, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, - // element: [ffffffffffffffffffffff:ffffffffffffffffffffff] - 0x82, - 0x76, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, - 0x76, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, - // element: [gggggggggggggggggggggg:gggggggggggggggggggggg] - 0x82, - 0x76, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, - 0x76, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, - // element: [hhhhhhhhhhhhhhhhhhhhhh:atree.SlabID(1,2,3,4,5,6,7,8,0,0,0,0,0,0,0,4)] - 0x82, - 0x76, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, - 0xd8, 0xff, 0x50, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, - }, - - // map data slab - nestedContainerRootID: { - // extra data - // version - 0x00, - // flag: root + map data - 0x88, - // extra data (CBOR encoded array of 3 elements) - 0x83, - // type info - 0x18, 0x2a, - // count: 1 - 0x01, - // seed - 0x1b, 0x52, 0xa8, 0x78, 0x3, 0x85, 0x2c, 0xaa, 0x49, - - // version - 0x00, - // flag: root + map data - 0x88, - - // the following encoded data is valid CBOR - - // elements (array of 3 elements) - 0x83, - - // level: 0 - 0x00, - - // hkeys (byte string of length 8 * 1) - 0x5b, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x08, - // hkey: 0 - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - - // elements (array of 1 elements) - // each element is encoded as CBOR array of 2 elements (key, value) - 0x9b, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, - // element: [uint64(0):atree.SlabID(0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1)] - 0x82, - 0xd8, 0xa4, 0x00, - 0xd8, 0xff, 0x50, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, - }, - } - - fixedData := map[atree.SlabID][]byte{ - // map data slab - nestedContainerRootID: { - // version - 0x10, - // flag: root + map data - 0x88, - - // extra data - // CBOR encoded array of 3 elements - 0x83, - // type info - 0x18, 0x2a, - // count: 0 - 0x00, - // seed - 0x1b, 0x52, 0xa8, 0x78, 0x3, 0x85, 0x2c, 0xaa, 0x49, - - // the following encoded data is valid CBOR - - // elements (array of 3 elements) - 0x83, - - // level: 0 - 0x00, - - // hkeys (byte string of length 8 * 1) - 0x59, 0x00, 0x00, - - // elements (array of 0 elements) - // each element is encoded as CBOR array of 2 elements (key, value) - 0x99, 0x00, 0x00, - }, - } - - storage := newTestPersistentStorageWithData(t, data) - - // Load data in storage - for id := range data { - _, found, err := storage.Retrieve(id) - require.NoError(t, err) - require.True(t, found) - } - - // Check health before fixing broken reference - _, err := atree.CheckStorageHealth(storage, -1) - require.ErrorContains(t, err, "slab (0x0.1) not found: slab not found during slab iteration") - - var fixedRootIDs map[atree.SlabID][]atree.SlabID - var skippedRootIDs map[atree.SlabID][]atree.SlabID - - // Don't fix any broken references - fixedRootIDs, skippedRootIDs, err = storage.FixLoadedBrokenReferences(func(_ atree.Value) bool { - return false - }) - require.NoError(t, err) - require.Equal(t, 0, len(fixedRootIDs)) - require.Equal(t, len(brokenRefs), len(skippedRootIDs)) - - for rootID, slabIDsWithBrokenRef := range brokenRefs { - require.ElementsMatch(t, slabIDsWithBrokenRef, skippedRootIDs[rootID]) - } - - // No data is modified because no fix happened - require.Equal(t, 0, GetDeltasCount(storage)) - - // Fix broken reference - fixedRootIDs, skippedRootIDs, err = storage.FixLoadedBrokenReferences(func(_ atree.Value) bool { - return true - }) - require.NoError(t, err) - require.Equal(t, len(brokenRefs), len(fixedRootIDs)) - - for rootID, slabIDsWithBrokenRef := range brokenRefs { - require.ElementsMatch(t, slabIDsWithBrokenRef, fixedRootIDs[rootID]) - } - - require.Equal(t, 0, len(skippedRootIDs)) - require.Equal(t, 1, GetDeltasCount(storage)) - - // Check health after fixing broken reference - rootIDs, err := atree.CheckStorageHealth(storage, -1) - require.NoError(t, err) - require.Equal(t, 1, len(rootIDs)) - - _, ok := rootIDs[parentContainerRootID] - require.True(t, ok) - - // Save data in storage - err = storage.FastCommit(runtime.NumCPU()) - require.NoError(t, err) - require.Equal(t, 0, GetDeltasCount(storage)) - - // Check encoded data - baseStorage := atree.GetBaseStorage(storage).(*testutils.InMemBaseStorage) - require.Equal(t, 4, baseStorage.SegmentCounts()) - - savedData, found, err := baseStorage.Retrieve(nestedContainerRootID) - require.NoError(t, err) - require.True(t, found) - require.Equal(t, fixedData[nestedContainerRootID], savedData) - }) - - t.Run("selectively fix maps", func(t *testing.T) { - rootID1 := atree.NewSlabID(address, atree.SlabIndex{0, 0, 0, 0, 0, 0, 0, 1}) - nonRootDataID1 := atree.NewSlabID(address, atree.SlabIndex{0, 0, 0, 0, 0, 0, 0, 2}) - nonRootDataID2 := atree.NewSlabID(address, atree.SlabIndex{0, 0, 0, 0, 0, 0, 0, 3}) // containing broken ref - - rootID2 := atree.NewSlabID(address, atree.SlabIndex{0, 0, 0, 0, 0, 0, 0, 4}) // containing broken ref - - rootIDs := []atree.SlabID{rootID1, rootID2} - - brokenRefs := map[atree.SlabID][]atree.SlabID{ - rootID1: {nonRootDataID2}, - rootID2: {rootID2}, - } - - data := map[atree.SlabID][]byte{ - // metadata slab - rootID1: { - // extra data - // version - 0x00, - // flag: root + map meta - 0x89, - // extra data (CBOR encoded array of 3 elements) - 0x83, - // type info: "map" - 0x18, 0x2A, - // count: 8 - 0x08, - // seed - 0x1b, 0x52, 0xa8, 0x78, 0x3, 0x85, 0x2c, 0xaa, 0x49, - - // version - 0x00, - // flag: root + meta - 0x89, - // child header count - 0x00, 0x02, - // child header 1 (storage id, first key, size) - 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x01, 0x02, - // child header 2 - 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, - 0x00, 0x00, 0x00, 0xfe, - }, - - // data slab - nonRootDataID1: { - // version - 0x00, - // flag: map data - 0x08, - // next storage id - 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, - - // the following encoded data is valid CBOR - - // elements (array of 3 elements) - 0x83, - - // level: 0 - 0x00, - - // hkeys (byte string of length 8 * 4) - 0x5b, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x20, - // hkey: 0 - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - // hkey: 1 - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, - // hkey: 2 - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, - // hkey: 3 - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, - - // elements (array of 4 elements) - // each element is encoded as CBOR array of 2 elements (key, value) - 0x9b, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, - // element: [aaaaaaaaaaaaaaaaaaaaaa:aaaaaaaaaaaaaaaaaaaaaa] - 0x82, - 0x76, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, - 0x76, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, - // element: [bbbbbbbbbbbbbbbbbbbbbb:bbbbbbbbbbbbbbbbbbbbbb] - 0x82, - 0x76, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, - 0x76, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, - // element: [cccccccccccccccccccccc:cccccccccccccccccccccc] - 0x82, - 0x76, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, - 0x76, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, - // element: [dddddddddddddddddddddd:dddddddddddddddddddddd] - 0x82, - 0x76, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, - 0x76, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, - }, - - // data slab - nonRootDataID2: { - // version - 0x00, - // flag: has pointer + map data - 0x48, - // next storage id - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - - // the following encoded data is valid CBOR - - // elements (array of 3 elements) - 0x83, - - // level: 0 - 0x00, - - // hkeys (byte string of length 8 * 4) - 0x5b, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x20, - // hkey: 4 - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, - // hkey: 5 - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x05, - // hkey: 6 - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x06, - // hkey: 7 - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x07, - - // elements (array of 4 elements) - // each element is encoded as CBOR array of 2 elements (key, value) - 0x9b, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, - // element: [eeeeeeeeeeeeeeeeeeeeee:eeeeeeeeeeeeeeeeeeeeee] - 0x82, - 0x76, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, - 0x76, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, - // element: [ffffffffffffffffffffff:ffffffffffffffffffffff] - 0x82, - 0x76, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, - 0x76, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, - // element: [gggggggggggggggggggggg:gggggggggggggggggggggg] - 0x82, - 0x76, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, - 0x76, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, - // element: [hhhhhhhhhhhhhhhhhhhhhh:atree.SlabID(0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1)] - 0x82, - 0x76, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, - 0xd8, 0xff, 0x50, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, - }, - - // map data slab - rootID2: { - // extra data - // version - 0x00, - // flag: root + map data - 0x88, - // extra data (CBOR encoded array of 3 elements) - 0x83, - // type info - 0x18, 0x2a, - // count: 1 - 0x01, - // seed - 0x1b, 0x52, 0xa8, 0x78, 0x3, 0x85, 0x2c, 0xaa, 0x4a, - - // version - 0x00, - // flag: root + map data - 0x88, - - // the following encoded data is valid CBOR - - // elements (array of 3 elements) - 0x83, - - // level: 0 - 0x00, - - // hkeys (byte string of length 8 * 1) - 0x5b, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x08, - // hkey: 0 - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - - // elements (array of 1 elements) - // each element is encoded as CBOR array of 2 elements (key, value) - 0x9b, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, - // element: [uint64(0):atree.SlabID(0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1)] - 0x82, - 0xd8, 0xa4, 0x00, - 0xd8, 0xff, 0x50, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, - }, - } - - fixedData := map[atree.SlabID][]byte{ - rootID1: { - // version - 0x10, - // flag: root + map data - 0x88, - - // extra data - // CBOR encoded array of 3 elements - 0x83, - // type info - 0x18, 0x2a, - // count: 0 - 0x00, - // seed - 0x1b, 0x52, 0xa8, 0x78, 0x3, 0x85, 0x2c, 0xaa, 0x49, - - // the following encoded data is valid CBOR - - // elements (array of 3 elements) - 0x83, - - // level: 0 - 0x00, - - // hkeys (byte string of length 8 * 1) - 0x59, 0x00, 0x00, - - // elements (array of 0 elements) - // each element is encoded as CBOR array of 2 elements (key, value) - 0x99, 0x00, 0x00, - }, - - rootID2: { - // version - 0x10, - // flag: root + map data - 0x88, - - // extra data - // CBOR encoded array of 3 elements - 0x83, - // type info - 0x18, 0x2a, - // count: 0 - 0x00, - // seed - 0x1b, 0x52, 0xa8, 0x78, 0x3, 0x85, 0x2c, 0xaa, 0x4a, - - // the following encoded data is valid CBOR - - // elements (array of 3 elements) - 0x83, - - // level: 0 - 0x00, - - // hkeys (byte string of length 8 * 1) - 0x59, 0x00, 0x00, - - // elements (array of 0 elements) - // each element is encoded as CBOR array of 2 elements (key, value) - 0x99, 0x00, 0x00, - }, - } - - storage := newTestPersistentStorageWithData(t, data) - - // Load data in storage - for id := range data { - _, found, err := storage.Retrieve(id) - require.NoError(t, err) - require.True(t, found) - } - - // Check health before fixing broken reference - _, err := atree.CheckStorageHealth(storage, -1) - require.ErrorContains(t, err, "slab not found during slab iteration") - - var fixedRootIDs map[atree.SlabID][]atree.SlabID - var skippedRootIDs map[atree.SlabID][]atree.SlabID - - // Don't fix any broken references - fixedRootIDs, skippedRootIDs, err = storage.FixLoadedBrokenReferences(func(_ atree.Value) bool { - return false - }) - require.NoError(t, err) - require.Equal(t, 0, len(fixedRootIDs)) - require.Equal(t, len(brokenRefs), len(skippedRootIDs)) - - for rootID, slabIDsWithBrokenRef := range brokenRefs { - require.ElementsMatch(t, slabIDsWithBrokenRef, skippedRootIDs[rootID]) - } - - // No data is modified because no fix happened - require.Equal(t, 0, GetDeltasCount(storage)) - - // Only fix one map with broken reference - fixedRootIDs, skippedRootIDs, err = storage.FixLoadedBrokenReferences(func(v atree.Value) bool { - m, ok := v.(*atree.OrderedMap) - require.True(t, ok) - return rootID1 == m.SlabID() - }) - require.NoError(t, err) - require.Equal(t, 1, len(fixedRootIDs)) - require.Equal(t, brokenRefs[rootID1], fixedRootIDs[rootID1]) - require.Equal(t, 1, len(skippedRootIDs)) - require.Equal(t, brokenRefs[rootID2], skippedRootIDs[rootID2]) - require.Equal(t, 3, GetDeltasCount(storage)) - - // Save data in storage - err = storage.FastCommit(runtime.NumCPU()) - require.NoError(t, err) - require.Equal(t, 0, GetDeltasCount(storage)) - - // Check health after only fixing one map with broken reference - _, err = atree.CheckStorageHealth(storage, -1) - require.ErrorContains(t, err, "slab not found during slab iteration") - - // Fix remaining map with broken reference - fixedRootIDs, skippedRootIDs, err = storage.FixLoadedBrokenReferences(func(atree.Value) bool { - return true - }) - require.NoError(t, err) - require.Equal(t, 1, len(fixedRootIDs)) - require.Equal(t, brokenRefs[rootID2], fixedRootIDs[rootID2]) - require.Equal(t, 0, len(skippedRootIDs)) - require.Equal(t, 1, GetDeltasCount(storage)) - - // Check health after fixing remaining maps with broken reference - returnedRootIDs, err := atree.CheckStorageHealth(storage, -1) - require.NoError(t, err) - require.Equal(t, len(rootIDs), len(returnedRootIDs)) - - // Save data in storage - err = storage.FastCommit(runtime.NumCPU()) - require.NoError(t, err) - require.Equal(t, 0, GetDeltasCount(storage)) - - // Check encoded data - baseStorage := atree.GetBaseStorage(storage).(*testutils.InMemBaseStorage) - require.Equal(t, 2, baseStorage.SegmentCounts()) - - savedData, found, err := baseStorage.Retrieve(rootID1) - require.NoError(t, err) - require.True(t, found) - require.Equal(t, fixedData[rootID1], savedData) - - savedData, found, err = baseStorage.Retrieve(rootID2) - require.NoError(t, err) - require.True(t, found) - require.Equal(t, fixedData[rootID2], savedData) - }) -} - func TestGetAllChildReferencesFromArray(t *testing.T) { address := atree.Address{1, 2, 3, 4, 5, 6, 7, 8}