From 565a39b85c59d623b9abdaf79edf80f4f1a9b092 Mon Sep 17 00:00:00 2001 From: QxBytes Date: Wed, 19 Mar 2025 22:36:06 -0700 Subject: [PATCH 1/2] delete cni statefile when unable to be parsed --- network/manager.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/network/manager.go b/network/manager.go index e5d6d57330..dd2262c24c 100644 --- a/network/manager.go +++ b/network/manager.go @@ -5,6 +5,7 @@ package network import ( "context" + "encoding/json" "net" "sync" "time" @@ -192,6 +193,7 @@ func (nm *networkManager) restore(isRehydrationRequired bool) error { // Read any persisted state. err := nm.store.Read(storeKey, nm) if err != nil { + var syntaxErr *json.SyntaxError if err == store.ErrKeyNotFound { logger.Info("network store key not found") // Considered successful. @@ -199,6 +201,11 @@ func (nm *networkManager) restore(isRehydrationRequired bool) error { } else if err == store.ErrStoreEmpty { logger.Info("network store empty") return nil + } else if errors.As(err, &syntaxErr) { + // if null chars detected or failed to parse, state is unrecoverable; delete it + logger.Error("Failed to parse corrupted state, deleting", zap.Error(err)) + nm.store.Remove() + return errors.Wrap(err, "failed to parse corrupted state") } else { logger.Error("Failed to restore state", zap.Error(err)) return err From c94573b47b64238f6baf7a3a32f59495bb4bc4c5 Mon Sep 17 00:00:00 2001 From: QxBytes Date: Fri, 25 Apr 2025 16:22:38 -0700 Subject: [PATCH 2/2] dump store contents when corrupted file detected adds Dump to store interface which returns the raw contents of the store in string format if the store is empty or not readable it returns an error adds to a ut to check that dump effectively prints the contents of the store --- network/manager.go | 6 ++++ store/json.go | 63 +++++++++++++++++++++++++++-------------- store/json_test.go | 21 ++++++++++++-- store/mockstore.go | 4 +++ store/store.go | 1 + testutils/store_mock.go | 4 +++ 6 files changed, 76 insertions(+), 23 deletions(-) diff --git a/network/manager.go b/network/manager.go index dd2262c24c..72e4bc0a4f 100644 --- a/network/manager.go +++ b/network/manager.go @@ -204,6 +204,12 @@ func (nm *networkManager) restore(isRehydrationRequired bool) error { } else if errors.As(err, &syntaxErr) { // if null chars detected or failed to parse, state is unrecoverable; delete it logger.Error("Failed to parse corrupted state, deleting", zap.Error(err)) + contents, readErr := nm.store.Dump() + if readErr != nil { + logger.Error("Could not read corrupted state", zap.Error(readErr)) + } else { + logger.Info("Logging state", zap.String("stateFile", contents)) + } nm.store.Remove() return errors.Wrap(err, "failed to parse corrupted state") } else { diff --git a/store/json.go b/store/json.go index c26773d262..f7de6541d9 100644 --- a/store/json.go +++ b/store/json.go @@ -70,30 +70,10 @@ func (kvs *jsonFileStore) Read(key string, value interface{}) error { // Read contents from file if memory is not in sync. if !kvs.inSync { - // Open and parse the file if it exists. - file, err := os.Open(kvs.fileName) + b, err := kvs.readBytes() if err != nil { - if os.IsNotExist(err) { - return ErrKeyNotFound - } return err } - defer file.Close() - - b, err := io.ReadAll(file) - if err != nil { - return err - } - - if len(b) == 0 { - if kvs.logger != nil { - kvs.logger.Info("Unable to read empty file", zap.String("fileName", kvs.fileName)) - } else { - log.Printf("Unable to read file %s, was empty", kvs.fileName) - } - - return ErrStoreEmpty - } // Decode to raw JSON messages. if err := json.Unmarshal(b, &kvs.data); err != nil { @@ -265,3 +245,44 @@ func (kvs *jsonFileStore) Remove() { } kvs.Mutex.Unlock() } + +func (kvs *jsonFileStore) Dump() (string, error) { + kvs.Mutex.Lock() + defer kvs.Mutex.Unlock() + + b, err := kvs.readBytes() + if err != nil { + return "", err + } + + return string(b), nil +} + +func (kvs *jsonFileStore) readBytes() ([]byte, error) { + // Open and parse the file if it exists. + file, err := os.Open(kvs.fileName) + if err != nil { + if os.IsNotExist(err) { + return []byte{}, ErrKeyNotFound + } + return []byte{}, errors.Wrap(err, "could not open file") + } + defer file.Close() + + b, err := io.ReadAll(file) + if err != nil { + return []byte{}, errors.Wrap(err, "could not read file") + } + + if len(b) == 0 { + if kvs.logger != nil { + kvs.logger.Info("Unable to read empty file", zap.String("fileName", kvs.fileName)) + } else { + log.Printf("Unable to read file %s, was empty", kvs.fileName) + } + + return []byte{}, ErrStoreEmpty + } + + return b, nil +} diff --git a/store/json_test.go b/store/json_test.go index 83451fbaca..61760ef82e 100644 --- a/store/json_test.go +++ b/store/json_test.go @@ -129,6 +129,14 @@ func TestKeyValuePairsAreWrittenAndReadCorrectly(t *testing.T) { t.Fatalf("Failed to create KeyValueStore %v\n", err) } + defer os.Remove(testFileName) + + // Dump empty store. + _, err = kvs.Dump() + if err == nil { + t.Fatal("Expected store to be empty") + } + // Write a key value pair. err = kvs.Write(testKey1, &writtenValue) if err != nil { @@ -153,8 +161,17 @@ func TestKeyValuePairsAreWrittenAndReadCorrectly(t *testing.T) { testKey1, readValue, testKey1, writtenValue) } - // Cleanup. - os.Remove(testFileName) + // Dump populated store. + val, err := kvs.Dump() + if err != nil { + t.Fatalf("Failed to dump store %v", err) + } + val = strings.ReplaceAll(val, " ", "") + val = strings.ReplaceAll(val, "\t", "") + val = strings.ReplaceAll(val, "\n", "") + if val != `{"key1":{"Field1":"test","Field2":42},"key2":{"Field1":"any","Field2":14}}` { + t.Errorf("Dumped data not expected: %v", val) + } } // test case for testing newjsonfilestore idempotent diff --git a/store/mockstore.go b/store/mockstore.go index 7dda8b1938..7af27d7c16 100644 --- a/store/mockstore.go +++ b/store/mockstore.go @@ -71,3 +71,7 @@ func (ms *mockStore) GetLockFileName() string { } func (ms *mockStore) Remove() {} + +func (ms *mockStore) Dump() (string, error) { + return fmt.Sprintf("%+v", ms.data), nil +} diff --git a/store/store.go b/store/store.go index 73b9e3fc72..13db8b3218 100644 --- a/store/store.go +++ b/store/store.go @@ -18,6 +18,7 @@ type KeyValueStore interface { Unlock() error GetModificationTime() (time.Time, error) Remove() + Dump() (string, error) } var ( diff --git a/testutils/store_mock.go b/testutils/store_mock.go index cead32138c..ebd6d4ed87 100644 --- a/testutils/store_mock.go +++ b/testutils/store_mock.go @@ -54,3 +54,7 @@ func (mockst *KeyValueStoreMock) GetModificationTime() (time.Time, error) { } func (mockst *KeyValueStoreMock) Remove() {} + +func (mockst *KeyValueStoreMock) Dump() (string, error) { + return "", nil +}