Skip to content

fix: delete cni statefile when unable to be parsed #3551

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
wants to merge 2 commits into
base: master
Choose a base branch
from
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 13 additions & 0 deletions network/manager.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ package network

import (
"context"
"encoding/json"
"net"
"sync"
"time"
Expand Down Expand Up @@ -192,13 +193,25 @@ func (nm *networkManager) restore(isRehydrationRequired bool) error {
// Read any persisted state.
err := nm.store.Read(storeKey, nm)
if err != nil {
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Does it make sense to log the contents invalid and all to help assist with troubleshooting?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I was looking into this and the store's file name and contents are not accessible-- currently leaning towards just adding a method to the interface to dump the contents of the store in string format.

var syntaxErr *json.SyntaxError
if err == store.ErrKeyNotFound {
logger.Info("network store key not found")
// Considered successful.
return nil
} else if err == store.ErrStoreEmpty {
logger.Info("network store empty")
return nil
} else if errors.As(err, &syntaxErr) {
// if null chars detected or failed to parse, state is unrecoverable; delete it
logger.Error("Failed to parse corrupted state, deleting", zap.Error(err))
contents, readErr := nm.store.Dump()
if readErr != nil {
logger.Error("Could not read corrupted state", zap.Error(readErr))
} else {
logger.Info("Logging state", zap.String("stateFile", contents))
}
nm.store.Remove()
return errors.Wrap(err, "failed to parse corrupted state")
} else {
logger.Error("Failed to restore state", zap.Error(err))
return err
Expand Down
63 changes: 42 additions & 21 deletions store/json.go
Original file line number Diff line number Diff line change
Expand Up @@ -70,30 +70,10 @@ func (kvs *jsonFileStore) Read(key string, value interface{}) error {

// Read contents from file if memory is not in sync.
if !kvs.inSync {
// Open and parse the file if it exists.
file, err := os.Open(kvs.fileName)
b, err := kvs.readBytes()
if err != nil {
if os.IsNotExist(err) {
return ErrKeyNotFound
}
return err
}
defer file.Close()

b, err := io.ReadAll(file)
if err != nil {
return err
}

if len(b) == 0 {
if kvs.logger != nil {
kvs.logger.Info("Unable to read empty file", zap.String("fileName", kvs.fileName))
} else {
log.Printf("Unable to read file %s, was empty", kvs.fileName)
}

return ErrStoreEmpty
}

// Decode to raw JSON messages.
if err := json.Unmarshal(b, &kvs.data); err != nil {
Expand Down Expand Up @@ -265,3 +245,44 @@ func (kvs *jsonFileStore) Remove() {
}
kvs.Mutex.Unlock()
}

func (kvs *jsonFileStore) Dump() (string, error) {
kvs.Mutex.Lock()
defer kvs.Mutex.Unlock()

b, err := kvs.readBytes()
if err != nil {
return "", err
}

return string(b), nil
}

func (kvs *jsonFileStore) readBytes() ([]byte, error) {
// Open and parse the file if it exists.
file, err := os.Open(kvs.fileName)
if err != nil {
if os.IsNotExist(err) {
return []byte{}, ErrKeyNotFound
}
return []byte{}, errors.Wrap(err, "could not open file")
}
defer file.Close()

b, err := io.ReadAll(file)
if err != nil {
return []byte{}, errors.Wrap(err, "could not read file")
}

if len(b) == 0 {
if kvs.logger != nil {
kvs.logger.Info("Unable to read empty file", zap.String("fileName", kvs.fileName))
} else {
log.Printf("Unable to read file %s, was empty", kvs.fileName)
}

return []byte{}, ErrStoreEmpty
}

return b, nil
}
21 changes: 19 additions & 2 deletions store/json_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -129,6 +129,14 @@ func TestKeyValuePairsAreWrittenAndReadCorrectly(t *testing.T) {
t.Fatalf("Failed to create KeyValueStore %v\n", err)
}

defer os.Remove(testFileName)

// Dump empty store.
_, err = kvs.Dump()
if err == nil {
t.Fatal("Expected store to be empty")
}

// Write a key value pair.
err = kvs.Write(testKey1, &writtenValue)
if err != nil {
Expand All @@ -153,8 +161,17 @@ func TestKeyValuePairsAreWrittenAndReadCorrectly(t *testing.T) {
testKey1, readValue, testKey1, writtenValue)
}

// Cleanup.
os.Remove(testFileName)
// Dump populated store.
val, err := kvs.Dump()
if err != nil {
t.Fatalf("Failed to dump store %v", err)
}
val = strings.ReplaceAll(val, " ", "")
val = strings.ReplaceAll(val, "\t", "")
val = strings.ReplaceAll(val, "\n", "")
if val != `{"key1":{"Field1":"test","Field2":42},"key2":{"Field1":"any","Field2":14}}` {
t.Errorf("Dumped data not expected: %v", val)
}
}

// test case for testing newjsonfilestore idempotent
Expand Down
4 changes: 4 additions & 0 deletions store/mockstore.go
Original file line number Diff line number Diff line change
Expand Up @@ -71,3 +71,7 @@ func (ms *mockStore) GetLockFileName() string {
}

func (ms *mockStore) Remove() {}

func (ms *mockStore) Dump() (string, error) {
return fmt.Sprintf("%+v", ms.data), nil
}
1 change: 1 addition & 0 deletions store/store.go
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ type KeyValueStore interface {
Unlock() error
GetModificationTime() (time.Time, error)
Remove()
Dump() (string, error)
}

var (
Expand Down
4 changes: 4 additions & 0 deletions testutils/store_mock.go
Original file line number Diff line number Diff line change
Expand Up @@ -54,3 +54,7 @@ func (mockst *KeyValueStoreMock) GetModificationTime() (time.Time, error) {
}

func (mockst *KeyValueStoreMock) Remove() {}

func (mockst *KeyValueStoreMock) Dump() (string, error) {
return "", nil
}
Loading