feat: add collections key encoders and value encoders for common type…

…s. (cosmos#14760)

Co-authored-by: testinginprod <[email protected]>

codec/collections.go

@@ -0,0 +1,44 @@
+package codec
+
+import (
+	"cosmossdk.io/collections"
+	"github.com/cosmos/gogoproto/proto"
+)
+
+type protoMessage[T any] interface {
+	*T
+	proto.Message
+}
+
+// CollValue inits a collections.ValueCodec for a generic gogo protobuf message.
+func CollValue[T any, PT protoMessage[T]](cdc BinaryCodec) collections.ValueCodec[T] {
+	return &collValue[T, PT]{cdc.(Codec)}
+}
+
+type collValue[T any, PT protoMessage[T]] struct{ cdc Codec }
+
+func (c collValue[T, PT]) Encode(value T) ([]byte, error) {
+	return c.cdc.Marshal(PT(&value))
+}
+
+func (c collValue[T, PT]) Decode(b []byte) (value T, err error) {
+	err = c.cdc.Unmarshal(b, PT(&value))
+	return value, err
+}
+
+func (c collValue[T, PT]) EncodeJSON(value T) ([]byte, error) {
+	return c.cdc.MarshalJSON(PT(&value))
+}
+
+func (c collValue[T, PT]) DecodeJSON(b []byte) (value T, err error) {
+	err = c.cdc.UnmarshalJSON(b, PT(&value))
+	return
+}
+
+func (c collValue[T, PT]) Stringify(value T) string {
+	return PT(&value).String()
+}
+
+func (c collValue[T, PT]) ValueType() string {
+	return "gogoproto/" + proto.MessageName(PT(new(T)))
+}

codec/collections_test.go

@@ -0,0 +1,18 @@
+package codec
+
+import (
+	"testing"
+
+	"cosmossdk.io/collections/colltest"
+	codectypes "github.com/cosmos/cosmos-sdk/codec/types"
+	"github.com/cosmos/gogoproto/types"
+)
+
+func TestCollectionsCorrectness(t *testing.T) {
+	cdc := NewProtoCodec(codectypes.NewInterfaceRegistry())
+	t.Run("CollValue", func(t *testing.T) {
+		colltest.TestValueCodec(t, CollValue[types.UInt64Value](cdc), types.UInt64Value{
+			Value: 500,
+		})
+	})
+}

collections/collections_test.go

@@ -20,7 +20,6 @@ func (t testStore) OpenKVStore(ctx context.Context) store.KVStore {
 }
 
 func (t testStore) Get(key []byte) ([]byte, error) {
-
 	return t.db.Get(key)
 }
 
@@ -52,44 +51,6 @@ func deps() (store.KVStoreService, context.Context) {
 	return &testStore{kv}, context.Background()
 }
 
-// checkKeyCodec asserts the correct behaviour of a KeyCodec over the type T.
-func checkKeyCodec[T any](t *testing.T, keyCodec KeyCodec[T], key T) {
-	buffer := make([]byte, keyCodec.Size(key))
-	written, err := keyCodec.Encode(buffer, key)
-	require.NoError(t, err)
-	require.Equal(t, len(buffer), written)
-	read, decodedKey, err := keyCodec.Decode(buffer)
-	require.NoError(t, err)
-	require.Equal(t, len(buffer), read, "encoded key and read bytes must have same size")
-	require.Equal(t, key, decodedKey, "encoding and decoding produces different keys")
-	// test if terminality is correctly applied
-	pairCodec := PairKeyCodec(keyCodec, StringKey)
-	pairKey := Join(key, "TEST")
-	buffer = make([]byte, pairCodec.Size(pairKey))
-	written, err = pairCodec.Encode(buffer, pairKey)
-	require.NoError(t, err)
-	read, decodedPairKey, err := pairCodec.Decode(buffer)
-	require.NoError(t, err)
-	require.Equal(t, len(buffer), read, "encoded non terminal key and pair key read bytes must have same size")
-	require.Equal(t, pairKey, decodedPairKey, "encoding and decoding produces different keys with non terminal encoding")
-
-	// check JSON
-	keyJSON, err := keyCodec.EncodeJSON(key)
-	require.NoError(t, err)
-	decoded, err := keyCodec.DecodeJSON(keyJSON)
-	require.NoError(t, err)
-	require.Equal(t, key, decoded, "json encoding and decoding did not produce the same results")
-}
-
-// checkValueCodec asserts the correct behaviour of a ValueCodec over the type T.
-func checkValueCodec[T any](t *testing.T, encoder ValueCodec[T], value T) {
-	encodedValue, err := encoder.Encode(value)
-	require.NoError(t, err)
-	decodedValue, err := encoder.Decode(encodedValue)
-	require.NoError(t, err)
-	require.Equal(t, value, decodedValue, "encoding and decoding produces different values")
-}
-
 func TestPrefix(t *testing.T) {
 	t.Run("panics on invalid int", func(t *testing.T) {
 		require.Panics(t, func() {

collections/colltest/codec.go

@@ -0,0 +1,55 @@
+package colltest
+
+import (
+	"cosmossdk.io/collections"
+	"github.com/stretchr/testify/require"
+	"testing"
+)
+
+// TestKeyCodec asserts the correct behaviour of a KeyCodec over the type T.
+func TestKeyCodec[T any](t *testing.T, keyCodec collections.KeyCodec[T], key T) {
+	buffer := make([]byte, keyCodec.Size(key))
+	written, err := keyCodec.Encode(buffer, key)
+	require.NoError(t, err)
+	require.Equal(t, len(buffer), written)
+	read, decodedKey, err := keyCodec.Decode(buffer)
+	require.NoError(t, err)
+	require.Equal(t, len(buffer), read, "encoded key and read bytes must have same size")
+	require.Equal(t, key, decodedKey, "encoding and decoding produces different keys")
+	// test if terminality is correctly applied
+	pairCodec := collections.PairKeyCodec(keyCodec, collections.StringKey)
+	pairKey := collections.Join(key, "TEST")
+	buffer = make([]byte, pairCodec.Size(pairKey))
+	written, err = pairCodec.Encode(buffer, pairKey)
+	require.NoError(t, err)
+	read, decodedPairKey, err := pairCodec.Decode(buffer)
+	require.NoError(t, err)
+	require.Equal(t, len(buffer), read, "encoded non terminal key and pair key read bytes must have same size")
+	require.Equal(t, pairKey, decodedPairKey, "encoding and decoding produces different keys with non terminal encoding")
+
+	// check JSON
+	keyJSON, err := keyCodec.EncodeJSON(key)
+	require.NoError(t, err)
+	decoded, err := keyCodec.DecodeJSON(keyJSON)
+	require.NoError(t, err)
+	require.Equal(t, key, decoded, "json encoding and decoding did not produce the same results")
+}
+
+// TestValueCodec asserts the correct behaviour of a ValueCodec over the type T.
+func TestValueCodec[T any](t *testing.T, encoder collections.ValueCodec[T], value T) {
+	encodedValue, err := encoder.Encode(value)
+	require.NoError(t, err)
+	decodedValue, err := encoder.Decode(encodedValue)
+	require.NoError(t, err)
+	require.Equal(t, value, decodedValue, "encoding and decoding produces different values")
+
+	encodedJSONValue, err := encoder.EncodeJSON(value)
+	require.NoError(t, err)
+	decodedJSONValue, err := encoder.DecodeJSON(encodedJSONValue)
+	require.NoError(t, err)
+	require.Equal(t, value, decodedJSONValue, "encoding and decoding in json format produces different values")
+
+	require.NotEmpty(t, encoder.ValueType())
+
+	_ = encoder.Stringify(value)
+}

collections/correctness_test.go

@@ -0,0 +1,38 @@
+package collections_test
+
+import (
+	"cosmossdk.io/collections"
+	"cosmossdk.io/collections/colltest"
+	"testing"
+)
+
+func TestKeyCorrectness(t *testing.T) {
+	t.Run("bytes", func(t *testing.T) {
+		colltest.TestKeyCodec(t, collections.BytesKey, []byte("some_cool_bytes"))
+	})
+
+	t.Run("string", func(t *testing.T) {
+		colltest.TestKeyCodec(t, collections.StringKey, "some string")
+	})
+
+	t.Run("uint64", func(t *testing.T) {
+		colltest.TestKeyCodec(t, collections.Uint64Key, 5949485)
+	})
+
+	t.Run("Pair", func(t *testing.T) {
+		colltest.TestKeyCodec(
+			t,
+			collections.PairKeyCodec(collections.StringKey, collections.StringKey),
+			collections.Join("hello", "testing"),
+		)
+	})
+}
+
+func TestValueCorrectness(t *testing.T) {
+	t.Run("string", func(t *testing.T) {
+		colltest.TestValueCodec(t, collections.StringValue, "i am a string")
+	})
+	t.Run("uint64", func(t *testing.T) {
+		colltest.TestValueCodec(t, collections.Uint64Value, 5948459845)
+	})
+}

collections/keys.go

@@ -3,27 +3,47 @@ package collections
 import (
 	"bytes"
 	"encoding/binary"
+	"encoding/hex"
 	"encoding/json"
 	"errors"
 	"fmt"
+	"math"
 	"strconv"
 )
 
 var (
-	// Uint64Key can be used to encode uint64 keys.
-	// Encoding is big endian to retain ordering.
+	// Uint64Key can be used to encode uint64 keys. Encoding is big endian to retain ordering.
 	Uint64Key KeyCodec[uint64] = uint64Key{}
-	// StringKey can be used to encode string keys.
-	// The encoding just converts the string to bytes.
+	// StringKey can be used to encode string keys. The encoding just converts the string
+	// to bytes.
+	// Non-terminality in multipart keys is handled by appending the StringDelimiter,
+	// this means that a string key when used as the non final part of a multipart key cannot
+	// contain the StringDelimiter.
+	// Lexicographical ordering is retained both in non and multipart keys.
 	StringKey KeyCodec[string] = stringKey{}
+	// BytesKey can be used to encode bytes keys. The encoding will just use
+	// the provided bytes.
+	// When used as the non-terminal part of a multipart key, we prefix the bytes key
+	// with a single byte representing the length of the key. This means two things:
+	// 1. When used in multipart keys the length can be at maximum 255 (max number that
+	// can be represented with a single byte).
+	// 2. When used in multipart keys the lexicographical ordering is lost due to the
+	// length prefixing.
+	// JSON encoding represents a bytes key as a hex encoded string.
+	BytesKey KeyCodec[[]byte] = bytesKey{}
+)
+
+const (
+	// StringDelimiter defines the delimiter of a string key when used in non-terminal encodings.
+	StringDelimiter uint8 = 0x0
+	// MaxBytesKeyNonTerminalSize defines the maximum length of a bytes key encoded
+	// using the BytesKey KeyCodec.
+	MaxBytesKeyNonTerminalSize = math.MaxUint8
 )
 
 // errDecodeKeySize is a sentinel error.
 var errDecodeKeySize = errors.New("decode error, wrong byte key size")
 
-// StringDelimiter defines the delimiter of a string key when used in non-terminal encodings.
-const StringDelimiter uint8 = 0x0
-
 type uint64Key struct{}
 
 func (uint64Key) Encode(buffer []byte, key uint64) (int, error) {
@@ -121,3 +141,71 @@ func (stringKey) Stringify(key string) string {
 func (stringKey) KeyType() string {
 	return "string"
 }
+
+type bytesKey struct{}
+
+func (b bytesKey) Encode(buffer []byte, key []byte) (int, error) {
+	return copy(buffer, key), nil
+}
+
+func (bytesKey) Decode(buffer []byte) (int, []byte, error) {
+	// todo: should we copy it? collections will just discard the buffer, so from coll POV is not needed.
+	return len(buffer), buffer, nil
+}
+
+func (bytesKey) Size(key []byte) int {
+	return len(key)
+}
+
+func (bytesKey) EncodeJSON(value []byte) ([]byte, error) {
+	return StringKey.EncodeJSON(hex.EncodeToString(value))
+}
+
+func (bytesKey) DecodeJSON(b []byte) ([]byte, error) {
+	hexBytes, err := StringKey.DecodeJSON(b)
+	if err != nil {
+		return nil, err
+	}
+	return hex.DecodeString(hexBytes)
+}
+
+func (b bytesKey) Stringify(key []byte) string {
+	return fmt.Sprintf("hexBytes:%x", key)
+}
+
+func (b bytesKey) KeyType() string {
+	return "bytes"
+}
+
+func (b bytesKey) EncodeNonTerminal(buffer []byte, key []byte) (int, error) {
+	if len(key) > MaxBytesKeyNonTerminalSize {
+		return 0, fmt.Errorf(
+			"%w: bytes key non terminal size cannot exceed: %d, got: %d",
+			ErrEncoding, MaxBytesKeyNonTerminalSize, len(key),
+		)
+	}
+
+	buffer[0] = uint8(len(key))
+	written := copy(buffer[1:], key)
+	return written + 1, nil
+}
+
+func (bytesKey) DecodeNonTerminal(buffer []byte) (int, []byte, error) {
+	l := len(buffer)
+	if l == 0 {
+		return 0, nil, fmt.Errorf("%w: bytes key non terminal decoding cannot have an empty buffer", ErrEncoding)
+	}
+
+	keyLength := int(buffer[0])
+	if len(buffer[1:]) < keyLength {
+		return 0, nil, fmt.Errorf(
+			"%w: bytes key non terminal decoding isn't big enough, want at least: %d, got: %d",
+			ErrEncoding, keyLength, len(buffer[1:]),
+		)
+	}
+	return 1 + keyLength, buffer[1 : keyLength+1], nil
+}
+
+func (bytesKey) SizeNonTerminal(key []byte) int {
+	return len(key) + 1
+}

collections/keys_test.go

@@ -7,18 +7,12 @@ import (
 )
 
 func TestUint64Key(t *testing.T) {
-	t.Run("correctness", func(t *testing.T) {
-		checkKeyCodec(t, Uint64Key, 55)
-	})
-
 	t.Run("invalid key size", func(t *testing.T) {
 		_, _, err := Uint64Key.Decode([]byte{0x0, 0x1})
 		require.ErrorIs(t, err, errDecodeKeySize)
 	})
 }
 
 func TestStringKey(t *testing.T) {
-	t.Run("correctness", func(t *testing.T) {
-		checkKeyCodec(t, StringKey, "test")
-	})
+
 }

collections/pair_test.go

@@ -7,10 +7,6 @@ import (
 
 func TestPair(t *testing.T) {
 	keyCodec := PairKeyCodec(StringKey, StringKey)
-	t.Run("correctness", func(t *testing.T) {
-		checkKeyCodec(t, keyCodec, Join("A", "B"))
-	})
-
 	t.Run("stringify", func(t *testing.T) {
 		s := keyCodec.Stringify(Join("a", "b"))
 		require.Equal(t, `("a", "b")`, s)

collections/values_test.go

@@ -7,9 +7,6 @@ import (
 )
 
 func TestUint64Value(t *testing.T) {
-	t.Run("bijective", func(t *testing.T) {
-		checkValueCodec(t, Uint64Value, 555)
-	})
 
 	t.Run("invalid size", func(t *testing.T) {
 		_, err := Uint64Value.Decode([]byte{0x1, 0x2})

go.mod

@@ -4,7 +4,7 @@ module github.com/cosmos/cosmos-sdk
 
 require (
 	cosmossdk.io/api v0.2.6
-	cosmossdk.io/collections v0.0.0-20230124101704-57bedb100648
+	cosmossdk.io/collections v0.0.0-20230124184726-872ec34a5846
 	cosmossdk.io/core v0.5.0
 	cosmossdk.io/depinject v1.0.0-alpha.3
 	cosmossdk.io/errors v1.0.0-beta.7