WIP: Commitments storage added to state. Basic functions implemented.

Commitments serialization implemented and tested.
Checks of CommitToGeneration transaction against storage of commitments added to transaction checker.
Tests on new checks implemented.
New setting MaxGenerators added to networks configurations.
StageNet settings updated.

Author: alexeykiselev

pkg/settings/blockchain_settings.go

@@ -106,6 +106,7 @@ type FunctionalitySettings struct {
 	LightNodeBlockFieldsAbsenceInterval uint64 `json:"light_node_block_fields_absence_interval"`
 
 	GenerationPeriod uint64 `json:"generation_period"`
+	MaxGenerators    int    `json:"max_generators"`
 }
 
 func (f *FunctionalitySettings) VotesForFeatureElection(height uint64) uint64 {

pkg/settings/embedded/mainnet.json

@@ -47,6 +47,7 @@
   "min_update_asset_info_interval": 100000,
   "light_node_block_fields_absence_interval": 1000,
   "generation_period": 10000,
+  "max_generators": 128,
   "type": 0,
   "genesis": {
     "version": 1,

pkg/settings/embedded/stagenet.json

@@ -60,7 +60,8 @@
   "block_reward_boost_period": 1000,
   "min_update_asset_info_interval": 10,
   "light_node_block_fields_absence_interval": 1000,
-  "generation_period": 100,
+  "generation_period": 1000,
+  "max_generators": 32,
   "type": 2,
   "genesis": {
     "version": 1,

pkg/settings/embedded/testnet.json

@@ -47,6 +47,7 @@
   "min_update_asset_info_interval": 100000,
   "light_node_block_fields_absence_interval": 1000,
   "generation_period": 3000,
+  "max_generators": 64,
   "type": 1,
   "genesis": {
     "version": 1,

pkg/state/commitments.go

@@ -0,0 +1,123 @@
+package state
+
+import (
+	"bytes"
+	"errors"
+	"fmt"
+
+	"github.com/fxamacker/cbor/v2"
+
+	"github.com/wavesplatform/gowaves/pkg/crypto"
+	"github.com/wavesplatform/gowaves/pkg/crypto/bls"
+	"github.com/wavesplatform/gowaves/pkg/keyvalue"
+	"github.com/wavesplatform/gowaves/pkg/proto"
+)
+
+// commitmentItem represents a single commitment made by a block generator.
+// It links the generator's Waves public key to its corresponding BLS endorser public key.
+type commitmentItem struct {
+	GeneratorPK crypto.PublicKey `cbor:"0,keyasint,omitempty"`
+	EndorserPK  bls.PublicKey    `cbor:"1,keyasint,omitempty"`
+}
+
+// commitmentsRecord holds all generator commitments for a specific generation period.
+type commitmentsRecord struct {
+	Commitments []commitmentItem `cbor:"0,keyasint,omitempty"`
+}
+
+func (cr *commitmentsRecord) append(generatorPK crypto.PublicKey, endorserPK bls.PublicKey) {
+	cr.Commitments = append(cr.Commitments, commitmentItem{
+		GeneratorPK: generatorPK,
+		EndorserPK:  endorserPK,
+	})
+}
+func (cr *commitmentsRecord) marshalBinary() ([]byte, error) { return cbor.Marshal(cr) }
+
+func (cr *commitmentsRecord) unmarshalBinary(data []byte) error { return cbor.Unmarshal(data, cr) }
+
+// commitments manages the storage and retrieval of generator commitments.
+type commitments struct {
+	db      keyvalue.IterableKeyVal
+	dbBatch keyvalue.Batch
+	hs      *historyStorage
+
+	scheme          proto.Scheme
+	calculateHashes bool
+	hasher          *stateHasher
+}
+
+func newCommitments(hs *historyStorage, scheme proto.Scheme, calcHashes bool) *commitments {
+	return &commitments{
+		db:              hs.db,
+		dbBatch:         hs.dbBatch,
+		hs:              hs,
+		scheme:          scheme,
+		calculateHashes: calcHashes,
+		hasher:          newStateHasher(),
+	}
+}
+
+func (c *commitments) store(
+	periodStart uint32, generatorPK crypto.PublicKey, endorserPK bls.PublicKey, blockID proto.BlockID,
+) error {
+	key := commitmentKey{periodStart: periodStart}
+	data, err := c.hs.newestTopEntryData(key.bytes())
+	if err != nil && !errors.Is(err, keyvalue.ErrNotFound) {
+		return fmt.Errorf("failed to retrieve commitments record: %w", err)
+	}
+	var rec commitmentsRecord
+	if data != nil {
+		if umErr := rec.unmarshalBinary(data); umErr != nil {
+			return fmt.Errorf("failed to unmarshal commitments record: %w", umErr)
+		}
+	}
+	rec.append(generatorPK, endorserPK)
+	newData, mErr := rec.marshalBinary()
+	if mErr != nil {
+		return fmt.Errorf("failed to marshal commitments record: %w", mErr)
+	}
+	if addErr := c.hs.addNewEntry(commitment, key.bytes(), newData, blockID); addErr != nil {
+		return fmt.Errorf("failed to add commitment record: %w", addErr)
+	}
+	return nil
+}
+
+// exists checks if a commitment exists for the given period start and generator public key.
+func (c *commitments) exists(periodStart uint32, generatorPK crypto.PublicKey) (bool, error) {
+	key := commitmentKey{periodStart: periodStart}
+	data, err := c.hs.newestTopEntryData(key.bytes())
+	if err != nil {
+		if errors.Is(err, keyvalue.ErrNotFound) {
+			return false, nil
+		}
+		return false, fmt.Errorf("failed to retrieve commitment record: %w", err)
+	}
+	var rec commitmentsRecord
+	if umErr := rec.unmarshalBinary(data); umErr != nil {
+		return false, fmt.Errorf("failed to unmarshal commitment record: %w", umErr)
+	}
+	pkb := generatorPK.Bytes()
+	for _, cm := range rec.Commitments {
+		if bytes.Equal(pkb, cm.GeneratorPK.Bytes()) {
+			return true, nil
+		}
+	}
+	return false, nil
+}
+
+// size returns the number of commitments for the given period start.
+func (c *commitments) size(periodStart uint32) (int, error) {
+	key := commitmentKey{periodStart: periodStart}
+	data, err := c.hs.newestTopEntryData(key.bytes())
+	if err != nil {
+		if errors.Is(err, keyvalue.ErrNotFound) {
+			return 0, nil
+		}
+		return 0, fmt.Errorf("failed to retrieve commitment record: %w", err)
+	}
+	var rec commitmentsRecord
+	if umErr := rec.unmarshalBinary(data); umErr != nil {
+		return 0, fmt.Errorf("failed to unmarshal commitment record: %w", umErr)
+	}
+	return len(rec.Commitments), nil
+}

pkg/state/commitments_internal_test.go

@@ -0,0 +1,164 @@
+package state
+
+import (
+	"crypto/rand"
+	"fmt"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/wavesplatform/gowaves/pkg/crypto"
+	"github.com/wavesplatform/gowaves/pkg/crypto/bls"
+	"github.com/wavesplatform/gowaves/pkg/proto"
+)
+
+func TestCommitmentsRecordRoundTrip(t *testing.T) {
+	for i, test := range []int{
+		1, 8, 16, 32, 64, 128, 256, 512, 1024,
+	} {
+		t.Run(fmt.Sprintf("%d", i+1), func(t *testing.T) {
+			rec := commitmentsRecord{Commitments: generateCommitments(t, test)}
+
+			data, err := rec.marshalBinary()
+			require.NoError(t, err)
+			assert.NotNil(t, data)
+
+			var decoded commitmentsRecord
+			err = decoded.unmarshalBinary(data)
+			require.NoError(t, err)
+			assert.Equal(t, rec, decoded)
+			for i, cm := range rec.Commitments {
+				assert.Equal(t, cm.GeneratorPK, decoded.Commitments[i].GeneratorPK)
+				assert.Equal(t, cm.EndorserPK, decoded.Commitments[i].EndorserPK)
+			}
+		})
+	}
+}
+
+func BenchmarkCommitmentsRecordMarshalling(b *testing.B) {
+	for _, n := range []int{
+		1, 8, 16, 32, 64, 128, 256, 512, 1024,
+	} {
+		b.Run(fmt.Sprintf("%d", n), func(b *testing.B) {
+			rec := commitmentsRecord{Commitments: generateCommitments(b, n)}
+			b.ResetTimer()
+			for i := 0; i < b.N; i++ {
+				_, err := rec.marshalBinary()
+				if err != nil {
+					b.Fatal(err)
+				}
+			}
+		})
+	}
+}
+
+func BenchmarkCommitmentsRecordUnmarshalling(b *testing.B) {
+	for _, n := range []int{
+		1, 8, 16, 32, 64, 128, 256, 512, 1024,
+	} {
+		b.Run(fmt.Sprintf("%d", n), func(b *testing.B) {
+			rec := commitmentsRecord{Commitments: generateCommitments(b, n)}
+			data, err := rec.marshalBinary()
+			if err != nil {
+				b.Fatal(err)
+			}
+			b.ResetTimer()
+			for i := 0; i < b.N; i++ {
+				var decoded commitmentsRecord
+				err = decoded.unmarshalBinary(data)
+				if err != nil {
+					b.Fatal(err)
+				}
+			}
+		})
+	}
+}
+
+func TestCommitments_Exists(t *testing.T) {
+	for i, test := range []struct {
+		periodStart uint32
+		n           int
+	}{
+		{periodStart: 1_000_000, n: 1},
+		{periodStart: 2_000_000, n: 32},
+		{periodStart: 3_000_000, n: 64},
+		{periodStart: 4_000_000, n: 128},
+	} {
+		t.Run(fmt.Sprintf("%d", i+1), func(t *testing.T) {
+			to := createStorageObjects(t, true)
+			cms := generateCommitments(t, test.n+1)
+			for j := range test.n {
+				blockID := generateRandomBlockID(t)
+				to.addBlock(t, blockID)
+				err := to.entities.commitments.store(test.periodStart, cms[j].GeneratorPK, cms[j].EndorserPK, blockID)
+				require.NoError(t, err)
+				to.flush(t)
+
+				// Check that all added commitments exist.
+				for k := range j {
+					ok, eErr := to.entities.commitments.exists(test.periodStart, cms[k].GeneratorPK)
+					require.NoError(t, eErr)
+					assert.True(t, ok)
+				}
+
+				// Check that non-existing commitment does not exist.
+				ok, err := to.entities.commitments.exists(test.periodStart, cms[test.n].GeneratorPK)
+				require.NoError(t, err)
+				assert.False(t, ok)
+			}
+		})
+	}
+}
+
+func TestCommitments_Size(t *testing.T) {
+	for i, test := range []struct {
+		periodStart uint32
+		n           int
+	}{
+		{periodStart: 1_000_000, n: 1},
+		{periodStart: 2_000_000, n: 32},
+		{periodStart: 3_000_000, n: 64},
+		{periodStart: 4_000_000, n: 128},
+	} {
+		t.Run(fmt.Sprintf("%d", i+1), func(t *testing.T) {
+			to := createStorageObjects(t, true)
+			cms := generateCommitments(t, test.n+1)
+			for j := range test.n {
+				blockID := generateRandomBlockID(t)
+				to.addBlock(t, blockID)
+				err := to.entities.commitments.store(test.periodStart, cms[j].GeneratorPK, cms[j].EndorserPK, blockID)
+				require.NoError(t, err)
+				to.flush(t)
+
+				s, err := to.entities.commitments.size(test.periodStart)
+				require.NoError(t, err)
+				assert.Equal(t, j+1, s)
+			}
+		})
+	}
+}
+
+func generateCommitments(t testing.TB, n int) []commitmentItem {
+	r := make([]commitmentItem, n)
+	for i := 0; i < n; i++ {
+		_, wpk, err := crypto.GenerateKeyPair([]byte(fmt.Sprintf("WAVES_%d", i)))
+		require.NoError(t, err)
+		bsk, err := bls.GenerateSecretKey([]byte(fmt.Sprintf("BLS_%d", i)))
+		require.NoError(t, err)
+		bpk, err := bsk.PublicKey()
+		require.NoError(t, err)
+		r[i] = commitmentItem{
+			GeneratorPK: wpk,
+			EndorserPK:  bpk,
+		}
+	}
+	return r
+}
+
+func generateRandomBlockID(t testing.TB) proto.BlockID {
+	var sig crypto.Signature
+	_, err := rand.Read(sig[:])
+	require.NoError(t, err)
+	return proto.NewBlockIDFromSignature(sig)
+}

pkg/state/history_storage.go

@@ -45,6 +45,7 @@ const (
 	snapshots
 	patches
 	challengedAddress
+	commitment
 )
 
 type blockchainEntityProperties struct {
@@ -212,6 +213,11 @@ var properties = map[blockchainEntity]blockchainEntityProperties{
 		needToCut:    true,
 		fixedSize:    false,
 	},
+	commitment: {
+		needToFilter: true,
+		needToCut:    true,
+		fixedSize:    false,
+	},
 }
 
 type historyEntry struct {

pkg/state/keys.go

@@ -133,8 +133,10 @@ const (
 	snapshotsKeyPrefix
 	// Blockchain patches storage.
 	patchKeyPrefix
-
+	// Key to store and retrieve challenged addresses.
 	challengedAddressKeyPrefix
+	// Key to store and retrieve generator's commitments for a specific generation period.
+	commitmentKeyPrefix
 )
 
 var (
@@ -761,3 +763,14 @@ func (k *challengedAddressKey) bytes() []byte {
 	copy(buf[1:], k.address[:])
 	return buf
 }
+
+type commitmentKey struct {
+	periodStart uint32
+}
+
+func (k *commitmentKey) bytes() []byte {
+	buf := make([]byte, 1+uint32Size)
+	buf[0] = commitmentKeyPrefix
+	binary.BigEndian.PutUint32(buf[1:], k.periodStart)
+	return buf
+}

pkg/state/state.go

@@ -68,6 +68,7 @@ type blockchainEntitiesStorage struct {
 	hitSources        *hitSources
 	snapshots         *snapshotsAtHeight
 	patches           *patchesStorage
+	commitments       *commitments
 	calculateHashes   bool
 }
 
@@ -102,6 +103,7 @@ func newBlockchainEntitiesStorage(hs *historyStorage, sets *settings.BlockchainS
 		newHitSources(hs),
 		newSnapshotsAtHeight(hs, sets.AddressSchemeCharacter),
 		newPatchesStorage(hs, sets.AddressSchemeCharacter),
+		newCommitments(hs, sets.AddressSchemeCharacter, calcHashes),
 		calcHashes,
 	}, nil
 }