refactor: stores into subdirs (precomputed_key and generated_key)

Layr-Labs · Sep 22, 2024 · 25d2c17 · 25d2c17
1 parent 0b0cf9d
commit 25d2c17
Show file tree

Hide file tree

Showing 11 changed files with 133 additions and 121 deletions.
diff --git a/cli/config.go b/cli/config.go
@@ -8,6 +8,8 @@ import (
 	"github.com/urfave/cli/v2"
 
 	"github.com/Layr-Labs/eigenda-proxy/store"
+	"github.com/Layr-Labs/eigenda-proxy/store/precomputed_key/redis"
+	"github.com/Layr-Labs/eigenda-proxy/store/precomputed_key/s3"
 	"github.com/Layr-Labs/eigenda-proxy/utils"
 	"github.com/Layr-Labs/eigenda-proxy/verify"
 	"github.com/Layr-Labs/eigenda/api/clients"
@@ -61,8 +63,8 @@ type Config struct {
 	CacheTargets    []string
 
 	// secondary storage
-	RedisConfig store.RedisConfig
-	S3Config    store.S3Config
+	RedisConfig redis.Config
+	S3Config    s3.Config
 }
 
 // GetMaxBlobLength ... returns the maximum blob length in bytes
@@ -111,14 +113,14 @@ func (cfg *Config) VerificationCfg() *verify.Config {
 // ReadConfig ... parses the Config from the provided flags or environment variables.
 func ReadConfig(ctx *cli.Context) Config {
 	cfg := Config{
-		RedisConfig: store.RedisConfig{
+		RedisConfig: redis.Config{
 			Endpoint: ctx.String(RedisEndpointFlagName),
 			Password: ctx.String(RedisPasswordFlagName),
 			DB:       ctx.Int(RedisDBFlagName),
 			Eviction: ctx.Duration(RedisEvictionFlagName),
 		},
-		S3Config: store.S3Config{
-			S3CredentialType: store.StringToS3CredentialType(ctx.String(S3CredentialTypeFlagName)),
+		S3Config: s3.Config{
+			S3CredentialType: s3.StringToCredentialType(ctx.String(S3CredentialTypeFlagName)),
 			Bucket:           ctx.String(S3BucketFlagName),
 			Path:             ctx.String(S3PathFlagName),
 			Endpoint:         ctx.String(S3EndpointFlagName),
@@ -214,10 +216,10 @@ func (cfg *Config) Check() error {
 		}
 	}
 
-	if cfg.S3Config.S3CredentialType == store.S3CredentialUnknown && cfg.S3Config.Endpoint != "" {
+	if cfg.S3Config.S3CredentialType == s3.CredentialTypeUnknown && cfg.S3Config.Endpoint != "" {
 		return fmt.Errorf("s3 credential type must be set")
 	}
-	if cfg.S3Config.S3CredentialType == store.S3CredentialStatic {
+	if cfg.S3Config.S3CredentialType == s3.CredentialTypeStatic {
 		if cfg.S3Config.Endpoint != "" && (cfg.S3Config.AccessKeyID == "" || cfg.S3Config.AccessKeySecret == "") {
 			return fmt.Errorf("s3 endpoint is set, but access key id or access key secret is not set")
 		}

diff --git a/cli/config_test.go b/cli/config_test.go
@@ -4,20 +4,21 @@ import (
 	"testing"
 	"time"
 
-	"github.com/Layr-Labs/eigenda-proxy/store"
+	"github.com/Layr-Labs/eigenda-proxy/store/precomputed_key/redis"
+	"github.com/Layr-Labs/eigenda-proxy/store/precomputed_key/s3"
 	"github.com/Layr-Labs/eigenda/api/clients"
 	"github.com/stretchr/testify/require"
 )
 
 func validCfg() *Config {
 	return &Config{
-		RedisConfig: store.RedisConfig{
+		RedisConfig: redis.Config{
 			Endpoint: "localhost:6379",
 			Password: "password",
 			DB:       0,
 			Eviction: 10 * time.Minute,
 		},
-		S3Config: store.S3Config{
+		S3Config: s3.Config{
 			Bucket:          "test-bucket",
 			Path:            "",
 			Endpoint:        "http://localhost:9000",
@@ -110,7 +111,7 @@ func TestConfigVerification(t *testing.T) {
 	t.Run("MissingS3AccessKeys", func(t *testing.T) {
 		cfg := validCfg()
 
-		cfg.S3Config.S3CredentialType = store.S3CredentialStatic
+		cfg.S3Config.S3CredentialType = s3.CredentialTypeStatic
 		cfg.S3Config.Endpoint = "http://localhost:9000"
 		cfg.S3Config.AccessKeyID = ""
 
@@ -121,7 +122,7 @@ func TestConfigVerification(t *testing.T) {
 	t.Run("MissingS3Credential", func(t *testing.T) {
 		cfg := validCfg()
 
-		cfg.S3Config.S3CredentialType = store.S3CredentialUnknown
+		cfg.S3Config.S3CredentialType = s3.CredentialTypeUnknown
 
 		err := cfg.Check()
 		require.Error(t, err)

diff --git a/e2e/server_test.go b/e2e/server_test.go
@@ -393,7 +393,7 @@ func TestProxyServerCachingWithRedis(t *testing.T) {
 	require.Equal(t, testPreimage, preimage)
 
 	// ensure that read was from cache
-	redStats, err := ts.Server.GetStoreStats(store.Redis)
+	redStats, err := ts.Server.GetStoreStats(store.RedisBackendType)
 	require.NoError(t, err)
 
 	require.Equal(t, 1, redStats.Reads)

diff --git a/e2e/setup.go b/e2e/setup.go
@@ -10,7 +10,8 @@ import (
 	"github.com/Layr-Labs/eigenda-proxy/cli"
 	"github.com/Layr-Labs/eigenda-proxy/metrics"
 	"github.com/Layr-Labs/eigenda-proxy/server"
-	"github.com/Layr-Labs/eigenda-proxy/store"
+	"github.com/Layr-Labs/eigenda-proxy/store/precomputed_key/redis"
+	"github.com/Layr-Labs/eigenda-proxy/store/precomputed_key/s3"
 	"github.com/Layr-Labs/eigenda/api/clients"
 	"github.com/ethereum/go-ethereum/log"
 	"github.com/minio/minio-go/v7"
@@ -54,7 +55,7 @@ func TestConfig(useMemory bool) *Cfg {
 }
 
 func createRedisConfig(eigendaCfg cli.Config) cli.CLIConfig {
-	eigendaCfg.RedisConfig = store.RedisConfig{
+	eigendaCfg.RedisConfig = redis.Config{
 		Endpoint: "127.0.0.1:9001",
 		Password: "",
 		DB:       0,
@@ -71,14 +72,14 @@ func createS3Config(eigendaCfg cli.Config) cli.CLIConfig {
 	bucketName := "eigenda-proxy-test-" + RandString(10)
 	createS3Bucket(bucketName)
 
-	eigendaCfg.S3Config = store.S3Config{
+	eigendaCfg.S3Config = s3.Config{
 		Profiling:        true,
 		Bucket:           bucketName,
 		Path:             "",
 		Endpoint:         "localhost:4566",
 		AccessKeySecret:  "minioadmin",
 		AccessKeyID:      "minioadmin",
-		S3CredentialType: store.S3CredentialStatic,
+		S3CredentialType: s3.CredentialTypeStatic,
 		Backup:           false,
 	}
 	return cli.CLIConfig{

diff --git a/server/load_store.go b/server/load_store.go
@@ -6,26 +6,30 @@ import (
 
 	"github.com/Layr-Labs/eigenda-proxy/cli"
 	"github.com/Layr-Labs/eigenda-proxy/store"
+	"github.com/Layr-Labs/eigenda-proxy/store/generated_key/eigenda"
+	"github.com/Layr-Labs/eigenda-proxy/store/generated_key/memstore"
+	"github.com/Layr-Labs/eigenda-proxy/store/precomputed_key/redis"
+	"github.com/Layr-Labs/eigenda-proxy/store/precomputed_key/s3"
 	"github.com/Layr-Labs/eigenda-proxy/verify"
 	"github.com/Layr-Labs/eigenda/api/clients"
 	"github.com/ethereum/go-ethereum/log"
 )
 
 // populateTargets ... creates a list of storage backends based on the provided target strings
-func populateTargets(targets []string, s3 store.PrecomputedKeyStore, redis *store.RedStore) []store.PrecomputedKeyStore {
+func populateTargets(targets []string, s3 store.PrecomputedKeyStore, redis *redis.Store) []store.PrecomputedKeyStore {
 	stores := make([]store.PrecomputedKeyStore, len(targets))
 
 	for i, f := range targets {
 		b := store.StringToBackendType(f)
 
 		switch b {
-		case store.Redis:
+		case store.RedisBackendType:
 			stores[i] = redis
 
-		case store.S3:
+		case store.S3BackendType:
 			stores[i] = s3
 
-		case store.EigenDA, store.Memory:
+		case store.EigenDABackendType, store.MemoryBackendType:
 			panic(fmt.Sprintf("Invalid target for fallback: %s", f))
 
 		case store.Unknown:
@@ -43,12 +47,12 @@ func populateTargets(targets []string, s3 store.PrecomputedKeyStore, redis *stor
 func LoadStoreRouter(ctx context.Context, cfg cli.CLIConfig, log log.Logger) (store.IRouter, error) {
 	// create S3 backend store (if enabled)
 	var err error
-	var s3 store.PrecomputedKeyStore
-	var redis *store.RedStore
+	var s3Store store.PrecomputedKeyStore
+	var redisStore *redis.Store
 
 	if cfg.EigenDAConfig.S3Config.Bucket != "" && cfg.EigenDAConfig.S3Config.Endpoint != "" {
 		log.Info("Using S3 backend")
-		s3, err = store.NewS3(cfg.EigenDAConfig.S3Config)
+		s3Store, err = s3.NewS3(cfg.EigenDAConfig.S3Config)
 		if err != nil {
 			return nil, fmt.Errorf("failed to create S3 store: %w", err)
 		}
@@ -57,7 +61,7 @@ func LoadStoreRouter(ctx context.Context, cfg cli.CLIConfig, log log.Logger) (st
 	if cfg.EigenDAConfig.RedisConfig.Endpoint != "" {
 		log.Info("Using Redis backend")
 		// create Redis backend store
-		redis, err = store.NewRedisStore(&cfg.EigenDAConfig.RedisConfig)
+		redisStore, err = redis.NewStore(&cfg.EigenDAConfig.RedisConfig)
 		if err != nil {
 			return nil, fmt.Errorf("failed to create Redis store: %w", err)
 		}
@@ -86,10 +90,10 @@ func LoadStoreRouter(ctx context.Context, cfg cli.CLIConfig, log log.Logger) (st
 	}
 
 	// create EigenDA backend store
-	var eigenda store.KeyGeneratedStore
+	var eigenDA store.KeyGeneratedStore
 	if cfg.EigenDAConfig.MemstoreEnabled {
 		log.Info("Using mem-store backend for EigenDA")
-		eigenda, err = store.NewMemStore(ctx, verifier, log, store.MemStoreConfig{
+		eigenDA, err = memstore.New(ctx, verifier, log, memstore.Config{
 			MaxBlobSizeBytes: maxBlobLength,
 			BlobExpiration:   cfg.EigenDAConfig.MemstoreBlobExpiration,
 			PutLatency:       cfg.EigenDAConfig.MemstorePutLatency,
@@ -103,11 +107,11 @@ func LoadStoreRouter(ctx context.Context, cfg cli.CLIConfig, log log.Logger) (st
 			return nil, err
 		}
 
-		eigenda, err = store.NewEigenDAStore(
+		eigenDA, err = eigenda.NewStore(
 			client,
 			verifier,
 			log,
-			&store.EigenDAStoreConfig{
+			&eigenda.StoreConfig{
 				MaxBlobSizeBytes:     maxBlobLength,
 				EthConfirmationDepth: uint64(cfg.EigenDAConfig.EthConfirmationDepth), // #nosec G115
 				StatusQueryTimeout:   cfg.EigenDAConfig.ClientConfig.StatusQueryTimeout,
@@ -120,9 +124,9 @@ func LoadStoreRouter(ctx context.Context, cfg cli.CLIConfig, log log.Logger) (st
 	}
 
 	// determine read fallbacks
-	fallbacks := populateTargets(cfg.EigenDAConfig.FallbackTargets, s3, redis)
-	caches := populateTargets(cfg.EigenDAConfig.CacheTargets, s3, redis)
+	fallbacks := populateTargets(cfg.EigenDAConfig.FallbackTargets, s3Store, redisStore)
+	caches := populateTargets(cfg.EigenDAConfig.CacheTargets, s3Store, redisStore)
 
-	log.Info("Creating storage router", "eigenda backend type", eigenda != nil, "s3 backend type", s3 != nil)
-	return store.NewRouter(eigenda, s3, log, caches, fallbacks)
+	log.Info("Creating storage router", "eigenda backend type", eigenDA != nil, "s3 backend type", s3Store != nil)
+	return store.NewRouter(eigenDA, s3Store, log, caches, fallbacks)
 }
diff --git a/store/eigenda.go → store/generated_key/eigenda/eigenda.go b/store/eigenda.go → store/generated_key/eigenda/eigenda.go
@@ -1,18 +1,19 @@
-package store
+package eigenda
 
 import (
 	"context"
 	"errors"
 	"fmt"
 	"time"
 
+	"github.com/Layr-Labs/eigenda-proxy/store"
 	"github.com/Layr-Labs/eigenda-proxy/verify"
 	"github.com/Layr-Labs/eigenda/api/clients"
 	"github.com/ethereum/go-ethereum/log"
 	"github.com/ethereum/go-ethereum/rlp"
 )
 
-type EigenDAStoreConfig struct {
+type StoreConfig struct {
 	MaxBlobSizeBytes uint64
 	// the # of Ethereum blocks to wait after the EigenDA L1BlockReference # before attempting to verify
 	// & accredit a blob
@@ -22,19 +23,19 @@ type EigenDAStoreConfig struct {
 	StatusQueryTimeout time.Duration
 }
 
-// EigenDAStore does storage interactions and verifications for blobs with DA.
-type EigenDAStore struct {
+// Store does storage interactions and verifications for blobs with DA.
+type Store struct {
 	client   *clients.EigenDAClient
 	verifier *verify.Verifier
-	cfg      *EigenDAStoreConfig
+	cfg      *StoreConfig
 	log      log.Logger
 }
 
-var _ KeyGeneratedStore = (*EigenDAStore)(nil)
+var _ store.KeyGeneratedStore = (*Store)(nil)
 
-func NewEigenDAStore(client *clients.EigenDAClient,
-	v *verify.Verifier, log log.Logger, cfg *EigenDAStoreConfig) (*EigenDAStore, error) {
-	return &EigenDAStore{
+func NewStore(client *clients.EigenDAClient,
+	v *verify.Verifier, log log.Logger, cfg *StoreConfig) (*Store, error) {
+	return &Store{
 		client:   client,
 		verifier: v,
 		log:      log,
@@ -44,7 +45,7 @@ func NewEigenDAStore(client *clients.EigenDAClient,
 
 // Get fetches a blob from DA using certificate fields and verifies blob
 // against commitment to ensure data is valid and non-tampered.
-func (e EigenDAStore) Get(ctx context.Context, key []byte) ([]byte, error) {
+func (e Store) Get(ctx context.Context, key []byte) ([]byte, error) {
 	var cert verify.Certificate
 	err := rlp.DecodeBytes(key, &cert)
 	if err != nil {
@@ -60,13 +61,13 @@ func (e EigenDAStore) Get(ctx context.Context, key []byte) ([]byte, error) {
 }
 
 // Put disperses a blob for some pre-image and returns the associated RLP encoded certificate commit.
-func (e EigenDAStore) Put(ctx context.Context, value []byte) ([]byte, error) {
+func (e Store) Put(ctx context.Context, value []byte) ([]byte, error) {
 	encodedBlob, err := e.client.GetCodec().EncodeBlob(value)
 	if err != nil {
 		return nil, fmt.Errorf("EigenDA client failed to re-encode blob: %w", err)
 	}
 	if uint64(len(encodedBlob)) > e.cfg.MaxBlobSizeBytes {
-		return nil, fmt.Errorf("%w: blob length %d, max blob size %d", ErrProxyOversizedBlob, len(value), e.cfg.MaxBlobSizeBytes)
+		return nil, fmt.Errorf("%w: blob length %d, max blob size %d", store.ErrProxyOversizedBlob, len(value), e.cfg.MaxBlobSizeBytes)
 	}
 
 	dispersalStart := time.Now()
@@ -116,18 +117,18 @@ func (e EigenDAStore) Put(ctx context.Context, value []byte) ([]byte, error) {
 }
 
 // Entries are a no-op for EigenDA Store
-func (e EigenDAStore) Stats() *Stats {
+func (e Store) Stats() *store.Stats {
 	return nil
 }
 
 // Backend returns the backend type for EigenDA Store
-func (e EigenDAStore) BackendType() BackendType {
-	return EigenDA
+func (e Store) BackendType() store.BackendType {
+	return store.EigenDABackendType
 }
 
 // Key is used to recover certificate fields and that verifies blob
 // against commitment to ensure data is valid and non-tampered.
-func (e EigenDAStore) Verify(key []byte, value []byte) error {
+func (e Store) Verify(key []byte, value []byte) error {
 	var cert verify.Certificate
 	err := rlp.DecodeBytes(key, &cert)
 	if err != nil {