Merge branch 'main' into public-s3

README.md

@@ -269,10 +269,9 @@ To quickly set up monitoring dashboard, add eigenda-proxy metrics endpoint to a
 | `--eigenda.signer-private-key-hex`          |                                 | `$EIGENDA_PROXY_EIGENDA_SIGNER_PRIVATE_KEY_HEX`          | Hex-encoded signer private key. This key should not be associated with an Ethereum address holding any funds.                                                                                                 |
 | `--eigenda.status-query-retry-interval`     | `5s`                            | `$EIGENDA_PROXY_EIGENDA_STATUS_QUERY_INTERVAL`           | Interval between retries when awaiting network blob finalization. Default is 5 seconds.                                                                                                                       |
 | `--eigenda.status-query-timeout`            | `30m0s`                         | `$EIGENDA_PROXY_EIGENDA_STATUS_QUERY_TIMEOUT`            | Duration to wait for a blob to finalize after being sent for dispersal. Default is 30 minutes.                                                                                                                |
-| `--log.color`                               | `false`                         | `$EIGENDA_PROXY_LOG_COLOR`                               | Color the log output if in terminal mode.                                                                                                                                                                     |
-| `--log.format`                              | `text`                          | `$EIGENDA_PROXY_LOG_FORMAT`                              | Format the log output. Supported formats: 'text', 'terminal', 'logfmt', 'json', 'json-pretty'.                                                                                                                |
-| `--log.level`                               | `INFO`                          | `$EIGENDA_PROXY_LOG_LEVEL`                               | The lowest log level that will be output.                                                                                                                                                                     |
-| `--log.pid`                                 | `false`                         | `$EIGENDA_PROXY_LOG_PID`                                 | Show pid in the log.                                                                                                                                                                                          |
+| `--log.format`                              | `text`                          | `$EIGENDA_PROXY_LOG_FORMAT`                              | The format of the log file. Accepted options are 'json' and 'text' (default: "json")                                                                                                               |
+| `--log.level`                               | `INFO`                          | `$EIGENDA_PROXY_LOG_LEVEL`                               | The lowest log level that will be output. The lowest log level that will be output. Accepted options are "debug", "info", "warn" "error"                                                                                                                                                                     |
+| `log.path`                                  | `""`                              | `$EIGENDA_PROXY_LOG_PATH`                                | Path to file where logs will be written |
 | `--memstore.enabled`                        | `false`                         | `$EIGENDA_PROXY_MEMSTORE_ENABLED`                        | Whether to use mem-store for DA logic.                                                                                                                                                                        |
 | `--memstore.expiration`                     | `25m0s`                         | `$EIGENDA_PROXY_MEMSTORE_EXPIRATION`                     | Duration that a mem-store blob/commitment pair are allowed to live.                                                                                                                                           |
 | `--memstore.put-latency`                    | `0`                             | `$EIGENDA_PROXY_MEMSTORE_PUT_LATENCY`                    | Artificial latency added for memstore backend to mimic EigenDA's dispersal latency.                                                                                                                           |

SECURITY.md

@@ -6,14 +6,14 @@ Please see [Releases](https://github.com/Layr-Labs/eigenda-proxy/releases)
 
 ## Audit reports
 
-Audit reports are published in the `docs/audits` folder: https://github.com/Layr-Labs/eigenda-proxy/main/docs/audits 
+Audit reports are published in the [docs/audits](https://github.com/Layr-Labs/eigenda-proxy/tree/main/docs/audits) folder.
 
-| Date | Report Link |
-| ------- | ----------- |
-| 202501 | [pdf](https://github.com/Layr-Labs/eigenda-proxy/blob/main/docs/audits/Sigma_Prime_EigenDA_Proxy_Security_Assessment_Report.pdf) |
+| Date   | Release (Commit) Audited | Report Link                                                                                                                      | Findings Addressed in Release |
+| ------ | ------------------------ | -------------------------------------------------------------------------------------------------------------------------------- | ----------------------------- |
+| 202501 | v1.6.1 (9e1b746)         | [pdf](https://github.com/Layr-Labs/eigenda-proxy/blob/main/docs/audits/Sigma_Prime_EigenDA_Proxy_Security_Assessment_Report.pdf) | v1.6.2                        |
 
 ## Reporting a Vulnerability
 
-**Please do not file a public ticket** mentioning the vulnerability.
+**Please do not file a public ticket** mentioning a vulnerability.
 
-Please report security vulnerabilities to [email protected] with the all the relavent details included in the email.
+Please report security vulnerabilities to [email protected] with all the relavent details included in the email.

cmd/server/entrypoint.go

@@ -5,26 +5,35 @@ import (
 	"encoding/json"
 	"fmt"
 
+	proxy_logging "github.com/Layr-Labs/eigenda-proxy/logging"
+	"github.com/Layr-Labs/eigensdk-go/logging"
+
 	"github.com/Layr-Labs/eigenda-proxy/flags"
 	"github.com/Layr-Labs/eigenda-proxy/metrics"
 	"github.com/Layr-Labs/eigenda-proxy/server"
-	"github.com/ethereum/go-ethereum/log"
 	"github.com/urfave/cli/v2"
 
 	"github.com/ethereum-optimism/optimism/op-service/ctxinterrupt"
-	oplog "github.com/ethereum-optimism/optimism/op-service/log"
 )
 
 func StartProxySvr(cliCtx *cli.Context) error {
-	log := oplog.NewLogger(oplog.AppOut(cliCtx), oplog.ReadCLIConfig(cliCtx)).New("role", "eigenda_proxy")
-	oplog.SetGlobalLogHandler(log.Handler())
+	logCfg, err := proxy_logging.ReadLoggerCLIConfig(cliCtx)
+	if err != nil {
+		return err
+	}
+
+	log, err := proxy_logging.NewLogger(*logCfg)
+	if err != nil {
+		return err
+	}
+
 	log.Info("Starting EigenDA Proxy Server", "version", Version, "date", Date, "commit", Commit)
 
 	cfg := server.ReadCLIConfig(cliCtx)
 	if err := cfg.Check(); err != nil {
 		return err
 	}
-	err := prettyPrintConfig(cliCtx, log)
+	err = prettyPrintConfig(cliCtx, log)
 	if err != nil {
 		return fmt.Errorf("failed to pretty print config: %w", err)
 	}
@@ -73,7 +82,7 @@ func StartProxySvr(cliCtx *cli.Context) error {
 }
 
 // TODO: we should probably just change EdaClientConfig struct definition in eigenda-client
-func prettyPrintConfig(cliCtx *cli.Context, log log.Logger) error {
+func prettyPrintConfig(cliCtx *cli.Context, log logging.Logger) error {
 	// we read a new config which we modify to hide private info in order to log the rest
 	cfg := server.ReadCLIConfig(cliCtx)
 	if cfg.EigenDAConfig.EdaClientConfig.SignerPrivateKeyHex != "" {

common/common.go

@@ -6,6 +6,12 @@ import (
 	"strings"
 )
 
+const GlobalPrefix = "EIGENDA_PROXY"
+
+func PrefixEnvVar(prefix, suffix string) []string {
+	return []string{prefix + "_" + suffix}
+}
+
 // Helper utility functions //
 
 func ContainsDuplicates[P comparable](s []P) bool {

docker-compose.yaml

@@ -1,22 +1,82 @@
+## The following is a proxy instance 
+## pointed to redis for storage caching and S3
+## for storage failovers
+
 services:
+  ## Used as secondary read failover target
+  minio:
+    image: minio/minio:latest
+    container_name: minio
+    environment:
+      - MINIO_ROOT_USER=minioadmin
+      - MINIO_ROOT_PASSWORD=minioadmin
+    ports:
+      - "9000:9000"
+      - "9001:9001"
+    command: server /data
+    volumes:
+      - minio_data:/data
+
+  minio-init:
+    ## Seed test bucket
+    image: minio/mc:latest
+    depends_on:
+      - minio
+    entrypoint: ["/bin/sh", "-c", "/usr/bin/create-bucket.sh"]
+    volumes:
+      - ./scripts/create-test-s3-bucket.sh:/usr/bin/create-bucket.sh
+
+  redis:
+    image: redis:latest
+    container_name: redis
+    command: redis-server --requirepass redispassword
+    environment:
+      - REDIS_PASSWORD=redispassword
+    ports:
+      - "6379:6379"
+
   eigenda_proxy:
+    depends_on:
+      - minio-init
     build:
       context: .
       dockerfile: Dockerfile
     container_name: eigenda-proxy
     environment:
+      - EIGENDA_PROXY_LOG_LEVEL=debug
       - EIGENDA_PROXY_ADDR=0.0.0.0
       - EIGENDA_PROXY_PORT=4242
-      - EIGENDA_PROXY_MEMSTORE_ENABLED=false
+      ## Turn this off to talk to actual eigenda network
+      - EIGENDA_PROXY_MEMSTORE_ENABLED=true
       - EIGENDA_PROXY_MEMSTORE_EXPIRATION=45m
-      - EIGENDA_PROXY_EIGENDA_SIGNER_PRIVATE_KEY_HEX=$PRIVATE_KEY
+      - EIGENDA_PROXY_EIGENDA_CERT_VERIFICATION_DISABLED=true
+      - EIGENDA_PROXY_EIGENDA_SIGNER_PRIVATE_KEY_HEX=${PRIVATE_KEY}
       - EIGENDA_PROXY_EIGENDA_DISPERSER_RPC=disperser-holesky.eigenda.xyz:443
       - EIGENDA_PROXY_EIGENDA_SERVICE_MANAGER_ADDR=0xD4A7E1Bd8015057293f0D0A557088c286942e84b
-      - EIGENDA_PROXY_EIGENDA_ETH_RPC=$ETH_RPC
+      - EIGENDA_PROXY_EIGENDA_ETH_RPC=https://ethereum-holesky-rpc.publicnode.com
       - EIGENDA_PROXY_EIGENDA_ETH_CONFIRMATION_DEPTH=0
       - EIGENDA_PROXY_METRICS_ADDR=0.0.0.0
       - EIGENDA_PROXY_METRICS_ENABLED=true
       - EIGENDA_PROXY_METRICS_PORT=7300
+      ## S3
+      - EIGENDA_PROXY_S3_CREDENTIAL_TYPE=static
+      - EIGENDA_PROXY_S3_ACCESS_KEY_ID=minioadmin
+      - EIGENDA_PROXY_S3_ACCESS_KEY_SECRET=minioadmin
+      - EIGENDA_PROXY_S3_BUCKET=eigenda-proxy-test
+      - EIGENDA_PROXY_S3_PATH=""
+      - EIGENDA_PROXY_S3_ENDPOINT=minio:9000
+      - EIGENDA_PROXY_S3_ENABLE_TLS=false
+
+      # Redis Configuration
+      - EIGENDA_PROXY_REDIS_DB=0
+      - EIGENDA_PROXY_REDIS_ENDPOINT=redis:6379
+      - EIGENDA_PROXY_REDIS_PASSWORD=redispassword
+      - EIGENDA_PROXY_REDIS_EVICTION=24h0m0s
+
+      ## Secondary routing
+      - EIGENDA_PROXY_STORAGE_FALLBACK_TARGETS=s3
+      - EIGENDA_PROXY_STORAGE_CACHE_TARGETS=redis
+
     ports:
       - 4242:4242
       - 7300:7300
@@ -44,14 +104,6 @@ services:
     depends_on:
       - prometheus
 
-  traffic-generator:
-    image: alpine:latest
-    build: scripts/
-    container_name: traffic_generator
-    depends_on:
-      - eigenda_proxy
-    volumes:
-      - ./scripts/:/scripts/
-
 volumes:
   grafana-data:
+  minio_data:

e2e/setup.go

@@ -18,14 +18,12 @@ import (
 	"github.com/Layr-Labs/eigenda-proxy/verify"
 	"github.com/Layr-Labs/eigenda/api/clients"
 	"github.com/Layr-Labs/eigenda/encoding/kzg"
+	"github.com/Layr-Labs/eigensdk-go/logging"
 	"github.com/ethereum/go-ethereum/log"
 	"github.com/minio/minio-go/v7"
 	"github.com/minio/minio-go/v7/pkg/credentials"
 	"golang.org/x/exp/rand"
 
-	oplog "github.com/ethereum-optimism/optimism/op-service/log"
-	opmetrics "github.com/ethereum-optimism/optimism/op-service/metrics"
-
 	miniotc "github.com/testcontainers/testcontainers-go/modules/minio"
 	redistc "github.com/testcontainers/testcontainers-go/modules/redis"
 )
@@ -243,7 +241,7 @@ func TestSuiteConfig(testCfg *Cfg) server.CLIConfig {
 	default:
 		cfg = server.CLIConfig{
 			EigenDAConfig: eigendaCfg,
-			MetricsCfg:    opmetrics.CLIConfig{},
+			MetricsCfg:    metrics.CLIConfig{},
 		}
 	}
 
@@ -252,17 +250,13 @@ func TestSuiteConfig(testCfg *Cfg) server.CLIConfig {
 
 type TestSuite struct {
 	Ctx     context.Context
-	Log     log.Logger
+	Log     logging.Logger
 	Server  *server.Server
 	Metrics *metrics.EmulatedMetricer
 }
 
 func CreateTestSuite(testSuiteCfg server.CLIConfig) (TestSuite, func()) {
-	log := oplog.NewLogger(os.Stdout, oplog.CLIConfig{
-		Level:  log.LevelDebug,
-		Format: oplog.FormatLogFmt,
-		Color:  true,
-	}).New("role", svcName)
+	log := logging.NewTextSLogger(os.Stdout, &logging.SLoggerOptions{})
 
 	m := metrics.NewEmulatedMetricer()
 	ctx := context.Background()

flags/flags.go

@@ -2,20 +2,23 @@ package flags
 
 import (
 	"github.com/Layr-Labs/eigenda-proxy/flags/eigendaflags"
+	"github.com/Layr-Labs/eigenda-proxy/logging"
+	"github.com/Layr-Labs/eigenda-proxy/metrics"
 	"github.com/Layr-Labs/eigenda-proxy/store"
 	"github.com/Layr-Labs/eigenda-proxy/store/generated_key/memstore"
 	"github.com/Layr-Labs/eigenda-proxy/store/precomputed_key/redis"
 	"github.com/Layr-Labs/eigenda-proxy/store/precomputed_key/s3"
 	"github.com/Layr-Labs/eigenda-proxy/verify"
+
 	"github.com/urfave/cli/v2"
 
-	opservice "github.com/ethereum-optimism/optimism/op-service"
-	oplog "github.com/ethereum-optimism/optimism/op-service/log"
-	opmetrics "github.com/ethereum-optimism/optimism/op-service/metrics"
+	"github.com/Layr-Labs/eigenda-proxy/common"
 )
 
 const (
 	EigenDAClientCategory      = "EigenDA Client"
+	LoggingFlagsCategory       = "Logging"
+	MetricsFlagCategory        = "Metrics"
 	EigenDADeprecatedCategory  = "DEPRECATED EIGENDA CLIENT FLAGS -- THESE WILL BE REMOVED IN V2.0.0"
 	MemstoreFlagsCategory      = "Memstore (for testing purposes - replaces EigenDA backend)"
 	StorageFlagsCategory       = "Storage"
@@ -31,26 +34,20 @@ const (
 	PortFlagName       = "port"
 )
 
-const EnvVarPrefix = "EIGENDA_PROXY"
-
-func prefixEnvVars(name string) []string {
-	return opservice.PrefixEnvVar(EnvVarPrefix, name)
-}
-
 func CLIFlags() []cli.Flag {
 	// TODO: Decompose all flags into constituent parts based on their respective category / usage
 	flags := []cli.Flag{
 		&cli.StringFlag{
 			Name:    ListenAddrFlagName,
 			Usage:   "Server listening address",
 			Value:   "0.0.0.0",
-			EnvVars: prefixEnvVars("ADDR"),
+			EnvVars: common.PrefixEnvVar(common.GlobalPrefix, "ADDR"),
 		},
 		&cli.IntFlag{
 			Name:    PortFlagName,
 			Usage:   "Server listening port",
 			Value:   3100,
-			EnvVars: prefixEnvVars("PORT"),
+			EnvVars: common.PrefixEnvVar(common.GlobalPrefix, "PORT"),
 		},
 	}
 
@@ -62,15 +59,15 @@ var Flags = []cli.Flag{}
 
 func init() {
 	Flags = CLIFlags()
-	Flags = append(Flags, oplog.CLIFlags(EnvVarPrefix)...)
-	Flags = append(Flags, opmetrics.CLIFlags(EnvVarPrefix)...)
-	Flags = append(Flags, eigendaflags.CLIFlags(EnvVarPrefix, EigenDAClientCategory)...)
-	Flags = append(Flags, eigendaflags.DeprecatedCLIFlags(EnvVarPrefix, EigenDADeprecatedCategory)...)
-	Flags = append(Flags, store.CLIFlags(EnvVarPrefix, StorageFlagsCategory)...)
-	Flags = append(Flags, store.DeprecatedCLIFlags(EnvVarPrefix, StorageDeprecatedCategory)...)
-	Flags = append(Flags, redis.CLIFlags(EnvVarPrefix, RedisCategory)...)
-	Flags = append(Flags, s3.CLIFlags(EnvVarPrefix, S3Category)...)
-	Flags = append(Flags, memstore.CLIFlags(EnvVarPrefix, MemstoreFlagsCategory)...)
-	Flags = append(Flags, verify.CLIFlags(EnvVarPrefix, VerifierCategory)...)
-	Flags = append(Flags, verify.DeprecatedCLIFlags(EnvVarPrefix, VerifierDeprecatedCategory)...)
+	Flags = append(Flags, logging.CLIFlags(common.GlobalPrefix, LoggingFlagsCategory)...)
+	Flags = append(Flags, metrics.CLIFlags(common.GlobalPrefix, MetricsFlagCategory)...)
+	Flags = append(Flags, eigendaflags.CLIFlags(common.GlobalPrefix, EigenDAClientCategory)...)
+	Flags = append(Flags, eigendaflags.DeprecatedCLIFlags(common.GlobalPrefix, EigenDADeprecatedCategory)...)
+	Flags = append(Flags, store.CLIFlags(common.GlobalPrefix, StorageFlagsCategory)...)
+	Flags = append(Flags, store.DeprecatedCLIFlags(common.GlobalPrefix, StorageDeprecatedCategory)...)
+	Flags = append(Flags, redis.CLIFlags(common.GlobalPrefix, RedisCategory)...)
+	Flags = append(Flags, s3.CLIFlags(common.GlobalPrefix, S3Category)...)
+	Flags = append(Flags, memstore.CLIFlags(common.GlobalPrefix, MemstoreFlagsCategory)...)
+	Flags = append(Flags, verify.CLIFlags(common.GlobalPrefix, VerifierCategory)...)
+	Flags = append(Flags, verify.DeprecatedCLIFlags(common.GlobalPrefix, VerifierDeprecatedCategory)...)
 }