diff --git a/.github/tests/buildoor-instances-matrix.yaml b/.github/tests/buildoor-instances-matrix.yaml new file mode 100644 index 000000000..92e496996 --- /dev/null +++ b/.github/tests/buildoor-instances-matrix.yaml @@ -0,0 +1,33 @@ +# Multiple buildoor builders on top of a participants_matrix. The matrix expands +# to one participant per el x cl combination; buildoor_params.instances then wires +# a dedicated builder (by 1-based participant index) to chosen participants. The +# builder config stays independent of the participants. Builders onboard +# themselves after genesis via their lifecycle deposit (default on). +participants_matrix: + el: + - el_type: geth + el_image: ethpandaops/geth:glamsterdam-devnet-6 + cl: + - cl_type: lighthouse + cl_image: ethpandaops/lighthouse:glamsterdam-devnet-6 + - cl_type: lodestar + cl_image: ethpandaops/lodestar:glamsterdam-devnet-6 +network_params: + preset: minimal + gloas_fork_epoch: 6 + gas_limit: 150000000 +buildoor_params: + image: ethpandaops/buildoor:glamsterdam-devnet-6 + builder_api: true + epbs_builder: true + lifecycle: true + instances: + - participant: 1 + count: 2 + - participant: 2 + count: 1 +additional_services: + - buildoor + - dora +dora_params: + image: ethpandaops/dora:glamsterdam-devnet-6 diff --git a/.github/tests/buildoor-instances.yaml b/.github/tests/buildoor-instances.yaml new file mode 100644 index 000000000..4f15befe2 --- /dev/null +++ b/.github/tests/buildoor-instances.yaml @@ -0,0 +1,42 @@ +# Multiple dedicated per-participant buildoor builders via buildoor_params.instances. +# No mev_type: builders are configured independently of the participants and +# onboard themselves after genesis via their lifecycle deposit (default on), so +# gloas does not have to be at genesis. This spins up three buildoor services: +# buildoor-lighthouse-geth-1 (participant 1) +# buildoor-lodestar-geth-2-1 (participant 2, instance 1) +# buildoor-lodestar-geth-2-2 (participant 2, instance 2) +participants: + - el_type: geth + el_image: ethpandaops/geth:glamsterdam-devnet-6 + cl_type: lighthouse + cl_image: ethpandaops/lighthouse:glamsterdam-devnet-6 + - el_type: geth + el_image: ethpandaops/geth:glamsterdam-devnet-6 + cl_type: lodestar + cl_image: ethpandaops/lodestar:glamsterdam-devnet-6 +network_params: + preset: minimal + gloas_fork_epoch: 6 + gas_limit: 150000000 +buildoor_params: + image: ethpandaops/buildoor:glamsterdam-devnet-6 + builder_api: true + epbs_builder: true + lifecycle: true + instances: + - participant: 1 + count: 1 + - participant: 2 + count: 2 +additional_services: + - buildoor + - dora + - spamoor +spamoor_params: + image: ethpandaops/spamoor:master + spammers: + - scenario: eoatx + config: + throughput: 2 +dora_params: + image: ethpandaops/dora:glamsterdam-devnet-6 diff --git a/.github/workflows/check-consensus-spec-values.yml b/.github/workflows/check-consensus-spec-values.yml index f64312330..1432091e5 100644 --- a/.github/workflows/check-consensus-spec-values.yml +++ b/.github/workflows/check-consensus-spec-values.yml @@ -22,7 +22,7 @@ jobs: - preset: minimal args_file: .github/tests/minimal.yaml steps: - - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - uses: ./.github/actions/docker-login with: username: ethpandaops diff --git a/.github/workflows/check-typos.yml b/.github/workflows/check-typos.yml index 3804dde8a..d6fd9dc6e 100644 --- a/.github/workflows/check-typos.yml +++ b/.github/workflows/check-typos.yml @@ -10,7 +10,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout repository - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - name: Check for typos uses: crate-ci/typos@37bb98842b0d8c4ffebdb75301a13db0267cef89 # v1.47.2 diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 16a2b7dc0..0d927628e 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -11,7 +11,7 @@ jobs: outputs: matrix: ${{ steps.set-matrix.outputs.matrix }} steps: - - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - id: set-matrix # List all yaml files in the .github/tests directory, except for the k8s.yaml file run: echo "matrix=$(ls ./.github/tests/*.yaml | grep -vE 'k8s.yaml$' | jq -R -s -c 'split("\n")[:-1]')" >> $GITHUB_OUTPUT @@ -25,7 +25,7 @@ jobs: continue-on-error: true steps: - name: Checkout Repository - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - uses: ./.github/actions/docker-login with: username: ethpandaops @@ -64,7 +64,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout Repository - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - uses: ./.github/actions/docker-login with: username: ethpandaops diff --git a/.github/workflows/per-pr.yml b/.github/workflows/per-pr.yml index cbc87a76e..f104101a4 100644 --- a/.github/workflows/per-pr.yml +++ b/.github/workflows/per-pr.yml @@ -25,7 +25,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout Repository - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - uses: ./.github/actions/docker-login with: username: ethpandaops @@ -72,7 +72,7 @@ jobs: preset: minimal steps: - name: Checkout Repository - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: name: ${{ matrix.artifact }} @@ -94,7 +94,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout Repository - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - name: Setup Kurtosis uses: ./.github/actions/kurtosis-install - name: Kurtosis Lint @@ -105,7 +105,7 @@ jobs: timeout-minutes: 30 steps: - name: Checkout Repository - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - uses: ./.github/actions/docker-login with: username: ethpandaops @@ -122,7 +122,7 @@ jobs: # runs-on: ubuntu-latest # steps: # - name: Check out Repository - # uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + # uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 # - name: Setup Kurtosis # uses: ./.github/actions/kurtosis-install # - name: Run L1 diff --git a/.github/workflows/run-k8s.yml b/.github/workflows/run-k8s.yml index aceb22151..3e2496042 100644 --- a/.github/workflows/run-k8s.yml +++ b/.github/workflows/run-k8s.yml @@ -20,7 +20,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout Repository - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - name: Setup minikube id: minikube diff --git a/README.md b/README.md index 3e8504220..8e6941164 100644 --- a/README.md +++ b/README.md @@ -571,6 +571,33 @@ participants: # network parameter num_validator_keys_per_node validator_count: null + # Charon Distributed Validator Configuration + # The number of Charon nodes to create for distributed validation + # Each Charon node will run as a middleware between the beacon node and validator client + # Minimum 4 nodes recommended for fault tolerance + # Only used when vc_type is set to "charon" + # Defaults to 4 + charon_node_count: 4 + + # Charon-specific parameters for distributed validator setup + # Only used when vc_type is set to "charon" + charon_params: + # The type of validator client to run with Charon + # Valid values are: lighthouse, lodestar, teku, nimbus, prysm + # Each Charon node will run this validator client type + # Defaults to "lighthouse" + charon_vc: lighthouse + + # The Docker image for the validator client used with Charon + # This allows you to specify custom validator client images + # Defaults by client: + # - lighthouse: sigp/lighthouse:latest + # - lodestar: chainsafe/lodestar:latest + # - teku: consensys/teku:latest + # - nimbus: statusim/nimbus-validator-client:multiarch-latest + # - prysm: gcr.io/prysmaticlabs/prysm/validator:latest + charon_vc_image: sigp/lighthouse:latest + # Whether to use a remote signer instead of the vc directly handling keys # Note Lighthouse VC does not support this flag # Defaults to false @@ -656,6 +683,9 @@ participants: # Additional labels to be added. Default to empty labels: {} + # Buildoor (a self-contained block builder) is configured independently of the + # participants via `buildoor_params.instances` (see below), not per participant. + # Blobber can be enabled with the `blobber_enabled` flag per client or globally # Defaults to false blobber_enabled: false @@ -1508,6 +1538,8 @@ mempool_bridge_params: # Note: Helix uses TimescaleDB (PostgreSQL with time-series extension) for data storage # "buildoor" - a self-contained builder+relay service & mev-boost are spun up, powered by [buildoor](https://github.com/ethpandaops/buildoor) # Supports both legacy builder API and ePBS bidding. No separate relay infrastructure or builder participant needed. +# DEPRECATED: this single shared-builder mode will be dropped after the gloas fork. Use +# `buildoor_params.instances` for dedicated per-participant builders instead. # We have seen instances of multibuilder instances failing to start mev-relay-api with non zero epochs mev_type: null @@ -1577,7 +1609,10 @@ mev_params: # level = "debug" commit_boost_config: "" -# Parameters for the buildoor builder+relay service (used when mev_type is "buildoor") +# Parameters for the buildoor builder service. +# buildoor is an additional_service: add "buildoor" to additional_services to spin +# it up, then configure its targeting here. With "buildoor" enabled and no +# instances set, a single builder is wired to the first participant by default. buildoor_params: # The image to use for buildoor image: ethpandaops/buildoor:main @@ -1587,6 +1622,31 @@ buildoor_params: epbs_builder: true # Extra parameters to pass to the buildoor service extra_args: [] + # Enable buildoor's builder lifecycle: each builder deposits/onboards itself + # after genesis (and tops itself up) via the EL, so builders work even when + # gloas is not at genesis. Built blocks are tagged with the instance's service + # name in their extra-data so they can be traced back to the builder. + # Defaults to true + lifecycle: true + # Dedicated per-participant buildoor builders, configured independently of the + # participants (a builder is independent of the network: it reads one + # participant's CL payload_attributes stream and, under ePBS, gossips bids to + # the whole network). Each entry spins up `count` buildoor builder instances + # wired to the named participant's CL/EL. Services are named + # `buildoor---` (with a `-` suffix when count > 1). + # Requires "buildoor" in additional_services; no `mev_type` is needed, and it + # cannot be combined with the (deprecated) network-wide `mev_type: buildoor`. + # Each instance is its own builder; with lifecycle enabled (default) it onboards + # itself after genesis, so genesis builder registration is not required and gloas + # may activate at any epoch. + # Defaults to [] (no per-participant buildoors). + # Example: + # instances: + # - participant: 1 # 1-based participant index + # count: 1 + # - participant: 3 + # count: 2 + instances: [] # Enables Xatu Sentry for all participants # Defaults to false @@ -1759,7 +1819,7 @@ slashoor_params: # Ethereum genesis generator params ethereum_genesis_generator_params: # The image to use for ethereum genesis generator - image: ethpandaops/ethereum-genesis-generator:6.0.8 + image: ethpandaops/ethereum-genesis-generator:6.1.1 # Pass custom environment variables to the genesis generator (e.g. MY_VAR: my_value) extra_env: {} @@ -1971,14 +2031,44 @@ network_params:
- A 2-node Ethereum network with buildoor (self-contained builder+relay) + A 2-node Ethereum network with dedicated per-participant buildoor builders + +```yaml +participants: + - el_type: geth + cl_type: lighthouse + - el_type: reth + cl_type: prysm +buildoor_params: + builder_api: true + epbs_builder: true + # one buildoor wired to participant 1, two wired to participant 2 + instances: + - participant: 1 + count: 1 + - participant: 2 + count: 2 +additional_services: + - buildoor + - dora + - spamoor +``` + +
+ +
+ A 2-node Ethereum network with the (deprecated) shared buildoor builder ```yaml participants: - el_type: geth cl_type: lighthouse count: 2 +# Deprecated: prefer buildoor_params.instances (see example above). mev_type: buildoor +network_params: + builder_count: 1 + gloas_fork_epoch: 0 buildoor_params: builder_api: true epbs_builder: true @@ -2117,16 +2207,135 @@ participants: "/configs": "validator_config.json" # File available at: /configs/validator_config.json ``` +### Notes + +- All file paths must be relative to the package root directory +- Files outside the package directory cannot be mounted directly +- The entire directory structure is preserved when mounting directories + +## Charon Distributed Validator Technology (DVT) + +[Charon](https://github.com/ObolNetwork/charon) is a distributed validator middleware that enables fault-tolerant Ethereum validation by running validator duties across multiple nodes. This package supports deploying Charon clusters with any of the supported validator clients. + +### What is Distributed Validation? + +Distributed validation splits validator duties across multiple nodes (typically 4-7), providing: + +- **Fault Tolerance**: Continue validating even if some nodes go offline +- **Reduced Slashing Risk**: Consensus mechanisms prevent double-signing +- **Improved Uptime**: No single point of failure +- **Decentralization**: Distribute validator operations across multiple operators + +### Charon Configuration + +To use Charon distributed validators, set `vc_type: charon` in your participant configuration: + +```yaml +participants: + - el_type: geth + el_image: ethereum/client-go:latest + cl_type: lighthouse + cl_image: sigp/lighthouse:latest-unstable + use_separate_vc: true + # Charon Configuration + vc_type: charon + vc_image: obolnetwork/charon:latest + charon_node_count: 4 + charon_params: + charon_vc: teku + charon_vc_image: consensys/teku:latest +``` + +### Supported Validator Clients with Charon + +All major Ethereum validator clients are supported with Charon: + +| Validator Client | Status | Implementation | Notes | +| ---------------- | ------ | ------------------------ | ----------------------------- | +| **Lighthouse** | ✅ | Two-stage health checks | Default choice, most tested | +| **Lodestar** | ✅ | Script-based execution | Custom key management | +| **Teku** | ✅ | Config file approach | External signer mode | +| **Nimbus** | ✅ | Two-service architecture | Key import + validator client | +| **Prysm** | ✅ | Wallet-based import | Prysm wallet integration | + +### Example Configurations + +#### Basic Charon Setup (4 nodes with Lighthouse) + +```yaml +participants: + - vc_type: charon + vc_image: obolnetwork/charon:latest + charon_node_count: 4 + charon_params: + charon_vc: lighthouse + charon_vc_image: sigp/lighthouse:latest-unstable +``` + +#### Mixed Network (Charon + Standard Validators) + +```yaml +participants: + # Distributed validator with Charon + - vc_type: charon + charon_node_count: 4 + charon_params: + charon_vc: prysm + charon_vc_image: gcr.io/prysmaticlabs/prysm/validator:latest + # Standard validator + - vc_type: lighthouse + vc_image: sigp/lighthouse:latest-unstable +``` + +### Running Charon Networks + +```bash +# Create your network_params.yaml with Charon configuration +kurtosis run --enclave charon-testnet github.com/ethpandaops/ethereum-package --args-file network_params.yaml +``` + +### Running Charon Networks from source code + +```bash +# Create your network_params.yaml with Charon configuration +kurtosis run . --args-file network_params_charon_example.yaml +``` + + +### Charon Architecture + +When you deploy a Charon participant, the package creates: + +1. **Charon Cluster**: Multiple Charon nodes that form a distributed validator cluster +2. **Validator Clients**: Each Charon node connects to a validator client of your chosen type +3. **Key Distribution**: Validator keys are split and distributed across the Charon nodes +4. **Consensus Layer**: All nodes connect to the same beacon node for chain data + +``` +Beacon Node + ↓ +┌─────────────────────────────────────┐ +│ Charon Cluster │ +│ ┌─────────┐ ┌─────────┐ ┌─────────┐ │ +│ │Charon-1 │ │Charon-2 │ │Charon-3 │ │ +│ │ ↓ │ │ ↓ │ │ ↓ │ │ +│ │ Teku VC │ │ Teku VC │ │ Teku VC │ │ +│ └─────────┘ └─────────┘ └─────────┘ │ +└─────────────────────────────────────┘ +``` + ## Beacon Node <> Validator Client compatibility -| | Lighthouse VC | Prysm VC | Teku VC | Lodestar VC | Nimbus VC -|---------------|---------------|----------|---------|-------------|-----------| -| Lighthouse BN | ✅ | ✅ | ✅ | ✅ | ✅ -| Prysm BN | ✅ | ✅ | ✅ | ✅ | ✅ -| Teku BN | ✅ | ✅ | ✅ | ✅ | ✅ -| Lodestar BN | ✅ | ✅ | ✅ | ✅ | ✅ -| Nimbus BN | ✅ | ✅ | ✅ | ✅ | ✅ -| Grandine BN | ✅ | ✅ | ✅ | ✅ | ✅ +| | Lighthouse VC | Prysm VC | Teku VC | Lodestar VC | Nimbus VC | Charon DVT | +|---------------|---------------|----------|---------|-------------|-----------|------------| +| Lighthouse BN | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | +| Prysm BN | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | +| Teku BN | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | +| Lodestar BN | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | +| Nimbus BN | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | +| Grandine BN | ✅ | ✅ | ✅ | ✅ | ✅ | | + +**Note**: Charon DVT (Distributed Validator Technology) is compatible with all beacon node clients and can run any of the supported validator clients (Lighthouse, Lodestar, Teku, Nimbus, Prysm) in a distributed configuration. ## Custom labels for Docker and Kubernetes @@ -2193,7 +2402,7 @@ The package also supports other MEV implementations: - `"mev_type": "helix"` - Uses the high-performance [Helix relay](https://github.com/gattaca-com/helix) with TimescaleDB backend for data storage - `"mev_type": "mev-rs"` - Alternative relay implementation powered by [mev-rs](https://github.com/ralexstokes/mev-rs/) - `"mev_type": "commit-boost"` - Infrastructure powered by [commit-boost](https://github.com/Commit-Boost/commit-boost-client) -- `"mev_type": "buildoor"` - A self-contained builder+relay service powered by [buildoor](https://github.com/ethpandaops/buildoor). Supports both legacy builder API and ePBS bidding without requiring separate relay infrastructure or a dedicated builder participant. +- `"mev_type": "buildoor"` - A self-contained builder+relay service powered by [buildoor](https://github.com/ethpandaops/buildoor). Supports both legacy builder API and ePBS bidding without requiring separate relay infrastructure or a dedicated builder participant. **Deprecated:** this single shared-builder mode will be dropped after the gloas fork - use `buildoor_params.instances` for dedicated per-participant builders instead. Each implementation provides different features and performance characteristics suitable for various testing and development scenarios. diff --git a/kurtosis.yml b/kurtosis.yml index 6b37a1c12..01f4dbe0e 100644 --- a/kurtosis.yml +++ b/kurtosis.yml @@ -1 +1 @@ -name: "github.com/ethpandaops/ethereum-package" +name: "github.com/ObolNetwork/ethereum-package" diff --git a/main.star b/main.star index 201ee3caa..a94e2cf09 100644 --- a/main.star +++ b/main.star @@ -530,6 +530,7 @@ def run(plan, args={}): network_id, osaka_time, shadowfork_block_height, + charon_metrics_jobs, ) = participant_network.launch_participant_network( plan, args_with_right_defaults, @@ -549,6 +550,10 @@ def run(plan, args={}): detected_backend, ) + # Charon clusters expose extra per-node/per-VC metrics endpoints that aren't + # captured by the single vc_context per participant; register them directly. + prometheus_additional_metrics_jobs.extend(charon_metrics_jobs) + for p in all_participants: if p.el_context != None: plan.print( @@ -559,32 +564,23 @@ def run(plan, args={}): ) break - builder_bls_secret_key = None + total_validator_count = 0 + for participant in args_with_right_defaults.participants: + total_validator_count += participant.validator_count + if network_params.builder_count > 0: - total_validator_count = 0 - for participant in args_with_right_defaults.participants: - total_validator_count += participant.validator_count - builder_key_result = plan.run_sh( - name="derive-builder-bls-key", - description="Deriving builder BLS private key from mnemonic", - run='/app/ethdo account derive --mnemonic="{0}" --path="m/12381/3600/{1}/0/0" --show-private-key | grep "Private key" | sed "s/Private key: 0x//" | tr -d "\n"'.format( - network_params.preregistered_validator_keys_mnemonic, - total_validator_count, - ), - image="wealdtech/ethdo:latest", - tolerations=shared_utils.get_tolerations( - global_tolerations=global_tolerations - ), - node_selectors=global_node_selectors, - ) - builder_bls_secret_key = builder_key_result.output plan.print( "Builder configuration: {0} builder(s) registered at genesis with 0x03 credentials".format( network_params.builder_count ) ) - plan.print("Builder mnemonic: '{0}'".format(constants.DEFAULT_MNEMONIC)) - plan.print("Builder BLS private key: {0}".format(builder_bls_secret_key)) + plan.print( + "Builder mnemonic: '{0}', keys derived at indices {1}..{2}".format( + network_params.preregistered_validator_keys_mnemonic, + total_validator_count, + total_validator_count + network_params.builder_count - 1, + ) + ) all_el_contexts = [] all_cl_contexts = [] @@ -704,8 +700,10 @@ def run(plan, args={}): args_with_right_defaults.buildoor_params, global_node_selectors, global_tolerations, - builder_bls_secret_key, + network_params.preregistered_validator_keys_mnemonic, + total_validator_count, ranges, + constants.BUILDOOR_SERVICE_NAME, ) mev_endpoints.append(buildoor_endpoints["mev_endpoint"]) mev_endpoint_names.append(constants.BUILDOOR_MEV_TYPE) @@ -822,6 +820,85 @@ def run(plan, args={}): else: fail("Invalid MEV type") + # buildoor is an additional_service: launch the dedicated buildoor instances + # declared in buildoor_params.instances only when "buildoor" is in + # additional_services, each wired to the named participant's own CL/EL. + # Builders are configured independently of the participants. The CL builder + # endpoint and payload_attributes flags are already set in + # enrich_buildoor_per_participant. No global mev_type is required. Each builder + # derives its key from the network's validator mnemonic and onboards itself + # after genesis via its lifecycle deposit (so gloas need not be at genesis). + # Remove it from additional_services so the generic dispatch loop below (which + # fails on unknown services) skips it. + if constants.BUILDOOR_SERVICE_NAME in args_with_right_defaults.additional_services: + args_with_right_defaults.additional_services.remove( + constants.BUILDOOR_SERVICE_NAME + ) + buildoor_builder_index = 0 + for buildoor_instance in args_with_right_defaults.buildoor_params.instances: + index = buildoor_instance.participant - 1 + instance_count = buildoor_instance.count + participant = all_participants[index] + participant_config = args_with_right_defaults.participants[index] + index_str = shared_utils.zfill_custom( + index + 1, len(str(len(all_participants))) + ) + cl_context = participant.cl_context + el_context = participant.el_context + beacon_uri = "http://{0}:{1}".format( + cl_context.beacon_service_name, + cl_context.http_port, + ) + el_rpc_uri = "http://{0}:{1}".format( + el_context.dns_name, + el_context.rpc_port_num, + ) + engine_rpc_uri = "http://{0}:{1}".format( + el_context.dns_name, + el_context.engine_rpc_port_num, + ) + # Each instance uses a distinct prefunded account so concurrent buildoors + # do not collide on transaction nonces. + buildoor_account = prefunded_accounts[index % len(prefunded_accounts)] + for instance in range(instance_count): + # Name the instance after the participant it is wired to, e.g. + # buildoor-lighthouse-geth-1, matching the cl/el naming convention. + buildoor_service_name = shared_utils.get_buildoor_service_name( + constants.BUILDOOR_SERVICE_NAME, + participant_config.cl_type, + participant_config.el_type, + index_str, + instance, + instance_count, + ) + # Each instance is its own builder with its own builder BLS key, + # derived by buildoor from the builder mnemonic at consecutive indices + # after the validators and any genesis-registered builders, so they do + # not collide. The builder is onboarded after genesis via its lifecycle + # deposit (buildoor_params.lifecycle), not registered at genesis. + instance_builder_key_index = ( + total_validator_count + + network_params.builder_count + + buildoor_builder_index + ) + buildoor_builder_index += 1 + buildoor_endpoints = buildoor.launch_buildoor( + plan, + beacon_uri, + el_rpc_uri, + engine_rpc_uri, + jwt_file, + buildoor_account.private_key, + args_with_right_defaults.buildoor_params, + global_node_selectors, + global_tolerations, + network_params.preregistered_validator_keys_mnemonic, + instance_builder_key_index, + ranges, + buildoor_service_name, + ) + buildoor_api_urls.append(buildoor_endpoints["api_url"]) + # spin up the mev boost contexts if some endpoints for relays have been passed all_mevboost_contexts = [] if mev_endpoints: diff --git a/network_params.yaml b/network_params.yaml index e2b8539df..49bbb4037 100644 --- a/network_params.yaml +++ b/network_params.yaml @@ -269,7 +269,7 @@ keymanager_enabled: false checkpoint_sync_enabled: false checkpoint_sync_url: "" ethereum_genesis_generator_params: - image: ethpandaops/ethereum-genesis-generator:6.0.8 + image: ethpandaops/ethereum-genesis-generator:6.1.1 extra_env: {} port_publisher: nat_exit_ip: KURTOSIS_IP_ADDR_PLACEHOLDER diff --git a/network_params_charon_example.yaml b/network_params_charon_example.yaml new file mode 100644 index 000000000..4bfa83df0 --- /dev/null +++ b/network_params_charon_example.yaml @@ -0,0 +1,183 @@ +participants: + # EL + - el_type: geth + el_image: ethereum/client-go:latest + # CL + cl_type: lighthouse + cl_image: sigp/lighthouse:latest-unstable + supernode: false + use_separate_vc: true + # Validator + vc_type: charon + vc_image: obolnetwork/charon:latest + charon_node_count: 4 + charon_params: + charon_vc: teku + charon_vc_image: consensys/teku:latest + validator_count: null + use_remote_signer: false + - el_type: geth + el_image: ethereum/client-go:latest + # CL + cl_type: lighthouse + cl_image: sigp/lighthouse:latest-unstable + supernode: false + use_separate_vc: true + # Validator + vc_type: lighthouse + vc_image: sigp/lighthouse:latest-unstable + use_remote_signer: false + - el_type: geth + el_image: ethereum/client-go:latest + # CL + cl_type: lighthouse + cl_image: sigp/lighthouse:latest-unstable + supernode: false + use_separate_vc: true + # Validator + vc_type: lighthouse + vc_image: sigp/lighthouse:latest-unstable + use_remote_signer: false +network_params: + network: kurtosis + network_id: "3151908" + deposit_contract_address: "0x00000000219ab540356cBB839Cbe05303d7705Fa" + seconds_per_slot: 12 + num_validator_keys_per_node: 64 + preregistered_validator_keys_mnemonic: + "giant issue aisle success illegal bike spike + question tent bar rely arctic volcano long crawl hungry vocal artwork sniff fantasy + very lucky have athlete" + preregistered_validator_count: 0 + genesis_delay: 20 + genesis_gaslimit: 30000000 + max_per_epoch_activation_churn_limit: 8 + churn_limit_quotient: 65536 + ejection_balance: 16000000000 + eth1_follow_distance: 2048 + min_validator_withdrawability_delay: 256 + shard_committee_period: 256 + deneb_fork_epoch: 0 + electra_fork_epoch: 0 + fulu_fork_epoch: 18446744073709551615 + network_sync_base_url: https://snapshots.ethpandaops.io/ + force_snapshot_sync: false + samples_per_slot: 8 + custody_requirement: 4 + max_blobs_per_block_electra: 9 + target_blobs_per_block_electra: 6 + base_fee_update_fraction_electra: 5007716 + additional_preloaded_contracts: {} + devnet_repo: ethpandaops + prefunded_accounts: {} +additional_services: [] +dora_params: + image: "" +tx_fuzz_params: + tx_fuzz_extra_args: [] +spamoor_params: + min_cpu: 100 + max_cpu: 1000 + min_mem: 20 + max_mem: 300 + extra_args: [] +prometheus_params: + storage_tsdb_retention_time: "1d" + storage_tsdb_retention_size: "512MB" + min_cpu: 10 + max_cpu: 1000 + min_mem: 128 + max_mem: 2048 + # remote_write stays off until you set remote_write_url (and, if the endpoint + # needs auth, remote_write_token), e.g.: + # remote_write_url: "https://vm.monitoring.gcp.obol.tech/write" + # remote_write_token: "" + # The relabel rules below only ship Charon node/VC jobs and rewrite their job + # label to "charon" so Obol dashboards (which query job="charon") line up. + remote_write_relabel_configs: + - SourceLabels: ["job"] + Regex: ".*charon.*" + Action: keep + - SourceLabels: ["client_name"] + Regex: "charon" + TargetLabel: job + Replacement: charon + Action: replace +grafana_params: + additional_dashboards: [] + min_cpu: 10 + max_cpu: 1000 + min_mem: 128 + max_mem: 2048 +assertoor_params: + image: "" + run_stability_check: false + run_block_proposal_check: false + run_transaction_test: false + run_blob_transaction_test: false + run_opcodes_transaction_test: false + run_lifecycle_test: false + tests: [] +wait_for_finalization: false +global_log_level: info +snooper_enabled: false +ethereum_metrics_exporter_enabled: false +parallel_keystore_generation: false +disable_peer_scoring: false +persistent: false +mev_type: null +mev_params: + mev_relay_image: ethpandaops/mev-boost-relay:main + mev_builder_image: ethpandaops/flashbots-builder:main + mev_builder_cl_image: sigp/lighthouse:latest + mev_boost_image: ethpandaops/mev-boost:develop + mev_boost_args: ["mev-boost", "--relay-check"] + mev_relay_api_extra_args: [] + mev_relay_housekeeper_extra_args: [] + mev_relay_website_extra_args: [] + mev_builder_extra_args: [] + mev_builder_prometheus_config: + scrape_interval: 15s + labels: {} + custom_flood_params: + interval_between_transactions: 1 +xatu_sentry_enabled: false +xatu_sentry_params: + xatu_sentry_image: ethpandaops/xatu-sentry + xatu_server_addr: localhost:8000 + xatu_server_tls: false + xatu_server_headers: {} + beacon_subscriptions: + - attestation + - block + - chain_reorg + - finalized_checkpoint + - head + - voluntary_exit + - contribution_and_proof + - blob_sidecar +apache_port: 40000 +global_tolerations: [] +global_node_selectors: {} +keymanager_enabled: false +checkpoint_sync_enabled: false +checkpoint_sync_url: "" +ethereum_genesis_generator_params: + image: ethpandaops/ethereum-genesis-generator:4.0.4 +port_publisher: + nat_exit_ip: KURTOSIS_IP_ADDR_PLACEHOLDER + el: + enabled: false + public_port_start: 32000 + cl: + enabled: false + public_port_start: 33000 + vc: + enabled: false + public_port_start: 34000 + remote_signer: + enabled: false + public_port_start: 35000 + additional_services: + enabled: false + public_port_start: 36000 diff --git a/src/el/ethrex/ethrex_launcher.star b/src/el/ethrex/ethrex_launcher.star index 509dccbf5..c4bd4e1a2 100644 --- a/src/el/ethrex/ethrex_launcher.star +++ b/src/el/ethrex/ethrex_launcher.star @@ -174,6 +174,7 @@ def get_config( "--metrics", "--metrics.addr=0.0.0.0", "--metrics.port={0}".format(METRICS_PORT_NUM), + "--nat.extip=" + port_publisher.el_nat_exit_ip, ] # Handle bootnode configuration with bootnodoor_enode override if bootnodoor_enode != None: diff --git a/src/mev/buildoor/buildoor_launcher.star b/src/mev/buildoor/buildoor_launcher.star index 090bb3853..1a37bd50e 100644 --- a/src/mev/buildoor/buildoor_launcher.star +++ b/src/mev/buildoor/buildoor_launcher.star @@ -25,8 +25,11 @@ def launch_buildoor( buildoor_params, global_node_selectors, global_tolerations, - builder_bls_secret_key=None, + builder_mnemonic=None, + builder_key_index=None, validator_ranges_artifact=None, + service_name=BUILDOOR_SERVICE_NAME, + extra_data=None, ): tolerations = shared_utils.get_tolerations(global_tolerations=global_tolerations) @@ -35,11 +38,11 @@ def launch_buildoor( if wallet_key.startswith("0x"): wallet_key = wallet_key[2:] - # Use injected builder BLS key if provided, otherwise fall back to default - if builder_bls_secret_key != None: - builder_bls_key = builder_bls_secret_key - else: - builder_bls_key = constants.DEFAULT_MEV_SECRET_KEY[2:] + # The builder API URL buildoor advertises in its bids. With multiple + # instances each one must advertise its OWN service URL; otherwise every + # instance but one is rejected by consumers as a builder_url mismatch and + # never wins a bid. Computed once and reused as the registered api_url. + api_url = "http://{0}:{1}".format(service_name, BUILDOOR_API_PORT) cmd = [ "run", @@ -47,17 +50,38 @@ def launch_buildoor( "--el-rpc={0}".format(el_rpc_uri), "--el-engine-api={0}".format(engine_rpc_uri), "--el-jwt-secret=" + constants.JWT_MOUNT_PATH_ON_CONTAINER, - "--builder-privkey={0}".format(builder_bls_key), "--wallet-privkey={0}".format(wallet_key), "--api-port={0}".format(BUILDOOR_API_PORT), + "--builder-api-url={0}".format(api_url), ] + # Builder BLS key: let buildoor derive it from the mnemonic at the given + # index (matching the 0x03 builder keys registered at genesis) when provided, + # otherwise fall back to the default static secret key. + if builder_mnemonic != None: + cmd.append("--builder-mnemonic={0}".format(builder_mnemonic)) + cmd.append("--builder-key-index={0}".format(builder_key_index)) + else: + cmd.append("--builder-privkey={0}".format(constants.DEFAULT_MEV_SECRET_KEY[2:])) + + # Tag built blocks so a given block can be traced back to the buildoor + # instance that built it. Defaults to the service name (a unique identifier); + # buildoor injects it as the extra-data prefix (truncated to 32 bytes). + cmd.append( + "--extra-data={0}".format(extra_data if extra_data != None else service_name) + ) + if buildoor_params.builder_api: cmd.append("--builder-api-enabled") if buildoor_params.epbs_builder: cmd.append("--epbs-enabled") + # Lifecycle lets buildoor deposit/onboard its own builder after genesis (and + # top it up), so builders work without genesis registration / gloas-at-genesis. + if buildoor_params.lifecycle: + cmd.append("--lifecycle") + if validator_ranges_artifact != None: cmd.append( "--validator-ranges-file={0}/{1}".format( @@ -75,7 +99,7 @@ def launch_buildoor( files[VALIDATOR_RANGES_MOUNT_DIRPATH_ON_SERVICE] = validator_ranges_artifact buildoor_service = plan.add_service( - name=BUILDOOR_SERVICE_NAME, + name=service_name, config=ServiceConfig( image=buildoor_params.image, ports={ @@ -98,11 +122,8 @@ def launch_buildoor( return { "mev_endpoint": "http://{0}@{1}:{2}".format( constants.DEFAULT_MEV_PUBKEY, - BUILDOOR_SERVICE_NAME, - BUILDOOR_API_PORT, - ), - "api_url": "http://{0}:{1}".format( - BUILDOOR_SERVICE_NAME, + service_name, BUILDOOR_API_PORT, ), + "api_url": api_url, } diff --git a/src/package_io/constants.star b/src/package_io/constants.star index 71edfb5df..509d99d7c 100644 --- a/src/package_io/constants.star +++ b/src/package_io/constants.star @@ -31,6 +31,8 @@ VC_TYPE = struct( vero="vero", grandine="grandine", consensoor="consensoor", + charon="charon", + vouch="vouch", ) REMOTE_SIGNER_TYPE = struct(web3signer="web3signer") @@ -72,6 +74,7 @@ LITTLE_BIGTABLE_PORT_ID = "littlebigtable" VALIDATOR_GRPC_PORT_ID = "grpc" VALIDATING_REWARDS_ACCOUNT = "0x8943545177806ED17B9F23F0a21ee5948eCaa776" +CHARON_WITHDRAWAL_ADDRESS = "0xBc7c960C1097ef1Af0FD32407701465f3c03e407" MAX_ENR_ENTRIES = 20 MAX_ENODE_ENTRIES = 20 MIN_PEERS = 0 @@ -111,7 +114,7 @@ DEFAULT_ASSERTOOR_IMAGE = "ethpandaops/assertoor:latest" DEFAULT_SNOOPER_IMAGE = "ethpandaops/rpc-snooper:latest" DEFAULT_BOOTNODOOR_IMAGE = "ethpandaops/bootnodoor:latest" DEFAULT_ETHEREUM_GENESIS_GENERATOR_IMAGE = ( - "ethpandaops/ethereum-genesis-generator:6.0.8" + "ethpandaops/ethereum-genesis-generator:6.1.1" ) DEFAULT_YQ_IMAGE = "linuxserver/yq" DEFAULT_FLASHBOTS_RELAY_IMAGE = "ethpandaops/mev-boost-relay:main" diff --git a/src/package_io/input_parser.star b/src/package_io/input_parser.star index bd9a99fb8..2f27e6a3e 100644 --- a/src/package_io/input_parser.star +++ b/src/package_io/input_parser.star @@ -48,6 +48,7 @@ DEFAULT_VC_IMAGES = { "grandine": "sifrai/grandine:stable", "vero": "ghcr.io/serenita-org/vero:latest", "consensoor": "ethpandaops/consensoor:main", + "charon": "obolnetwork/charon:latest", } DEFAULT_VC_IMAGES_MINIMAL = { @@ -59,6 +60,7 @@ DEFAULT_VC_IMAGES_MINIMAL = { "grandine": "ethpandaops/grandine:develop-minimal", "vero": "ghcr.io/serenita-org/vero:latest", "consensoor": "ethpandaops/consensoor:main", + "charon": "obolnetwork/charon:latest", } DEFAULT_REMOTE_SIGNER_IMAGES = { @@ -281,6 +283,14 @@ def input_parser(plan, input_args): if result.get("disable_peer_scoring"): result = enrich_disable_peer_scoring(result) + if result.get("mev_type") == constants.BUILDOOR_MEV_TYPE: + plan.print( + "DEPRECATION WARNING: mev_type: buildoor (single shared builder) is " + + "deprecated and will be dropped after the gloas fork. Use the " + + "buildoor_params.instances config instead, which spins up dedicated " + + "buildoor builders wired to specific participants." + ) + if result.get("mev_type") in ( constants.MOCK_MEV_TYPE, constants.FLASHBOTS_MEV_TYPE, @@ -304,6 +314,67 @@ def input_parser(plan, input_args): ) ) + # Per-participant buildoor builders. buildoor is an additional_service: it is + # only spun up when "buildoor" is in additional_services. Its targeting is + # configured (independently of the participants) via buildoor_params.instances + # ([{participant, count}]); a builder is independent of the network - it reads + # one participant's CL/EL and, in ePBS, gossips bids to all. + buildoor_enabled = constants.BUILDOOR_SERVICE_NAME in result["additional_services"] + buildoor_instances = result["buildoor_params"]["instances"] + if buildoor_instances and not buildoor_enabled: + fail( + "buildoor_params.instances is set but 'buildoor' is not in additional_services. " + + "Add 'buildoor' to additional_services to spin it up." + ) + if buildoor_enabled: + if result.get("mev_type") != None: + fail( + "additional_services buildoor cannot be combined with a global mev_type ({0}). ".format( + result.get("mev_type") + ) + + "Use additional_services: [buildoor] with buildoor_params.instances for dedicated " + + "per-participant builders, or mev_type: buildoor for a single shared builder, but not both." + ) + # Default to a single builder on the first participant when none configured. + if not buildoor_instances: + buildoor_instances = [{"participant": 1, "count": 1}] + result["buildoor_params"]["instances"] = buildoor_instances + seen_buildoor_participants = {} + for instance in buildoor_instances: + participant_num = instance["participant"] + if ( + type(participant_num) != "int" + or participant_num < 1 + or participant_num > len(result["participants"]) + ): + fail( + "buildoor_params.instances participant must be a 1-based participant index between 1 and {0}, got {1}.".format( + len(result["participants"]), + participant_num, + ) + ) + if participant_num in seen_buildoor_participants: + fail( + "buildoor_params.instances has duplicate entries for participant {0}; use a single entry with the desired count.".format( + participant_num + ) + ) + seen_buildoor_participants[participant_num] = True + if type(instance["count"]) != "int" or instance["count"] < 1: + fail( + "buildoor_params.instances count for participant {0} must be an integer >= 1, got {1}.".format( + participant_num, + instance["count"], + ) + ) + # Each buildoor instance is its own builder, onboarded after genesis via + # its lifecycle deposit (buildoor_params.lifecycle) rather than registered + # at genesis. So no genesis builder registration is needed and gloas does + # not have to be at genesis - buildoor works with gloas at any epoch. Its + # builder keys are derived at indices after any genesis builders (see + # main.star) to avoid colliding with validator/genesis-builder keys. + result = enrich_buildoor_per_participant(result) + if ( result["mev_params"].get("mev_builder_subsidy") != 0 and result["network_params"].get("prefunded_accounts") == {} @@ -799,6 +870,9 @@ def input_parser(plan, input_args): vc_beacon_node_indices=participant["vc_beacon_node_indices"], checkpoint_sync_enabled=participant["checkpoint_sync_enabled"], skip_start=participant["skip_start"], + # Charon-specific parameters + charon_node_count=participant["charon_node_count"], + charon_params=participant["charon_params"], ) for participant in result["participants"] ], @@ -1025,6 +1099,11 @@ def input_parser(plan, input_args): min_mem=result["prometheus_params"]["min_mem"], max_mem=result["prometheus_params"]["max_mem"], image=result["prometheus_params"]["image"], + remote_write_url=result["prometheus_params"]["remote_write_url"], + remote_write_token=result["prometheus_params"]["remote_write_token"], + remote_write_relabel_configs=result["prometheus_params"][ + "remote_write_relabel_configs" + ], ), grafana_params=struct( additional_dashboards=result["grafana_params"]["additional_dashboards"], @@ -1203,6 +1282,14 @@ def input_parser(plan, input_args): extra_args=result["buildoor_params"]["extra_args"], builder_api=result["buildoor_params"]["builder_api"], epbs_builder=result["buildoor_params"]["epbs_builder"], + lifecycle=result["buildoor_params"]["lifecycle"], + instances=[ + struct( + participant=instance["participant"], + count=instance["count"], + ) + for instance in result["buildoor_params"]["instances"] + ], ), trueblocks_params=struct( image=result["trueblocks_params"]["image"], @@ -1702,7 +1789,9 @@ def parse_network_params(plan, input_args): ) ) builder_mnemonic_entry = { - "mnemonic": constants.DEFAULT_MNEMONIC, + "mnemonic": result["network_params"][ + "preregistered_validator_keys_mnemonic" + ], "start": actual_num_validators, "count": result["network_params"]["builder_count"], "wd_prefix": "0x03", @@ -2020,6 +2109,12 @@ def default_participant(): "vc_min_cpu": 0, "vc_max_cpu": 0, "vc_min_mem": 0, + # Charon-specific parameters + "charon_node_count": 4, + "charon_params": { + "charon_vc": "lighthouse", + "charon_vc_image": DEFAULT_CL_IMAGES[constants.CL_TYPE.lighthouse], + }, "vc_max_mem": 0, "vc_force_restart": False, "use_remote_signer": None, @@ -2242,6 +2337,15 @@ def get_default_prometheus_params(): "min_mem": 128, "max_mem": 2048, "image": "prom/prometheus:v3.2.1", + # remote_write: ship scraped metrics to an external endpoint. Disabled + # unless remote_write_url is set. remote_write_token is sent as a bearer + # credential (optional). remote_write_relabel_configs is an optional list + # of Prometheus write_relabel_configs (each a dict with SourceLabels and + # optionally Regex/Action/TargetLabel/Replacement) to filter/rewrite what + # is shipped. All are intended to be supplied at runtime via Kurtosis args. + "remote_write_url": "", + "remote_write_token": "", + "remote_write_relabel_configs": [], } @@ -2416,6 +2520,16 @@ def get_default_buildoor_params(): "extra_args": [], "builder_api": True, "epbs_builder": True, + # Enable buildoor's builder lifecycle (it deposits/onboards its own + # builder after genesis and tops it up), so builders work even when gloas + # is not at genesis. Requires the EL RPC + wallet key, both already wired. + "lifecycle": True, + # List of {participant: <1-based index>, count: } entries. Each entry + # spins up `count` dedicated buildoor builder instances wired to that + # participant's CL/EL. Builders are independent of the participants - a + # builder reads one CL's payload_attributes stream and (in ePBS) gossips + # bids to the whole network. Empty => no per-participant buildoors. + "instances": [], } @@ -2491,6 +2605,113 @@ def enrich_disable_peer_scoring(parsed_arguments_dict): return parsed_arguments_dict +# Wire the CL (and VC) of a single participant to an external builder/relay at +# mev_url. Shared by the global mev_type flow and the per-participant buildoor +# flow so both stay in sync as new clients are added. +def apply_external_builder_flags(participant, mev_url, gas_limit): + if participant["cl_type"] == "lighthouse": + participant["cl_extra_params"].append("--builder={0}".format(mev_url)) + if participant["vc_type"] == "lighthouse": + if ( + gas_limit == 0 + ): # if the gas limit is set we already enable builder-proposals + participant["vc_extra_params"].append("--builder-proposals") + if participant["cl_type"] == "lodestar": + participant["cl_extra_params"].append("--builder") + participant["cl_extra_params"].append("--builder.urls={0}".format(mev_url)) + if participant["vc_type"] == "lodestar": + participant["vc_extra_params"].append("--builder") + if participant["cl_type"] == "nimbus": + participant["cl_extra_params"].append("--payload-builder=true") + participant["cl_extra_params"].append( + "--payload-builder-url={0}".format(mev_url) + ) + if participant["vc_type"] == "nimbus": + participant["vc_extra_params"].append("--payload-builder=true") + if participant["cl_type"] == "teku": + participant["cl_extra_params"].append("--builder-endpoint={0}".format(mev_url)) + participant["cl_extra_params"].append( + "--validators-builder-registration-default-enabled=true" + ) + if participant["vc_type"] == "teku": + participant["vc_extra_params"].append( + "--validators-builder-registration-default-enabled=true" + ) + if participant["cl_type"] == "prysm": + participant["cl_extra_params"].append("--http-mev-relay={0}".format(mev_url)) + if participant["vc_type"] == "prysm": + participant["vc_extra_params"].append("--enable-builder") + if participant["cl_type"] == "grandine": + participant["cl_extra_params"].append("--builder-url={0}".format(mev_url)) + if participant["vc_type"] == "vero": + participant["vc_extra_params"].append("--use-external-builder") + + +# buildoor builds a payload from the CL's payload_attributes SSE stream, so the +# CL feeding a buildoor instance must emit them on every slot. +def apply_buildoor_payload_attributes_flags(participant): + if participant["cl_type"] == "lodestar": + participant["cl_extra_params"].append("--emitPayloadAttributes=true") + elif participant["cl_type"] == "prysm": + participant["cl_extra_params"].append("--prepare-all-payloads") + elif participant["cl_type"] == "lighthouse": + participant["cl_extra_params"].append("--always-prepare-payload") + elif participant["cl_type"] == "grandine": + participant["cl_extra_params"].append( + "--features=AlwaysPrepareExecutionPayload" + ) + elif participant["cl_type"] == "consensoor": + participant["cl_extra_params"].append("--emit-payload-attributes") + elif participant["cl_type"] == "teku": + participant["cl_extra_params"].append( + "--Xfork-choice-updated-always-send-payload-attributes=true" + ) + else: + # nimbus has no flag to emit payload_attributes. + fail( + "buildoor requires the CL feeding it to be one of " + + "[lodestar, prysm, lighthouse, grandine, consensoor, teku]: '{0}' has no flag to build a payload on each slot ".format( + participant["cl_type"] + ) + + "(emit payload_attributes for all slots), which buildoor needs to trigger block building." + ) + + +# Per-participant buildoor: each buildoor_params.instances entry spins up +# `count` dedicated buildoor services wired to the named participant's CL/EL. +# Builders are configured independently of the participants (a builder reads one +# CL's payload_attributes stream and, in ePBS, gossips bids to the whole +# network). This is independent of the global mev_type buildoor flow (a single +# shared buildoor for the whole network). +def enrich_buildoor_per_participant(parsed_arguments_dict): + participants = parsed_arguments_dict["participants"] + gas_limit = parsed_arguments_dict["network_params"]["gas_limit"] + num_participants = len(participants) + for instance in parsed_arguments_dict["buildoor_params"]["instances"]: + index = instance["participant"] - 1 + count = instance["count"] + participant = participants[index] + index_str = shared_utils.zfill_custom(index + 1, len(str(num_participants))) + # The CL has a single external-builder endpoint, so it is wired to the + # participant's first buildoor instance. Any additional instances still + # spin up (e.g. competing ePBS builders) but are not the CL's builder. + service_name = shared_utils.get_buildoor_service_name( + constants.BUILDOOR_SERVICE_NAME, + participant["cl_type"], + participant["el_type"], + index_str, + 0, + count, + ) + mev_url = "http://{0}:{1}".format( + service_name, + constants.BUILDOOR_API_PORT, + ) + apply_external_builder_flags(participant, mev_url, gas_limit) + apply_buildoor_payload_attributes_flags(participant) + return parsed_arguments_dict + + # TODO perhaps clean this up into a map def enrich_mev_extra_params(parsed_arguments_dict, mev_prefix, mev_port, mev_type): for index, participant in enumerate(parsed_arguments_dict["participants"]): @@ -2521,47 +2742,20 @@ def enrich_mev_extra_params(parsed_arguments_dict, mev_prefix, mev_port, mev_typ mev_port, ) - if participant["cl_type"] == "lighthouse": - participant["cl_extra_params"].append("--builder={0}".format(mev_url)) - if participant["vc_type"] == "lighthouse": - if ( - parsed_arguments_dict["network_params"]["gas_limit"] == 0 - ): # if the gas limit is set we already enable builder-proposals - participant["vc_extra_params"].append("--builder-proposals") - if participant["cl_type"] == "lodestar": - participant["cl_extra_params"].append("--builder") - participant["cl_extra_params"].append("--builder.urls={0}".format(mev_url)) - if participant["vc_type"] == "lodestar": - participant["vc_extra_params"].append("--builder") - if participant["cl_type"] == "nimbus": - participant["cl_extra_params"].append("--payload-builder=true") - participant["cl_extra_params"].append( - "--payload-builder-url={0}".format(mev_url) - ) - if participant["vc_type"] == "nimbus": - participant["vc_extra_params"].append("--payload-builder=true") - if participant["cl_type"] == "teku": - participant["cl_extra_params"].append( - "--builder-endpoint={0}".format(mev_url) - ) - participant["cl_extra_params"].append( - "--validators-builder-registration-default-enabled=true" - ) - if participant["vc_type"] == "teku": - participant["vc_extra_params"].append( - "--validators-builder-registration-default-enabled=true" - ) - if participant["cl_type"] == "prysm": - participant["cl_extra_params"].append( - "--http-mev-relay={0}".format(mev_url) - ) - if participant["vc_type"] == "prysm": - participant["vc_extra_params"].append("--enable-builder") - if participant["cl_type"] == "grandine": - participant["cl_extra_params"].append("--builder-url={0}".format(mev_url)) + apply_external_builder_flags( + participant, + mev_url, + parsed_arguments_dict["network_params"]["gas_limit"], + ) + + # buildoor builds against the first participant's payload_attributes + # SSE stream, so that CL must emit them on every slot. + if mev_type == constants.BUILDOOR_MEV_TYPE and index == 0: + apply_buildoor_payload_attributes_flags(participant) - if participant["vc_type"] == "vero": - participant["vc_extra_params"].append("--use-external-builder") + # Note: Charon enables the builder API via the CHARON_BUILDER_API env var + # set in charon_launcher.star, not through vc_extra_params (those flow into + # the inner validator client, where "--builder-api" is not a valid flag). num_participants = len(parsed_arguments_dict["participants"]) index_str = shared_utils.zfill_custom( diff --git a/src/package_io/sanity_check.star b/src/package_io/sanity_check.star index 547fbaf76..1a1172d44 100644 --- a/src/package_io/sanity_check.star +++ b/src/package_io/sanity_check.star @@ -55,6 +55,9 @@ PARTICIPANT_CATEGORIES = { "remote_signer_type", "remote_signer_image", "remote_signer_extra_env_vars", + # Charon-specific parameters + "charon_node_count", + "charon_params", "remote_signer_extra_labels", "remote_signer_extra_params", "remote_signer_tolerations", @@ -156,6 +159,9 @@ PARTICIPANT_MATRIX_PARAMS = { "vc_max_mem", "vc_force_restart", "validator_count", + # Charon-specific parameters + "charon_node_count", + "charon_params", ], "remote_signer": [ "remote_signer_type", @@ -333,6 +339,9 @@ SUBCATEGORY_PARAMS = { "storage_tsdb_retention_time", "storage_tsdb_retention_size", "image", + "remote_write_url", + "remote_write_token", + "remote_write_relabel_configs", ], "grafana_params": [ "additional_dashboards", @@ -468,6 +477,8 @@ SUBCATEGORY_PARAMS = { "extra_args", "builder_api", "epbs_builder", + "lifecycle", + "instances", ], "trueblocks_params": [ "image", @@ -510,6 +521,7 @@ ADDITIONAL_SERVICES_PARAMS = [ "zkboost", "trueblocks", "otel", + "buildoor", ] ADDITIONAL_CATEGORY_PARAMS = { diff --git a/src/participant_network.star b/src/participant_network.star index e47e41829..13de0d931 100644 --- a/src/participant_network.star +++ b/src/participant_network.star @@ -27,6 +27,7 @@ vc = import_module("./vc/vc_launcher.star") vc_shared = import_module("./vc/shared.star") vc_context_l = import_module("./vc/vc_context.star") node_metrics = import_module("./node_metrics_info.star") +charon_launcher = import_module("./vc/charon_launcher.star") remote_signer = import_module("./remote_signer/remote_signer_launcher.star") beacon_snooper = import_module("./snooper/snooper_beacon_launcher.star") @@ -345,6 +346,13 @@ def launch_participant_network( vc_service_configs = {} vc_service_info = {} + # Charon launches its own cluster of services immediately (rather than + # deferring to the parallel launch below), so its contexts are collected + # here keyed by participant index and merged in afterwards. It also produces + # extra Prometheus scrape jobs (one per Charon node and validator client) + # that are returned to the caller to register with Prometheus. + charon_vc_contexts = {} + charon_metrics_jobs = [] for index, participant in enumerate(args_with_right_defaults.participants): el_type = participant.el_type cl_type = participant.cl_type @@ -551,6 +559,36 @@ def launch_participant_network( remote_signer_context.metrics_info["config"] = participant.prometheus_config service_name = "vc-{0}".format(full_name) + + # Charon launches a full distributed-validator cluster (its own Charon + # nodes plus their validator clients) immediately and returns a ready + # vc_context, so it bypasses the deferred config / parallel-launch path + # used by the other validator clients. + if vc_type == constants.VC_TYPE.charon: + charon_vc_contexts[index], charon_jobs = charon_launcher.launch( + plan=plan, + launcher=charon_launcher.new_charon_launcher( + el_cl_genesis_data=el_cl_data + ), + keymanager_file=keymanager_file, + service_name=service_name, + image=participant.vc_image, + global_log_level=args_with_right_defaults.global_log_level, + cl_context=cl_context, + full_name=full_name, + node_keystore_files=vc_keystores, + participant=participant, + global_tolerations=global_tolerations, + node_selectors=node_selectors, + network_params=network_params, + port_publisher=args_with_right_defaults.port_publisher, + vc_index=current_vc_index, + genesis_timestamp=final_genesis_timestamp, + ) + charon_metrics_jobs.extend(charon_jobs) + current_vc_index += 1 + continue + vc_binary_artifact = binary_artifacts.get(index, {}).get("vc", None) vc_service_config = vc.get_vc_config( plan=plan, @@ -615,6 +653,14 @@ def launch_participant_network( vc_contexts_temp[participant_index] = vc_context + # Charon contexts were launched outside the parallel path above; fold them in. + for participant_index, vc_context in charon_vc_contexts.items(): + if vc_context and vc_context.metrics_info: + vc_context.metrics_info["config"] = args_with_right_defaults.participants[ + participant_index + ].prometheus_config + vc_contexts_temp[participant_index] = vc_context + # Convert to ordered list all_vc_contexts = [] for i in range(len(args_with_right_defaults.participants)): @@ -697,4 +743,5 @@ def launch_participant_network( network_id, el_cl_data.osaka_time, el_cl_data.shadowfork_block_height, + charon_metrics_jobs, ) diff --git a/src/prelaunch_data_generator/validator_keystores/keystore_files.star b/src/prelaunch_data_generator/validator_keystores/keystore_files.star index d10f359b4..967e12bf0 100644 --- a/src/prelaunch_data_generator/validator_keystores/keystore_files.star +++ b/src/prelaunch_data_generator/validator_keystores/keystore_files.star @@ -8,6 +8,7 @@ def new_keystore_files( prysm_relative_dirpath, teku_keys_relative_dirpath, teku_secrets_relative_dirpath, + charon_keys_relative_dirpath, ): return struct( files_artifact_uuid=files_artifact_uuid, @@ -19,4 +20,7 @@ def new_keystore_files( prysm_relative_dirpath=prysm_relative_dirpath, teku_keys_relative_dirpath=teku_keys_relative_dirpath, teku_secrets_relative_dirpath=teku_secrets_relative_dirpath, + # Flat keystore-N.json + keystore-N.txt pairs that Charon's + # `create cluster --split-keys-dir` consumes. Empty for non-Charon VCs. + charon_keys_relative_dirpath=charon_keys_relative_dirpath, ) diff --git a/src/prelaunch_data_generator/validator_keystores/validator_keystore_generator.star b/src/prelaunch_data_generator/validator_keystores/validator_keystore_generator.star index 8bc8d6d08..8838cb43b 100644 --- a/src/prelaunch_data_generator/validator_keystores/validator_keystore_generator.star +++ b/src/prelaunch_data_generator/validator_keystores/validator_keystore_generator.star @@ -17,6 +17,7 @@ SUCCESSFUL_EXEC_CMD_EXIT_CODE = 0 RAW_KEYS_DIRNAME = "keys" RAW_SECRETS_DIRNAME = "secrets" +CHARON_KEYS_DIRNAME = "charon-keys" NIMBUS_KEYS_DIRNAME = "nimbus-keys" PRYSM_DIRNAME = "prysm" @@ -53,6 +54,35 @@ def keystore_artifact_basename( ) +# Reshape the raw eth2-val-tools layout (keys//voting-keystore.json + +# secrets/) into the flat keystore-N.json + keystore-N.txt pairs that +# Charon's `create cluster --split-keys-dir` consumes, written to charon-keys/. +# Returned as a single shell command so it can be appended to the existing +# keystore-generation command (same pattern as the per-client chmod steps). +def charon_keystore_format_cmd(output_dirpath): + charon_dir = output_dirpath + CHARON_KEYS_DIRNAME + keys_dir = output_dirpath + RAW_KEYS_DIRNAME + secrets_dir = output_dirpath + RAW_SECRETS_DIRNAME + return ( + "mkdir -p " + + charon_dir + + " && i=0" + + " && for d in " + + keys_dir + + '/*/; do [ -d "$d" ] || continue;' + + ' cp "${d}voting-keystore.json" "' + + charon_dir + + '/keystore-${i}.json";' + + ' pubkey=$(basename "$d");' + + ' cp "' + + secrets_dir + + '/${pubkey}" "' + + charon_dir + + '/keystore-${i}.txt";' + + " i=$((i+1)); done" + ) + + # Launches a prelaunch data generator IMAGE, for use in various of the genesis generation def launch_prelaunch_data_generator( plan, @@ -139,6 +169,11 @@ def generate_validator_keystores(plan, mnemonic, participants, docker_cache_para running_total_validator_count += participant.validator_count + # Charon consumes a single dir of flat keystore-N.json + keystore-N.txt + # pairs; reshape the raw layout into charon-keys/. + if participant.vc_type == constants.VC_TYPE.charon: + all_sub_command_strs.append(charon_keystore_format_cmd(output_dirpath)) + command_str = " && ".join(all_sub_command_strs) command_result = plan.exec( @@ -169,6 +204,7 @@ def generate_validator_keystores(plan, mnemonic, participants, docker_cache_para keystore_start_index, keystore_stop_index - 1, ) + artifact_name = plan.store_service_files( service_name, output_dirpath, @@ -177,6 +213,11 @@ def generate_validator_keystores(plan, mnemonic, participants, docker_cache_para ) base_dirname_in_artifact = shared_utils.path_base(output_dirpath) + charon_keys_relative_dirpath = "" + if participant.vc_type == constants.VC_TYPE.charon: + charon_keys_relative_dirpath = shared_utils.path_join( + base_dirname_in_artifact, CHARON_KEYS_DIRNAME + ) to_add = keystore_files_module.new_keystore_files( artifact_name, shared_utils.path_join(base_dirname_in_artifact), @@ -186,6 +227,7 @@ def generate_validator_keystores(plan, mnemonic, participants, docker_cache_para shared_utils.path_join(base_dirname_in_artifact, PRYSM_DIRNAME), shared_utils.path_join(base_dirname_in_artifact, TEKU_KEYS_DIRNAME), shared_utils.path_join(base_dirname_in_artifact, TEKU_SECRETS_DIRNAME), + charon_keys_relative_dirpath, ) keystore_files.append(to_add) @@ -273,6 +315,14 @@ def generate_validator_keystores_in_parallel( ) generate_keystores_cmd += teku_permissions_cmd generate_keystores_cmd += raw_secret_permissions_cmd + + # Charon consumes a single dir of flat keystore-N.json + keystore-N.txt + # pairs; reshape the raw layout into charon-keys/. + if participant.vc_type == constants.VC_TYPE.charon: + generate_keystores_cmd += " && " + charon_keystore_format_cmd( + output_dirpath + ) + all_generation_commands.append(generate_keystores_cmd) all_output_dirpaths.append(output_dirpath) @@ -336,6 +386,11 @@ def generate_validator_keystores_in_parallel( # This is necessary because the way Kurtosis currently implements artifact-storing is base_dirname_in_artifact = shared_utils.path_base(output_dirpath) + charon_keys_relative_dirpath = "" + if participant.vc_type == constants.VC_TYPE.charon: + charon_keys_relative_dirpath = shared_utils.path_join( + base_dirname_in_artifact, CHARON_KEYS_DIRNAME + ) to_add = keystore_files_module.new_keystore_files( artifact_name, shared_utils.path_join(base_dirname_in_artifact), @@ -345,6 +400,7 @@ def generate_validator_keystores_in_parallel( shared_utils.path_join(base_dirname_in_artifact, PRYSM_DIRNAME), shared_utils.path_join(base_dirname_in_artifact, TEKU_KEYS_DIRNAME), shared_utils.path_join(base_dirname_in_artifact, TEKU_SECRETS_DIRNAME), + charon_keys_relative_dirpath, ) keystore_files.append(to_add) diff --git a/src/prometheus/prometheus_launcher.star b/src/prometheus/prometheus_launcher.star index 54572566a..a44693a62 100644 --- a/src/prometheus/prometheus_launcher.star +++ b/src/prometheus/prometheus_launcher.star @@ -1,5 +1,7 @@ shared_utils = import_module("../shared_utils/shared_utils.star") -prometheus = import_module("github.com/kurtosis-tech/prometheus-package/main.star") +prometheus = import_module( + "github.com/KaloyanTanev/prometheus-package/main.star@kalo/add-remote-write-kt-package-name" +) constants = import_module("../package_io/constants.star") EXECUTION_CLIENT_TYPE = "execution" @@ -48,7 +50,21 @@ def launch_prometheus( 0, ) - prometheus_url = prometheus.run( + # remote_write is enabled only when a URL is configured; empty => no + # remote_write block, identical to upstream behaviour. The relabel configs are + # passed through verbatim, so any filtering/rewriting (e.g. for a Charon + # cluster) is supplied via args rather than hardcoded here. + remote_write_configs = [] + if prometheus_params.remote_write_url != "": + remote_write_configs = [ + { + "Url": prometheus_params.remote_write_url, + "BearerToken": prometheus_params.remote_write_token, + "WriteRelabelConfigs": prometheus_params.remote_write_relabel_configs, + }, + ] + + return prometheus.run( plan, metrics_jobs, "prometheus", @@ -61,10 +77,9 @@ def launch_prometheus( storage_tsdb_retention_size=prometheus_params.storage_tsdb_retention_size, image=prometheus_params.image, public_ports=public_ports, + remote_write_configs=remote_write_configs, ) - return prometheus_url - def get_metrics_jobs( el_contexts, diff --git a/src/shared_utils/shared_utils.star b/src/shared_utils/shared_utils.star index c630a7716..80afa058b 100644 --- a/src/shared_utils/shared_utils.star +++ b/src/shared_utils/shared_utils.star @@ -100,6 +100,18 @@ def zfill_custom(value, width): return ("0" * (width - len(str(value)))) + str(value) +# Builds the service name for a per-participant buildoor instance, e.g. +# buildoor-lighthouse-geth-1. When a participant runs more than one buildoor +# (count > 1) a 1-based instance suffix is appended, e.g. +# buildoor-lighthouse-geth-1-2. Used by both the input parser (to wire the CL's +# builder endpoint) and main.star (to add the service), so they never drift. +def get_buildoor_service_name(prefix, cl_type, el_type, index_str, instance, count): + base = "{0}-{1}-{2}-{3}".format(prefix, cl_type, el_type, index_str) + if count <= 1: + return base + return "{0}-{1}".format(base, instance + 1) + + def label_maker( client, client_type, image, connected_client, extra_labels, supernode=False ): diff --git a/src/vc/charon_launcher.star b/src/vc/charon_launcher.star new file mode 100644 index 000000000..0e65cfff3 --- /dev/null +++ b/src/vc/charon_launcher.star @@ -0,0 +1,882 @@ +shared_utils = import_module("../shared_utils/shared_utils.star") +input_parser = import_module("../package_io/input_parser.star") +constants = import_module("../package_io/constants.star") +vc_shared = import_module("./shared.star") +vc_context = import_module("./vc_context.star") +node_metrics = import_module("../node_metrics_info.star") +prometheus = import_module("../prometheus/prometheus_launcher.star") +lighthouse = import_module("./lighthouse.star") +lodestar = import_module("./lodestar.star") +teku = import_module("./teku.star") +nimbus = import_module("./nimbus.star") +prysm = import_module("./prysm.star") +vouch = import_module("./vouch.star") +keystore_files_module = import_module( + "../prelaunch_data_generator/validator_keystores/keystore_files.star" +) + +# Charon specific ports +CHARON_VALIDATOR_API_PORT = 3600 +CHARON_P2P_TCP_PORT = 3610 +CHARON_MONITORING_PORT = 3620 +CHARON_RELAY_HTTP_PORT = 3640 + +# Fallback node count if the participant doesn't request a valid one. +DEFAULT_CHARON_NODE_COUNT = 4 + +# Official ethdo image, used to build the Vouch wallet from the split keystores +# (so the Vouch container itself needs no ethdo download). +ETHDO_IMAGE = "wealdtech/ethdo:latest" + +# Verbosity levels mapping +VERBOSITY_LEVELS = { + constants.GLOBAL_LOG_LEVEL.error: "error", + constants.GLOBAL_LOG_LEVEL.warn: "warn", + constants.GLOBAL_LOG_LEVEL.info: "info", + constants.GLOBAL_LOG_LEVEL.debug: "debug", +} + + +def launch( + plan, + launcher, + keymanager_file, + service_name, + image, + global_log_level, + cl_context, + full_name, + node_keystore_files, + participant, + global_tolerations, + node_selectors, + network_params, + port_publisher, + vc_index, + genesis_timestamp, +): + """ + Launch a Charon distributed validator client + """ + if node_keystore_files == None: + return None, [] + + tolerations = shared_utils.get_tolerations( + specific_container_tolerations=participant.vc_tolerations, + participant_tolerations=participant.tolerations, + global_tolerations=global_tolerations, + ) + + log_level = input_parser.get_client_log_level_or_default( + participant.vc_log_level, global_log_level, VERBOSITY_LEVELS + ) + + # Number of Charon nodes to create. + charon_node_count = participant.charon_node_count + if charon_node_count <= 0: + charon_node_count = DEFAULT_CHARON_NODE_COUNT + + # Validator client type/image to run behind each Charon node. + vc_type = constants.CL_TYPE.lighthouse + vc_image = input_parser.DEFAULT_CL_IMAGES[constants.CL_TYPE.lighthouse] + if participant.charon_params != None: + vc_type = participant.charon_params.get("charon_vc", vc_type) + vc_image = participant.charon_params.get("charon_vc_image", vc_image) + + # All Charon nodes connect to the same beacon node. + beacon_endpoint = cl_context.beacon_http_url + + # The genesis timestamp is already known from genesis generation, so use it + # directly rather than querying the (possibly not-yet-ready) beacon node. + genesis_time = genesis_timestamp + + charon_service_name = service_name + "-charon-split-keys-" + str(vc_index) + CHARON_DATA_DIRPATH_ON_CLIENT_CONTAINER = "/opt/charon/" + persistent_key = "data-{0}".format(charon_service_name) + + charon_keys_dirpath = shared_utils.path_join( + constants.VALIDATOR_KEYS_DIRPATH_ON_SERVICE_CONTAINER, + node_keystore_files.charon_keys_relative_dirpath, + ) + + files = { + CHARON_DATA_DIRPATH_ON_CLIENT_CONTAINER: Directory( + persistent_key=persistent_key, + ), + constants.VALIDATOR_KEYS_DIRPATH_ON_SERVICE_CONTAINER: node_keystore_files.files_artifact_uuid, + } + + # Run the Charon cluster creation (splits the existing keys across nodes). + plan.add_service( + name=charon_service_name, + config=ServiceConfig( + image=image, + cmd=[ + "create", + "cluster", + # cluster_name label shown in Charon dashboards; mirror the + # docker-compose convention "kurtosis--". + "--name=kurtosis-" + cl_context.client_name + "-" + vc_type, + "--nodes=" + str(charon_node_count), + "--fee-recipient-addresses=" + constants.VALIDATING_REWARDS_ACCOUNT, + "--withdrawal-addresses=" + constants.CHARON_WITHDRAWAL_ADDRESS, + "--split-existing-keys", + "--split-keys-dir=" + charon_keys_dirpath, + "--testnet-chain-id=" + network_params.network_id, + "--testnet-fork-version=" + constants.GENESIS_FORK_VERSION, + "--testnet-genesis-timestamp=" + str(genesis_time), + "--testnet-name=kurtosis", + "--cluster-dir=" + CHARON_DATA_DIRPATH_ON_CLIENT_CONTAINER, + ], + files=files, + user=User(uid=0, gid=0), + ), + ) + + # Keep a busybox service running on the cluster volume so we can read the + # generated per-node files back out as artifacts. + cluster_files_service = plan.add_service( + name=charon_service_name + "-keep-running", + config=ServiceConfig( + image="busybox:latest", + cmd=["tail", "-f", "/dev/null"], # Keep the service running + files=files, + user=User(uid=0, gid=0), + ), + ) + + # Wait a moment for files to be fully written + plan.exec( + service_name=cluster_files_service.name, + recipe=ExecRecipe( + command=["sleep", "5"], + ), + ) + + # Spin up a local Charon relay so the nodes can discover each other within + # the enclave instead of depending on the public Obol relay network. + relay_service = plan.add_service( + name=service_name + "-charon-relay-" + str(vc_index), + config=ServiceConfig( + image=image, + cmd=[ + "relay", + "--data-dir=/opt/charon", + "--http-address=0.0.0.0:" + str(CHARON_RELAY_HTTP_PORT), + "--p2p-tcp-address=0.0.0.0:" + str(CHARON_P2P_TCP_PORT), + "--monitoring-address=0.0.0.0:" + str(CHARON_MONITORING_PORT), + # The relay lives on a private Kurtosis network; without this it + # advertises no addresses in its ENR and nodes fail to resolve it + # ("timeout resolving bootnode ENR"). + "--p2p-advertise-private-addresses=true", + ], + ports={ + "relay-http": PortSpec( + number=CHARON_RELAY_HTTP_PORT, + transport_protocol="TCP", + application_protocol="http", + ), + "p2p-tcp": PortSpec( + number=CHARON_P2P_TCP_PORT, + transport_protocol="TCP", + ), + }, + user=User(uid=0, gid=0), + ), + ) + charon_relay_url = "http://{0}:{1}".format( + relay_service.ip_address, CHARON_RELAY_HTTP_PORT + ) + + # Launch Charon nodes + charon_services = [] + for i in range(charon_node_count): + node_name = service_name + "-charon-" + str(i) + + env_vars = { + "CHARON_LOG_LEVEL": log_level, + "CHARON_LOG_FORMAT": "console", + "CHARON_P2P_RELAYS": charon_relay_url, + "CHARON_BUILDER_API": "true", + "CHARON_VALIDATOR_API_ADDRESS": "0.0.0.0:" + str(CHARON_VALIDATOR_API_PORT), + "CHARON_P2P_TCP_ADDRESS": "0.0.0.0:" + str(CHARON_P2P_TCP_PORT), + "CHARON_MONITORING_ADDRESS": "0.0.0.0:" + str(CHARON_MONITORING_PORT), + "CHARON_PRIVATE_KEY_FILE": "/opt/charon/.charon/node" + + str(i) + + "/charon-enr-private-key", + "CHARON_LOCK_FILE": "/opt/charon/.charon/node" + + str(i) + + "/cluster-lock.json", + "CHARON_JAEGER_SERVICE": "node" + str(i), + "CHARON_P2P_EXTERNAL_HOSTNAME": "node" + str(i), + "CHARON_BEACON_NODE_ENDPOINTS": beacon_endpoint, + "CHARON_TESTNET_CHAIN_ID": network_params.network_id, + "CHARON_TESTNET_FORK_VERSION": constants.GENESIS_FORK_VERSION, + "CHARON_TESTNET_GENESIS_TIMESTAMP": str(genesis_time), + "CHARON_TESTNET_NAME": "kurtosis", + } + + # Add any extra environment variables + if participant.vc_extra_env_vars: + env_vars.update(participant.vc_extra_env_vars) + + # Ports configuration + ports = { + "validator-api": PortSpec( + number=CHARON_VALIDATOR_API_PORT, + transport_protocol="TCP", + application_protocol="http", + ), + "p2p-tcp": PortSpec( + number=CHARON_P2P_TCP_PORT, + transport_protocol="TCP", + ), + "monitoring": PortSpec( + number=CHARON_MONITORING_PORT, + transport_protocol="TCP", + application_protocol="http", + ), + } + + # Charon run command + cmd = [ + "run", + "--testnet-chain-id=" + network_params.network_id, + "--testnet-fork-version=" + constants.GENESIS_FORK_VERSION, + "--testnet-genesis-timestamp=" + str(genesis_time), + "--testnet-name=kurtosis", + ] + + # Add the service + charon_service = plan.add_service( + name=node_name, + config=ServiceConfig( + image=image, + ports=ports, + cmd=cmd, + env_vars=env_vars, + labels=shared_utils.label_maker( + client=constants.VC_TYPE.charon, + client_type=constants.CLIENT_TYPES.validator, + image=image[-constants.MAX_LABEL_LENGTH :], + connected_client=cl_context.client_name, + extra_labels=participant.vc_extra_labels, + supernode=participant.supernode, + ), + tolerations=tolerations, + node_selectors=node_selectors, + files={ + "/opt/charon/.charon/": Directory(persistent_key=persistent_key), + }, + user=User(uid=0, gid=0), + ), + ) + charon_services.append(charon_service) + + vc_launchers = { + constants.VC_TYPE.lighthouse: launch_lighthouse, + constants.VC_TYPE.lodestar: launch_lodestar, + constants.VC_TYPE.teku: launch_teku, + constants.VC_TYPE.nimbus: launch_nimbus, + constants.VC_TYPE.prysm: launch_prysm, + constants.VC_TYPE.vouch: launch_vouch, + } + if vc_type not in vc_launchers: + fail( + "Unsupported Charon validator client '{0}'. Supported clients: {1}".format( + vc_type, ", ".join(vc_launchers.keys()) + ) + ) + + # Launch one validator client per Charon node, connected to that node's validator API. + vc_services = [] + for i in range(charon_node_count): + charon_validator_api_url = "http://{0}:{1}".format( + charon_services[i].ip_address, CHARON_VALIDATOR_API_PORT + ) + vc_service_name = service_name + "-vc-" + str(i) + "-" + vc_type + + # Each node's validator keys come from the cluster-creation output. + validator_keys_for_node = plan.store_service_files( + service_name=cluster_files_service.name, + src=CHARON_DATA_DIRPATH_ON_CLIENT_CONTAINER + + "/node" + + str(i) + + "/validator_keys", + name="validator-keys-node-" + str(i) + "-" + str(vc_index), + ) + + vc_services.append( + vc_launchers[vc_type]( + plan=plan, + vc_service_name=vc_service_name, + charon_validator_api_url=charon_validator_api_url, + split_keys_artifact=validator_keys_for_node, + launcher=launcher, + keymanager_file=keymanager_file, + participant=participant, + global_log_level=global_log_level, + cl_context=cl_context, + tolerations=tolerations, + node_selectors=node_selectors, + network_params=network_params, + port_publisher=port_publisher, + full_name=full_name + "-node" + str(i), + vc_index=vc_index, + node_index=i, + vc_image=vc_image, + ) + ) + + # The cluster files have all been extracted as artifacts; drop the busybox + # helper that was only kept alive (tail -f) to read them. + plan.remove_service(name=cluster_files_service.name) + + # Node 0 is surfaced as the participant's primary vc_context (below). Register + # every other Charon node and all validator clients as additional Prometheus + # scrape jobs so the whole cluster is monitored, not just node 0. + metrics_jobs = [] + for i in range(charon_node_count): + if i != 0: + charon_service = charon_services[i] + metrics_jobs.append( + prometheus.new_metrics_job( + job_name=charon_service.name, + endpoint="{0}:{1}".format( + charon_service.ip_address, CHARON_MONITORING_PORT + ), + metrics_path=vc_shared.METRICS_PATH, + labels={ + "service": charon_service.name, + "client_type": constants.CLIENT_TYPES.validator, + "client_name": constants.VC_TYPE.charon, + }, + ) + ) + vc_service = vc_services[i] + vc_metrics_port = vc_service.ports[constants.METRICS_PORT_ID] + metrics_jobs.append( + prometheus.new_metrics_job( + job_name=vc_service.name, + endpoint="{0}:{1}".format( + vc_service.ip_address, vc_metrics_port.number + ), + metrics_path=vc_shared.METRICS_PATH, + labels={ + "service": vc_service.name, + "client_type": constants.CLIENT_TYPES.validator, + "client_name": vc_type, + }, + ) + ) + + # Surface Charon node 0 as the participant's primary vc_context. + validator_metrics_port = charon_services[0].ports["monitoring"] + validator_metrics_url = "{0}:{1}".format( + charon_services[0].ip_address, validator_metrics_port.number + ) + validator_node_metrics_info = node_metrics.new_node_metrics_info( + charon_services[0].name, vc_shared.METRICS_PATH, validator_metrics_url + ) + + return ( + vc_context.new_vc_context( + client_name=constants.VC_TYPE.charon, + service_name=charon_services[0].name, + metrics_info=validator_node_metrics_info, + ), + metrics_jobs, + ) + + +def _charon_split_keys_to_keystore_files( + plan, split_keys_artifact, vc_index, node_index +): + """ + Convert a Charon node's split keystores (a flat dir of keystore-N.json + + keystore-N.txt) into the on-disk layouts the stock vc/.star launchers + consume, and wrap them in a node_keystore_files struct. + + Produces, in the returned artifact: + keys//voting-keystore.json + secrets/ (lighthouse, lodestar) + nimbus-keys//keystore.json (nimbus; reuses secrets/) + teku-keys/.json + teku-secrets/.txt (teku) + matching the eth2-val-tools layouts, so no per-client import step is needed. + (prysm and vouch instead need a wallet, built separately in + _charon_split_keys_to_prysm_wallet / _charon_split_keys_to_vouch_wallet.) + """ + converter_name = "charon-keys-convert-" + str(node_index) + "-" + str(vc_index) + converter = plan.add_service( + name=converter_name, + config=ServiceConfig( + image="busybox:latest", + cmd=["tail", "-f", "/dev/null"], + files={"/split-keys": split_keys_artifact}, + ), + ) + + # Reorganise each keystore-N.json/.txt pair into the on-disk layouts the stock + # launchers expect: raw (lighthouse/lodestar), nimbus-keys (nimbus), and + # teku-keys/teku-secrets (teku). Prysm needs a wallet, handled separately. + convert_script = """#!/bin/sh +set -e +mkdir -p /out/keys /out/secrets /out/nimbus-keys /out/teku-keys /out/teku-secrets +for f in /split-keys/keystore-*.json; do + [ -f "$f" ] || continue + pubkey="0x$(grep '"pubkey"' "$f" | head -1 | awk -F'"' '{print $4}')" + pw="${f%.json}.txt" + + # raw layout (lighthouse, lodestar): /voting-keystore.json + secrets/ + mkdir -p "/out/keys/${pubkey}" + cp "$f" "/out/keys/${pubkey}/voting-keystore.json" + cp "$pw" "/out/secrets/${pubkey}" + + # nimbus layout: /keystore.json (secrets reuse the raw secrets dir) + mkdir -p "/out/nimbus-keys/${pubkey}" + cp "$f" "/out/nimbus-keys/${pubkey}/keystore.json" + + # teku layout: flat .json + .txt + cp "$f" "/out/teku-keys/${pubkey}.json" + cp "$pw" "/out/teku-secrets/${pubkey}.txt" +done +# charon writes password files as 0400/root; make them readable by non-root VC +# images (teku, lodestar), and make dirs writable so teku can create its .lock files +chmod -R a+rwX /out +""" + plan.exec( + service_name=converter.name, + recipe=ExecRecipe(command=["sh", "-c", convert_script]), + ) + + keystore_artifact = plan.store_service_files( + service_name=converter.name, + src="/out", + name="charon-raw-keys-" + str(node_index) + "-" + str(vc_index), + ) + plan.remove_service(name=converter.name) + + return keystore_files_module.new_keystore_files( + files_artifact_uuid=keystore_artifact, + raw_root_dirpath="", + raw_keys_relative_dirpath="keys", + raw_secrets_relative_dirpath="secrets", + nimbus_keys_relative_dirpath="nimbus-keys", + prysm_relative_dirpath="", + teku_keys_relative_dirpath="teku-keys", + teku_secrets_relative_dirpath="teku-secrets", + charon_keys_relative_dirpath="", + ) + + +def launch_lighthouse( + plan, + vc_service_name, + charon_validator_api_url, + split_keys_artifact, + launcher, + keymanager_file, + participant, + global_log_level, + cl_context, + tolerations, + node_selectors, + network_params, + port_publisher, + full_name, + vc_index, + node_index, + vc_image, +): + node_keystore_files = _charon_split_keys_to_keystore_files( + plan, split_keys_artifact, vc_index, node_index + ) + + config = lighthouse.get_config( + plan=plan, + participant=participant, + el_cl_genesis_data=launcher.el_cl_genesis_data, + image=vc_image, + service_name=vc_service_name, + global_log_level=global_log_level, + beacon_http_urls=[charon_validator_api_url], + cl_context=cl_context, + el_context=None, # unused by lighthouse.get_config + full_name=full_name, + node_keystore_files=node_keystore_files, + tolerations=tolerations, + node_selectors=node_selectors, + keymanager_enabled=False, + network_params=network_params, + port_publisher=port_publisher, + vc_index=vc_index, + extra_files_artifacts=[], + distributed=True, + ) + + return plan.add_service(name=vc_service_name, config=config) + + +def launch_lodestar( + plan, + vc_service_name, + charon_validator_api_url, + split_keys_artifact, + launcher, + keymanager_file, + participant, + global_log_level, + cl_context, + tolerations, + node_selectors, + network_params, + port_publisher, + full_name, + vc_index, + node_index, + vc_image, +): + node_keystore_files = _charon_split_keys_to_keystore_files( + plan, split_keys_artifact, vc_index, node_index + ) + + config = lodestar.get_config( + plan=plan, + participant=participant, + el_cl_genesis_data=launcher.el_cl_genesis_data, + keymanager_file=keymanager_file, + image=vc_image, + global_log_level=global_log_level, + beacon_http_urls=[charon_validator_api_url], + cl_context=cl_context, + el_context=None, + remote_signer_context=None, + full_name=full_name, + node_keystore_files=node_keystore_files, + tolerations=tolerations, + node_selectors=node_selectors, + keymanager_enabled=False, + network_params=network_params, + port_publisher=port_publisher, + vc_index=vc_index, + extra_files_artifacts=[], + distributed=True, + ) + + return plan.add_service(name=vc_service_name, config=config) + + +def launch_teku( + plan, + vc_service_name, + charon_validator_api_url, + split_keys_artifact, + launcher, + keymanager_file, + participant, + global_log_level, + cl_context, + tolerations, + node_selectors, + network_params, + port_publisher, + full_name, + vc_index, + node_index, + vc_image, +): + node_keystore_files = _charon_split_keys_to_keystore_files( + plan, split_keys_artifact, vc_index, node_index + ) + + config = teku.get_config( + plan=plan, + participant=participant, + el_cl_genesis_data=launcher.el_cl_genesis_data, + keymanager_file=keymanager_file, + image=vc_image, + beacon_http_urls=[charon_validator_api_url], + cl_context=cl_context, + el_context=None, + remote_signer_context=None, + full_name=full_name, + node_keystore_files=node_keystore_files, + tolerations=tolerations, + node_selectors=node_selectors, + keymanager_enabled=False, + network_params=network_params, + port_publisher=port_publisher, + vc_index=vc_index, + extra_files_artifacts=[], + distributed=True, + ) + + return plan.add_service(name=vc_service_name, config=config) + + +def launch_nimbus( + plan, + vc_service_name, + charon_validator_api_url, + split_keys_artifact, + launcher, + keymanager_file, + participant, + global_log_level, + cl_context, + tolerations, + node_selectors, + network_params, + port_publisher, + full_name, + vc_index, + node_index, + vc_image, +): + node_keystore_files = _charon_split_keys_to_keystore_files( + plan, split_keys_artifact, vc_index, node_index + ) + + config = nimbus.get_config( + plan=plan, + participant=participant, + el_cl_genesis_data=launcher.el_cl_genesis_data, + image=vc_image, + keymanager_file=keymanager_file, + beacon_http_urls=[charon_validator_api_url], + cl_context=cl_context, + el_context=None, + remote_signer_context=None, + full_name=full_name, + node_keystore_files=node_keystore_files, + tolerations=tolerations, + node_selectors=node_selectors, + keymanager_enabled=False, + network_params=network_params, + port_publisher=port_publisher, + vc_index=vc_index, + extra_files_artifacts=[], + distributed=True, + ) + + return plan.add_service(name=vc_service_name, config=config) + + +def _charon_split_keys_to_prysm_wallet( + plan, split_keys_artifact, vc_index, node_index, prysm_image +): + """ + Build a Prysm wallet from a Charon node's split keystores so the stock + vc/prysm.star launcher (which expects a wallet, not raw keystores) can run it. + + Returns an artifact containing: + prysm/ the imported direct-keymanager wallet + wallet-password.txt the wallet password + plus the node_keystore_files struct that points at them. + """ + builder_name = "charon-prysm-wallet-" + str(node_index) + "-" + str(vc_index) + builder = plan.add_service( + name=builder_name, + config=ServiceConfig( + image=prysm_image, + entrypoint=["bash", "-c"], + cmd=["tail -f /dev/null"], + files={"/split-keys": split_keys_artifact}, + user=User(uid=0, gid=0), + ), + ) + + build_script = """#!/usr/bin/env bash +set -e +mkdir -p /out +echo "prysm-validator-secret" > /out/wallet-password.txt +/app/cmd/validator/validator wallet create \\ + --accept-terms-of-use \\ + --keymanager-kind=direct \\ + --wallet-dir=/out/prysm \\ + --wallet-password-file=/out/wallet-password.txt +tmpkeys=/tmp/keys +mkdir -p "$tmpkeys" +for f in /split-keys/keystore-*.json; do + [ -f "$f" ] || continue + cp "$f" "$tmpkeys/" + /app/cmd/validator/validator accounts import \\ + --accept-terms-of-use=true \\ + --wallet-dir=/out/prysm \\ + --keys-dir="$tmpkeys" \\ + --account-password-file="${f%.json}.txt" \\ + --wallet-password-file=/out/wallet-password.txt + rm "$tmpkeys/$(basename "$f")" +done +""" + plan.exec( + service_name=builder.name, + recipe=ExecRecipe(command=["bash", "-c", build_script]), + ) + + wallet_artifact = plan.store_service_files( + service_name=builder.name, + src="/out", + name="charon-prysm-wallet-files-" + str(node_index) + "-" + str(vc_index), + ) + plan.remove_service(name=builder.name) + + node_keystore_files = keystore_files_module.new_keystore_files( + files_artifact_uuid=wallet_artifact, + raw_root_dirpath="", + raw_keys_relative_dirpath="", + raw_secrets_relative_dirpath="", + nimbus_keys_relative_dirpath="", + prysm_relative_dirpath="prysm", + teku_keys_relative_dirpath="", + teku_secrets_relative_dirpath="", + charon_keys_relative_dirpath="", + ) + return wallet_artifact, node_keystore_files + + +def _charon_split_keys_to_vouch_wallet(plan, split_keys_artifact, vc_index, node_index): + """ + Build an ethdo wallet from a Charon node's split keystores by running the + official ethdo image, so the Vouch container itself needs no ethdo (and no + apt/download) — it just mounts the result. + + Returns an artifact containing: + wallets/ the ethdo wallet store + accounts.txt one account path per line (vals/valN) + account-passphrase.txt the account passphrase + """ + builder_name = "charon-vouch-wallet-" + str(node_index) + "-" + str(vc_index) + builder = plan.add_service( + name=builder_name, + config=ServiceConfig( + image=ETHDO_IMAGE, + entrypoint=["sh", "-c"], + cmd=["tail -f /dev/null"], + files={"/split-keys": split_keys_artifact}, + user=User(uid=0, gid=0), + ), + ) + + build_script = """set -e +mkdir -p /out/wallets +echo "1234" > /out/account-passphrase.txt +: > /out/accounts.txt +/app/ethdo --base-dir=/out/wallets wallet create --wallet=vals --passphrase="" +index=0 +for f in /split-keys/keystore-*.json; do + [ -f "$f" ] || continue + /app/ethdo --base-dir=/out/wallets account import \\ + --account="vals/val${index}" \\ + --keystore="$f" \\ + --keystore-passphrase="$(cat "${f%.json}.txt")" \\ + --passphrase="1234" --allow-weak-passphrases + echo "vals/val${index}" >> /out/accounts.txt + index=$((index + 1)) +done +""" + plan.exec( + service_name=builder.name, + recipe=ExecRecipe(command=["sh", "-c", build_script]), + ) + + wallet_artifact = plan.store_service_files( + service_name=builder.name, + src="/out", + name="charon-vouch-wallet-files-" + str(node_index) + "-" + str(vc_index), + ) + plan.remove_service(name=builder.name) + return wallet_artifact + + +def launch_prysm( + plan, + vc_service_name, + charon_validator_api_url, + split_keys_artifact, + launcher, + keymanager_file, + participant, + global_log_level, + cl_context, + tolerations, + node_selectors, + network_params, + port_publisher, + full_name, + vc_index, + node_index, + vc_image, +): + wallet_artifact, node_keystore_files = _charon_split_keys_to_prysm_wallet( + plan, split_keys_artifact, vc_index, node_index, vc_image + ) + + config = prysm.get_config( + plan=plan, + participant=participant, + el_cl_genesis_data=launcher.el_cl_genesis_data, + keymanager_file=keymanager_file, + image=vc_image, + beacon_http_urls=[charon_validator_api_url], + cl_context=cl_context, + el_context=None, + remote_signer_context=None, + full_name=full_name, + node_keystore_files=node_keystore_files, + prysm_password_relative_filepath="wallet-password.txt", + prysm_password_artifact_uuid=wallet_artifact, + tolerations=tolerations, + node_selectors=node_selectors, + keymanager_enabled=False, + network_params=network_params, + port_publisher=port_publisher, + vc_index=vc_index, + extra_files_artifacts=[], + distributed=True, + ) + + return plan.add_service(name=vc_service_name, config=config) + + +def launch_vouch( + plan, + vc_service_name, + charon_validator_api_url, + split_keys_artifact, + launcher, + keymanager_file, + participant, + global_log_level, + cl_context, + tolerations, + node_selectors, + network_params, + port_publisher, + full_name, + vc_index, + node_index, + vc_image, +): + vouch_wallet_artifact = _charon_split_keys_to_vouch_wallet( + plan, split_keys_artifact, vc_index, node_index + ) + + config = vouch.get_config( + plan=plan, + participant=participant, + image=vc_image, + global_log_level=global_log_level, + beacon_http_urls=[charon_validator_api_url], + cl_context=cl_context, + vouch_wallet_artifact=vouch_wallet_artifact, + tolerations=tolerations, + node_selectors=node_selectors, + ) + + return plan.add_service(name=vc_service_name, config=config) + + +def new_charon_launcher(el_cl_genesis_data): + return struct( + el_cl_genesis_data=el_cl_genesis_data, + ) diff --git a/src/vc/lighthouse.star b/src/vc/lighthouse.star index 5414b0082..9e9220d31 100644 --- a/src/vc/lighthouse.star +++ b/src/vc/lighthouse.star @@ -37,6 +37,7 @@ def get_config( tempo_otlp_grpc_url=None, otel_otlp_grpc_url=None, vc_binary_artifact=None, + distributed=False, ): log_level = input_parser.get_client_log_level_or_default( participant.vc_log_level, global_log_level, VERBOSITY_LEVELS @@ -85,6 +86,12 @@ def get_config( cmd.append("--gas-limit={0}".format(network_params.gas_limit)) cmd.append("--builder-proposals") + if distributed: + cmd.append("--distributed") + if "--builder-proposals" not in cmd: + cmd.append("--builder-proposals") + cmd.append("--use-long-timeouts") + telemetry_url = ( otel_otlp_grpc_url if otel_otlp_grpc_url != None else tempo_otlp_grpc_url ) diff --git a/src/vc/lodestar.star b/src/vc/lodestar.star index 2a5f9c470..c3c1dcc6d 100644 --- a/src/vc/lodestar.star +++ b/src/vc/lodestar.star @@ -34,6 +34,7 @@ def get_config( extra_files_artifacts, otel_otlp_grpc_url=None, vc_binary_artifact=None, + distributed=False, ): log_level = input_parser.get_client_log_level_or_default( participant.vc_log_level, global_log_level, VERBOSITY_LEVELS @@ -93,6 +94,11 @@ def get_config( if network_params.gas_limit > 0: cmd.append("--defaultGasLimit={0}".format(network_params.gas_limit)) + if distributed: + cmd.append("--distributed") + cmd.append("--builder") + cmd.append("--builder.selection=builderalways") + if len(participant.vc_extra_params) > 0: # this is a repeated, we convert it into Starlark cmd.extend([param for param in participant.vc_extra_params]) diff --git a/src/vc/nimbus.star b/src/vc/nimbus.star index f124d5278..00f7f5f07 100644 --- a/src/vc/nimbus.star +++ b/src/vc/nimbus.star @@ -24,6 +24,7 @@ def get_config( extra_files_artifacts, otel_otlp_grpc_url=None, vc_binary_artifact=None, + distributed=False, ): validator_keys_dirpath = "" validator_secrets_dirpath = "" @@ -77,6 +78,10 @@ def get_config( if network_params.gas_limit > 0: cmd.append("--suggested-gas-limit={0}".format(network_params.gas_limit)) + if distributed: + cmd.append("--distributed") + cmd.append("--payload-builder=true") + if len(participant.vc_extra_params) > 0: # this is a repeated, we convert it into Starlark cmd.extend([param for param in participant.vc_extra_params]) diff --git a/src/vc/prysm.star b/src/vc/prysm.star index cdd010df1..4f56ccba3 100644 --- a/src/vc/prysm.star +++ b/src/vc/prysm.star @@ -29,6 +29,7 @@ def get_config( extra_files_artifacts, otel_otlp_grpc_url=None, vc_binary_artifact=None, + distributed=False, ): validator_keys_dirpath = shared_utils.path_join( constants.VALIDATOR_KEYS_DIRPATH_ON_SERVICE_CONTAINER, @@ -54,8 +55,11 @@ def get_config( ] # Only add RPC provider if we're not using a blobber (blobber doesn't proxy RPC) - # Blobber uses port 5000, so check if that's in the URL - if ":5000" not in beacon_http_urls[0]: + # Blobber uses port 5000, so check if that's in the URL. + # In DV mode the only beacon endpoint is the Charon REST + # validator API, so skip the gRPC provider (which would point at the real + # beacon and bypass Charon). + if not distributed and ":5000" not in beacon_http_urls[0]: cmd.append("--beacon-rpc-provider=" + cl_context.beacon_grpc_url) if remote_signer_context == None: @@ -78,6 +82,9 @@ def get_config( if network_params.gas_limit > 0: cmd.append("--suggested-gas-limit={0}".format(network_params.gas_limit)) + if distributed: + cmd.append("--distributed") + keymanager_api_cmd = [ "--rpc", "--http-port={0}".format(vc_shared.VALIDATOR_HTTP_PORT_NUM), @@ -88,10 +95,15 @@ def get_config( # Check if we're using a blobber by checking for port 5000 is_using_blobber = ":5000" in beacon_http_urls[0] - if cl_context.client_name != constants.CL_TYPE.prysm or is_using_blobber: + if ( + cl_context.client_name != constants.CL_TYPE.prysm + or is_using_blobber + or distributed + ): # Use Beacon API if: # 1. Prysm VC wants to connect to a non-Prysm BN, OR # 2. Blobber is enabled (since blobber only proxies REST, not RPC) + # 3. Distributed (Charon) mode — Charon only exposes a REST validator API, OR cmd.append("--enable-beacon-rest-api") if len(participant.vc_extra_params) > 0: diff --git a/src/vc/teku.star b/src/vc/teku.star index ec2dab662..badd76a41 100644 --- a/src/vc/teku.star +++ b/src/vc/teku.star @@ -24,6 +24,7 @@ def get_config( extra_files_artifacts, otel_otlp_grpc_url=None, vc_binary_artifact=None, + distributed=False, ): validator_keys_dirpath = "" validator_secrets_dirpath = "" @@ -88,6 +89,10 @@ def get_config( "--Xvalidator-api-unsafe-hosts-enabled=true", ] + if distributed: + cmd.append("--Xobol-dvt-integration-enabled=true") + cmd.append("--validators-builder-registration-default-enabled=true") + if len(participant.vc_extra_params) > 0: # this is a repeated, we convert it into Starlark cmd.extend([param for param in participant.vc_extra_params]) diff --git a/src/vc/vc_launcher.star b/src/vc/vc_launcher.star index 72c4e05fd..08c1fb3f0 100644 --- a/src/vc/vc_launcher.star +++ b/src/vc/vc_launcher.star @@ -217,6 +217,14 @@ def get_vc_config( otel_otlp_grpc_url=otel_otlp_grpc_url, vc_binary_artifact=vc_binary_artifact, ) + elif vc_type == constants.VC_TYPE.charon: + # Charon is launched separately by charon_launcher.star. + return None + elif vc_type == constants.VC_TYPE.vouch: + fail( + "vouch VC is only supported as a Charon distributed-validator client; " + + "set vc_type=charon with charon_params.charon_vc=vouch" + ) elif vc_type == constants.VC_TYPE.grandine: fail("Grandine VC is not yet supported") elif vc_type == constants.VC_TYPE.consensoor: diff --git a/src/vc/vouch.star b/src/vc/vouch.star new file mode 100644 index 000000000..6728fe0df --- /dev/null +++ b/src/vc/vouch.star @@ -0,0 +1,138 @@ +shared_utils = import_module("../shared_utils/shared_utils.star") +constants = import_module("../package_io/constants.star") +input_parser = import_module("../package_io/input_parser.star") +vc_shared = import_module("./shared.star") + +# Where the prebuilt ethdo wallet artifact is mounted in the Vouch container. +VOUCH_WALLET_DIRPATH = "/vouch-wallet" + +VERBOSITY_LEVELS = { + constants.GLOBAL_LOG_LEVEL.error: "error", + constants.GLOBAL_LOG_LEVEL.warn: "warn", + constants.GLOBAL_LOG_LEVEL.info: "info", + constants.GLOBAL_LOG_LEVEL.debug: "debug", +} + + +def get_config( + plan, + participant, + image, + global_log_level, + beacon_http_urls, + cl_context, + vouch_wallet_artifact, + tolerations, + node_selectors, +): + """ + Vouch validator client config. + + The ethdo wallet is built up-front by the caller and passed in as + vouch_wallet_artifact, so this container needs no ethdo/apt/download: it just + writes ~/.vouch.yml pointing at the mounted wallet and the beacon API, then + runs vouch. + + The wallet artifact contains: + wallets/ the ethdo wallet store + accounts.txt one account path per line + account-passphrase.txt the account passphrase + """ + log_level = input_parser.get_client_log_level_or_default( + participant.vc_log_level, global_log_level, VERBOSITY_LEVELS + ) + + startup_script = ( + """#!/usr/bin/env bash +set -e + +# Turn the account list shipped with the wallet into the YAML the wallet +# account manager expects. +accounts_yaml=$(sed 's/^/ - /' """ + + VOUCH_WALLET_DIRPATH + + """/accounts.txt) + +cat > ~/.vouch.yml < 0: + config_args["min_cpu"] = participant.vc_min_cpu + if participant.vc_max_cpu > 0: + config_args["max_cpu"] = participant.vc_max_cpu + if participant.vc_min_mem > 0: + config_args["min_memory"] = participant.vc_min_mem + if participant.vc_max_mem > 0: + config_args["max_memory"] = participant.vc_max_mem + + return ServiceConfig(**config_args)