action.yaml

name: 'Benchmarkoor'
description: 'Run Ethereum execution client benchmarks using benchmarkoor'
author: 'ethpandaops'
branding:
  icon: 'activity'
  color: 'gray-dark'

inputs:
  github-token:
    description: 'GitHub token for API access'
    required: true

  mode:
    description: 'Which phase(s) to run: "all" (build if a build-config is set, then run — the default), "build" (build only, no run), or "run" (run only, skip build even if a build-config is set).'
    required: false
    default: 'all'

  image:
    description: 'Docker image to extract the benchmarkoor binary from (e.g., ghcr.io/ethpandaops/benchmarkoor:master). If not provided, will build the image locally from git-ref/git-repo and extract from it.'
    required: false
    default: ''

  git-ref:
    description: 'Git branch or commit hash to build from. Only used when image is not provided. Defaults to master.'
    required: false
    default: ''

  git-repo:
    description: 'Git repository URL to clone for building the image. Only used when image is not provided. Defaults to ethpandaops/benchmarkoor.'
    required: false
    default: 'https://github.com/ethpandaops/benchmarkoor.git'

  upload-artifacts:
    description: 'Whether to upload run results as GitHub artifacts'
    required: false
    default: 'false'

  upload-build-artifacts:
    description: 'Whether to upload build artifacts as a GitHub artifact: each state-actor target''s metadata JSON (not the datadir), each eest-payloads target''s full fixtures directory, and each pre-runs target''s replay bundle (pre-run.request, not the datadir).'
    required: false
    default: 'false'

  build-artifact-name:
    description: 'Name for the uploaded build artifact. Override to disambiguate matrix legs (artifact names must be unique within a run), e.g. append the client. Leave empty for the default `benchmarkoor-build-<run_id>`. Only used when upload-build-artifacts is true.'
    required: false
    default: ''

  run-config-urls:
    description: 'Comma-separated URLs for config files to download and pass via --config (merged in order)'
    required: false
    default: ''

  run-config:
    description: 'Raw YAML config content to pass via --config (appended after URL config)'
    required: false
    default: ''

  run-args:
    description: 'Extra flags passed to benchmarkoor run command'
    required: false
    default: ''

  build-config-urls:
    description: 'Comma-separated URLs for config files to download and pass to `benchmarkoor build` via --config (merged in order). Setting this or build-config enables the build phase (state-actor snapshots + eest payloads), which runs before the benchmark run.'
    required: false
    default: ''

  build-config:
    description: 'Raw YAML config content to pass to `benchmarkoor build` via --config (appended after URL config). Setting this or build-config-urls enables the build phase.'
    required: false
    default: ''

  build-args:
    description: 'Extra flags passed to the benchmarkoor build command (e.g. --force, --limit-state-actor-target=)'
    required: false
    default: ''

  tmp-dir:
    description: 'Temporary directory path used for benchmarkoor data (overlayfs) and cache'
    required: false
    default: '/tmp'

runs:
  using: 'composite'
  steps:
    - name: Validate mode
      shell: bash
      run: |
        case "${{ inputs.mode }}" in
          all|build|run) ;;
          *) echo "Invalid mode '${{ inputs.mode }}' (expected: all, build, or run)" >&2; exit 1 ;;
        esac

    - name: Verify and install dependencies
      uses: ethpandaops/benchmarkoor/.github/actions/dependencies@0ff45ea539387c901ac4a7845e2c6e6aeb9d8f25

    - name: Build Docker image locally
      if: inputs.image == ''
      shell: bash
      run: |
        GIT_REF="${{ inputs.git-ref }}"
        if [ -z "$GIT_REF" ]; then
          GIT_REF="master"
        fi

        GIT_REPO="${{ inputs.git-repo }}"

        echo "Cloning ${GIT_REPO} at ref $GIT_REF"
        git clone "${GIT_REPO}" benchmarkoor-build
        cd benchmarkoor-build
        git checkout "$GIT_REF"

        echo "Building Docker image..."
        docker build -t benchmarkoor:local .
        cd ..

        rm -rf benchmarkoor-build

    - name: Extract benchmarkoor binary
      id: extract-binary
      shell: bash
      run: |
        if [ -n "${{ inputs.image }}" ]; then
          IMAGE="${{ inputs.image }}"
          echo "Pulling Docker image to extract binary: ${IMAGE}"
          docker pull "${IMAGE}" || {
            echo "Failed to pull Docker image: ${IMAGE}"
            exit 1
          }
        else
          IMAGE="benchmarkoor:local"
          echo "Using locally built image: ${IMAGE}"
        fi

        BIN="${{ runner.temp }}/benchmarkoor"

        # docker create + docker cp gives us the binary owned by the
        # runner user without needing a writable bind-mount or matching
        # uid inside the container.
        cid=$(docker create "${IMAGE}")
        trap "docker rm -f \"$cid\" >/dev/null 2>&1 || true" EXIT
        docker cp "$cid:/usr/local/bin/benchmarkoor" "$BIN"
        chmod +x "$BIN"

        echo "Extracted binary to $BIN"
        "$BIN" version || true

        echo "binary=${BIN}" >> "$GITHUB_OUTPUT"

    - name: Build Benchmarkoor artifacts
      id: build-benchmarkoor
      if: (inputs.mode == 'all' || inputs.mode == 'build') && (inputs.build-config != '' || inputs.build-config-urls != '')
      shell: bash
      env:
        INPUT_BUILD_CONFIG: ${{ inputs.build-config }}
        INPUT_BUILD_CONFIG_URLS: ${{ inputs.build-config-urls }}
      run: |
        BIN="${{ steps.extract-binary.outputs.binary }}"

        # Download URL configs if provided (comma-separated)
        if [ -n "$INPUT_BUILD_CONFIG_URLS" ]; then
          IFS=',' read -ra CONFIG_URLS <<< "$INPUT_BUILD_CONFIG_URLS"
          for i in "${!CONFIG_URLS[@]}"; do
            URL=$(echo "${CONFIG_URLS[$i]}" | xargs)  # trim whitespace
            echo "Downloading build config from ${URL}"
            # --retry covers the transient class (5xx, 429, 408, timeouts) and
            # still fails fast on a 404, so a raw.githubusercontent.com blip
            # cannot kill a multi-hour benchmark before it starts.
            curl -fsSL --retry 5 --retry-delay 2 --retry-max-time 120 \
              "${URL}" -o "${{ runner.temp }}/benchmarkoor-build-config-url-${i}.yaml"
          done
        fi

        # Write inline config if provided
        if [ -n "$INPUT_BUILD_CONFIG" ]; then
          echo "$INPUT_BUILD_CONFIG" > "${{ runner.temp }}/benchmarkoor-build-config-inline.yaml"
        fi

        # Build argv as an array so quoted values survive intact. Config files
        # apply in order: URL configs, then inline config (last wins). This step
        # runs from the workspace, so any relative paths in the config (e.g. a
        # custom fill_dockerfile) resolve against the checkout.
        ARGS=("build")
        for cfg in "${{ runner.temp }}"/benchmarkoor-build-config-url-*.yaml; do
          [ -f "$cfg" ] && ARGS+=("--config=$cfg")
        done
        if [ -f "${{ runner.temp }}/benchmarkoor-build-config-inline.yaml" ]; then
          ARGS+=("--config=${{ runner.temp }}/benchmarkoor-build-config-inline.yaml")
        fi

        # Emit a machine-readable build summary for the markdown step below.
        ARGS+=("--summary-json=${{ runner.temp }}/benchmarkoor-build-summary.json")

        # Append extra build args if provided (whitespace-split, same as run).
        if [ -n "${{ inputs.build-args }}" ]; then
          # shellcheck disable=SC2206 # intentional word-split for argv
          EXTRA_ARGS=(${{ inputs.build-args }})
          ARGS+=("${EXTRA_ARGS[@]}")
        fi

        # `sudo -E` for the same reasons as the run step: keeps the built
        # snapshots/fixtures root-owned (so the rootful run phase can read
        # them) and propagates caller env (e.g. STATE_DIR) into the build.
        echo "Running: sudo -E $BIN ${ARGS[*]}"
        sudo -E "$BIN" "${ARGS[@]}"

    - name: Generate build markdown summary
      # Runs even if the build failed so a partial summary (incl. ERR targets)
      # still reaches the job summary. sudo because the build artifacts are
      # root-owned (see above).
      if: always() && (inputs.mode == 'all' || inputs.mode == 'build') && (inputs.build-config != '' || inputs.build-config-urls != '')
      shell: bash
      run: |
        BIN="${{ steps.extract-binary.outputs.binary }}"
        SUMMARY="${{ runner.temp }}/benchmarkoor-build-summary.json"
        if [ -f "$SUMMARY" ]; then
          sudo -E "$BIN" generate-build-markdown-summary \
            --input="$SUMMARY" \
            --output="${{ runner.temp }}/benchmarkoor-build-summary.md"
          if [ -f "${{ runner.temp }}/benchmarkoor-build-summary.md" ]; then
            cat "${{ runner.temp }}/benchmarkoor-build-summary.md" >> "$GITHUB_STEP_SUMMARY"
            echo "" >> "$GITHUB_STEP_SUMMARY"
          fi
        fi

    - name: Archive build artifacts
      # Collect each built target's artifacts (discovered from build-summary.json):
      # eest-payloads → the full fixtures directory; pre-runs → the whole
      # pre_run_bundle directory (the .request bundle plus its metadata sidecar,
      # located via the summary's bundle_dir), NOT the multi-GB datadir;
      # state-actor → only the state-actor-* metadata files, NOT the multi-GB
      # datadir. sudo because the
      # build artifacts are root-owned; chown the staging copy so the upload (and
      # tar) can read it. Runs on build modes, even on failure (partial artifacts).
      if: always() && inputs.upload-build-artifacts == 'true' && (inputs.mode == 'all' || inputs.mode == 'build') && (inputs.build-config != '' || inputs.build-config-urls != '')
      shell: bash
      run: |
        SUMMARY="${{ runner.temp }}/benchmarkoor-build-summary.json"
        STAGING="${{ runner.temp }}/benchmarkoor-build-artifacts"
        if [ ! -f "$SUMMARY" ]; then
          echo "No build summary at $SUMMARY; nothing to archive"
          exit 0
        fi

        sudo rm -rf "$STAGING"
        mkdir -p "$STAGING"

        while IFS=$'\t' read -r builder name outdir bundledir; do
          [ -d "$outdir" ] || continue
          # Group by builder (state-actor/<target>, eest-payloads/<target>) so a
          # state-actor and eest target of the same name don't collide.
          dest="$STAGING/${builder}/${name}"
          mkdir -p "$dest"
          if [ "$builder" = "eest-payloads" ]; then
            sudo cp -a "$outdir/." "$dest/"
          elif [ "$builder" = "pre-runs" ]; then
            # pre-runs: the whole pre_run_bundle directory, NOT the multi-GB
            # advanced datadir. That means the .request bundle AND the
            # pre-run.meta.json beside it: the runner uses that metadata to tell
            # whether a datadir already sits at the bundle's end block, and
            # skipping the replay outright rather than streaming a multi-GB file
            # to skip every line is worth minutes per run. Copying the directory
            # (rather than naming the bundle file) also keeps any future sidecar.
            #
            # It usually sits under output_dir, but a target's bundle_dir moves it
            # off the advanced datadir (a schelk target needs that, since a later
            # restore discards the scratch), so follow the summary's bundle_dir.
            # An empty bundle_dir means either a replay-only target, which records
            # nothing, or a summary from before the field existed — fall back to
            # output_dir and stay quiet. A declared bundle_dir with no bundle IS
            # worth flagging.
            if [ -n "$bundledir" ]; then
              bundlesrc="$bundledir/pre_run_bundle"
              if [ -f "$bundlesrc/pre-run.request" ]; then
                mkdir -p "$dest/pre_run_bundle"
                sudo cp -a "$bundlesrc/." "$dest/pre_run_bundle/"
              else
                echo "::warning::pre-runs target $name declared bundle_dir but no bundle at $bundlesrc/pre-run.request"
              fi
            elif [ -f "$outdir/pre_run_bundle/pre-run.request" ]; then
              mkdir -p "$dest/pre_run_bundle"
              sudo cp -a "$outdir/pre_run_bundle/." "$dest/pre_run_bundle/"
            fi
          else
            # state-actor: metadata files only (find runs as root to read the datadir).
            sudo find "$outdir" -maxdepth 1 -name 'state-actor-*' -exec cp -a {} "$dest/" \;
          fi
        done < <(jq -r '.targets[] | [.builder, .name, .output_dir, (.bundle_dir // "")] | @tsv' "$SUMMARY")

        sudo chown -R "$(id -u):$(id -g)" "$STAGING"
        tar -czf "${{ runner.temp }}/benchmarkoor-build-artifacts.tar.gz" -C "${{ runner.temp }}" benchmarkoor-build-artifacts

    - name: Upload build artifacts
      uses: actions/upload-artifact@v7
      if: always() && inputs.upload-build-artifacts == 'true' && (inputs.mode == 'all' || inputs.mode == 'build') && (inputs.build-config != '' || inputs.build-config-urls != '')
      with:
        name: ${{ inputs.build-artifact-name != '' && inputs.build-artifact-name || format('benchmarkoor-build-{0}', github.run_id) }}
        path: ${{ runner.temp }}/benchmarkoor-build-artifacts.tar.gz
        if-no-files-found: ignore

    - name: Get Job ID from GH API
      id: get-job-id
      if: inputs.mode == 'all' || inputs.mode == 'run'
      shell: bash
      env:
        GH_TOKEN: ${{ inputs.github-token }}
      run: |
        jobs=$(gh api -F per_page=100 -X GET repos/${{ github.repository }}/actions/runs/${{ github.run_id }}/attempts/${{ github.run_attempt }}/jobs)
        job_id=$(echo "$jobs" | jq -r '.jobs[] | select(.runner_name=="${{ runner.name }}" and .status=="in_progress") | .id')
        echo "job_id=${job_id}" >> "$GITHUB_OUTPUT"

    - name: Run Benchmarkoor
      id: run-benchmarkoor
      if: inputs.mode == 'all' || inputs.mode == 'run'
      shell: bash
      env:
        INPUT_RUN_CONFIG: ${{ inputs.run-config }}
        INPUT_RUN_CONFIG_URLS: ${{ inputs.run-config-urls }}
      run: |
        BIN="${{ steps.extract-binary.outputs.binary }}"
        mkdir -p "${{ runner.temp }}/results"

        # Download URL configs if provided (comma-separated)
        if [ -n "$INPUT_RUN_CONFIG_URLS" ]; then
          IFS=',' read -ra CONFIG_URLS <<< "$INPUT_RUN_CONFIG_URLS"
          for i in "${!CONFIG_URLS[@]}"; do
            URL=$(echo "${CONFIG_URLS[$i]}" | xargs)  # trim whitespace
            echo "Downloading config from ${URL}"
            # See the build-config download above: retry the transient class,
            # fail fast on a genuine 404.
            curl -fsSL --retry 5 --retry-delay 2 --retry-max-time 120 \
              "${URL}" -o "${{ runner.temp }}/benchmarkoor-config-url-${i}.yaml"
          done
        fi

        # Write inline config if provided
        if [ -n "$INPUT_RUN_CONFIG" ]; then
          echo "$INPUT_RUN_CONFIG" > "${{ runner.temp }}/benchmarkoor-config-inline.yaml"
        fi

        # Export benchmarkoor's own config env vars. Anything the caller
        # set via `env:` on the action step is already in our process
        # environment; `sudo -E` below propagates both sets to the
        # benchmarkoor process (which it needs to be: root, for
        # drop_memory_caches, schelk dm_era ops, rootful podman, etc.).
        export BENCHMARKOOR_RUNNER_GITHUB_TOKEN="${{ inputs.github-token }}"
        export BENCHMARKOOR_RUNNER_BENCHMARK_RESULTS_DIR="${{ runner.temp }}/results"
        export BENCHMARKOOR_RUNNER_BENCHMARK_RESULTS_OWNER="$(id -u):$(id -g)"
        export BENCHMARKOOR_RUNNER_DIRECTORIES_TMP_DATADIR="${{ inputs.tmp-dir }}"
        export BENCHMARKOOR_RUNNER_DIRECTORIES_TMP_CACHEDIR="${{ inputs.tmp-dir }}/benchmarkoor-cache"

        # Build argv as an array so quoted values survive intact.
        ARGS=("run")

        # Add config files in order: URL configs, then inline config (last wins).
        for cfg in "${{ runner.temp }}"/benchmarkoor-config-url-*.yaml; do
          [ -f "$cfg" ] && ARGS+=("--config=$cfg")
        done
        if [ -f "${{ runner.temp }}/benchmarkoor-config-inline.yaml" ]; then
          ARGS+=("--config=${{ runner.temp }}/benchmarkoor-config-inline.yaml")
        fi

        # GitHub Actions context as metadata labels.
        ARGS+=(
          "--metadata.label=github.run_id=${{ github.run_id }}"
          "--metadata.label=github.run_number=${{ github.run_number }}"
          "--metadata.label=github.job=${{ github.job }}"
          "--metadata.label=github.job_id=${{ steps.get-job-id.outputs.job_id }}"
          "--metadata.label=github.repository=${{ github.repository }}"
          "--metadata.label=github.workflow=${{ github.workflow }}"
          "--metadata.label=github.sha=${{ github.sha }}"
          "--metadata.label=github.actor=${{ github.actor }}"
          "--metadata.label=github.event_name=${{ github.event_name }}"
          "--metadata.label=github.ref=${{ github.ref }}"
        )

        # Append extra run args if provided (whitespace-split, same as before).
        if [ -n "${{ inputs.run-args }}" ]; then
          # shellcheck disable=SC2206 # intentional word-split for argv
          EXTRA_ARGS=(${{ inputs.run-args }})
          ARGS+=("${EXTRA_ARGS[@]}")
        fi

        echo "Running: sudo -E $BIN ${ARGS[*]}"

        set +e
        sudo -E "$BIN" "${ARGS[@]}"
        EXIT_CODE=$?
        set -e

        # Find run directories
        RUN_DIRS=""
        if [ -d "${{ runner.temp }}/results/runs" ]; then
          RUN_DIRS=$(find "${{ runner.temp }}/results/runs" -mindepth 1 -maxdepth 1 -type d | sort | tr '\n' ' ')
        fi

        echo "run-dirs=${RUN_DIRS}" >> "$GITHUB_OUTPUT"

        if [ $EXIT_CODE -ne 0 ]; then
          echo "Benchmarkoor failed with exit code $EXIT_CODE"
          exit $EXIT_CODE
        fi

        echo "Benchmarkoor completed successfully"

    - name: Generate markdown summaries
      if: always() && steps.run-benchmarkoor.outputs.run-dirs != ''
      shell: bash
      run: |
        BIN="${{ steps.extract-binary.outputs.binary }}"

        for RUN_DIR_HOST in ${{ steps.run-benchmarkoor.outputs.run-dirs }}; do
          RUN_DIR_NAME=$(basename "$RUN_DIR_HOST")

          echo "Generating summary for ${RUN_DIR_NAME}"

          "$BIN" generate-markdown-summary \
            --run-dir="${RUN_DIR_HOST}" \
            --output="${RUN_DIR_HOST}/summary.md"

          if [ -f "${RUN_DIR_HOST}/summary.md" ]; then
            cat "${RUN_DIR_HOST}/summary.md" >> "$GITHUB_STEP_SUMMARY"
            echo "" >> "$GITHUB_STEP_SUMMARY"
          fi
        done

    - name: Create results archive
      if: always() && inputs.upload-artifacts == 'true' && inputs.mode != 'build'
      shell: bash
      run: |
        if [ -d "${{ runner.temp }}/results" ]; then
          tar -czf "${{ runner.temp }}/benchmarkoor-results.tar.gz" -C "${{ runner.temp }}" results
        fi

    - name: Upload artifacts
      uses: actions/upload-artifact@v7
      if: always() && inputs.upload-artifacts == 'true' && inputs.mode != 'build'
      with:
        name: benchmarkoor-${{ github.run_id }}
        path: ${{ runner.temp }}/benchmarkoor-results.tar.gz

    - name: Cleanup
      shell: bash
      if: always()
      run: |
        echo "Cleaning up"
        rm -rf "${{ runner.temp }}/results"
        rm -f "${{ runner.temp }}/benchmarkoor-config-url-"*.yaml
        rm -f "${{ runner.temp }}/benchmarkoor-config-inline.yaml"
        rm -f "${{ runner.temp }}/benchmarkoor-build-config-url-"*.yaml
        rm -f "${{ runner.temp }}/benchmarkoor-build-config-inline.yaml"
        rm -f "${{ runner.temp }}/benchmarkoor-build-summary.json" "${{ runner.temp }}/benchmarkoor-build-summary.md"
        rm -f "${{ runner.temp }}/benchmarkoor-results.tar.gz"
        sudo rm -rf "${{ runner.temp }}/benchmarkoor-build-artifacts" "${{ runner.temp }}/benchmarkoor-build-artifacts.tar.gz"
        rm -f "${{ runner.temp }}/benchmarkoor"