feat(integration): Parallelize integration tests

.github/actions/nimbus-build-system/action.yml

@@ -29,6 +29,7 @@ runs:
       shell: ${{ inputs.shell }} {0}
       run: |
         curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs/ | sh -s -- --default-toolchain=${{ inputs.rust_version }} -y
+        echo "$HOME/.cargo/bin" >> "$GITHUB_PATH"
 
     - name: APT (Linux amd64/arm64)
       if: inputs.os == 'linux' && (inputs.cpu == 'amd64' || inputs.cpu == 'arm64')
@@ -89,7 +90,7 @@ runs:
 
     - name: Install gcc 14 on Linux
       # We don't want to install gcc 14 for coverage (Ubuntu 20.04)
-      if : ${{ inputs.os == 'linux' && inputs.coverage != 'true' }}
+      if: ${{ inputs.os == 'linux' && inputs.coverage != 'true' }}
       shell: ${{ inputs.shell }} {0}
       run: |
         # Add GCC-14 to alternatives
@@ -101,10 +102,17 @@ runs:
       if: inputs.os == 'linux' || inputs.os == 'macos'
       uses: hendrikmuhs/[email protected]
       with:
-        create-symlink: true
+        create-symlink: false
         key: ${{ matrix.os }}-${{ matrix.builder }}-${{ matrix.cpu }}-${{ matrix.tests }}-${{ matrix.nim_version }}
         evict-old-files: 7d
 
+    - name: Add ccache to path on Linux/Mac
+      if: inputs.os == 'linux' || inputs.os == 'macos'
+      shell: ${{ inputs.shell }} {0}
+      run: |
+        echo "/usr/lib/ccache:/usr/local/opt/ccache/libexec" >> "$GITHUB_PATH"
+        echo "/usr/local/opt/ccache/libexec" >> "$GITHUB_PATH"
+
     - name: Install ccache on Windows
       if: inputs.os == 'windows'
       uses: hendrikmuhs/[email protected]
@@ -117,11 +125,11 @@ runs:
       shell: ${{ inputs.shell }} {0}
       run: |
         CCACHE_DIR=$(dirname $(which ccache))/ccached
-        mkdir ${CCACHE_DIR}
-        ln -s $(which ccache) ${CCACHE_DIR}/gcc.exe
-        ln -s $(which ccache) ${CCACHE_DIR}/g++.exe
-        ln -s $(which ccache) ${CCACHE_DIR}/cc.exe
-        ln -s $(which ccache) ${CCACHE_DIR}/c++.exe
+        mkdir -p ${CCACHE_DIR}
+        ln -sf $(which ccache) ${CCACHE_DIR}/gcc.exe
+        ln -sf $(which ccache) ${CCACHE_DIR}/g++.exe
+        ln -sf $(which ccache) ${CCACHE_DIR}/cc.exe
+        ln -sf $(which ccache) ${CCACHE_DIR}/c++.exe
         echo "export PATH=${CCACHE_DIR}:\$PATH" >> $HOME/.bash_profile # prefix path in MSYS2
 
     - name: Derive environment variables
@@ -202,7 +210,7 @@ runs:
     - name: Restore Nim toolchain binaries from cache
       id: nim-cache
       uses: actions/cache@v4
-      if : ${{ inputs.coverage != 'true'  }}
+      if: ${{ inputs.coverage != 'true' }}
       with:
         path: NimBinaries
         key: ${{ inputs.os }}-${{ inputs.cpu }}-nim-${{ inputs.nim_version }}-cache-${{ env.cache_nonce }}-${{ github.run_id }}
@@ -225,4 +233,4 @@ runs:
         gcc --version
         make -j${ncpu} CI_CACHE=NimBinaries ${ARCH_OVERRIDE} QUICK_AND_DIRTY_COMPILER=1 update
         echo
-        ./env.sh nim --version
+        ./env.sh nim --version

.github/workflows/ci-reusable.yml

@@ -42,6 +42,46 @@ jobs:
           nim_version: ${{ matrix.nim_version }}
           coverage: false
 
+      - name: Check runner resources for parallel integration tests
+        run: |
+          echo "Determining runner"
+          case "${{ matrix.os }}" in
+            linux)    CPU=$(nproc --all)
+                      RAM=$(awk '/MemTotal/ {print int($2 / 1024 / 1024 + 0.5)}' /proc/meminfo)
+                      ;;
+            macos)    CPU=$(sysctl -n hw.ncpu)
+                      RAM=$(sysctl -n hw.memsize | awk '{print $0/1073741824}')
+                      sysctl -n hw.ncpu
+                      ;;
+            windows)  CPU=$NUMBER_OF_PROCESSORS
+                      RAM=$(systeminfo | awk '/Total Physical Memory:/ { gsub(/,/,"."); print int($4 + 0.5) }')
+                      ;;
+            *)        CPU=2
+                      RAM=8
+                      echo "Unknown runner"
+                      ;;
+          esac
+          echo "CPU=${CPU}" >> $GITHUB_ENV
+          echo "RAM=${RAM}" >> $GITHUB_ENV
+          echo "TYPE=${RUNNER_ENVIRONMENT}" >> $GITHUB_ENV
+
+          # Set PARALLEL=1 if the runner has enough resources
+          if [[ ("${{ matrix.os }}" == "linux" || "${{ matrix.os }}" == "windows") && "${CPU}" -ge 16 ]]; then
+            echo "PARALLEL=1" >> $GITHUB_ENV
+          elif [[ "${{ matrix.os }}" == "macos" && "${CPU}" -ge 6 ]]; then
+            echo "PARALLEL=1" >> $GITHUB_ENV
+          else
+            echo "PARALLEL=0" >> $GITHUB_ENV
+          fi
+
+      - name: Show runner information
+        run: |
+          echo "OS: ${{ matrix.os }}"
+          echo "CPU: ${{ env.CPU }}"
+          echo "RAM: ${{ env.RAM }} GB"
+          echo "TYPE: ${{ env.TYPE }}"
+          echo "PARALLEL: ${{ env.PARALLEL }}"
+
       ## Part 1 Tests ##
       - name: Unit tests
         if: matrix.tests == 'unittest' || matrix.tests == 'all'
@@ -53,13 +93,20 @@ jobs:
         with:
           node-version: 18.15
 
-      - name: Start Ethereum node with Codex contracts
+      - name: Install Ethereum node dependencies
         if: matrix.tests == 'contract' || matrix.tests == 'integration' || matrix.tests == 'tools' || matrix.tests == 'all'
         working-directory: vendor/codex-contracts-eth
         env:
           MSYS2_PATH_TYPE: inherit
         run: |
           npm install
+
+      - name: Run Ethereum node with Codex contracts
+        if: matrix.tests == 'contract' || (matrix.tests == 'integration' && env.PARALLEL != 1) || matrix.tests == 'tools' || matrix.tests == 'all'
+        working-directory: vendor/codex-contracts-eth
+        env:
+          MSYS2_PATH_TYPE: inherit
+        run: |
           npm start &
 
       ## Part 2 Tests ##
@@ -70,13 +117,13 @@ jobs:
       ## Part 3 Tests ##
       - name: Integration tests
         if: matrix.tests == 'integration' || matrix.tests == 'all'
-        run: make -j${ncpu} testIntegration
+        run: make -j${ncpu} PARALLEL=${{ env.PARALLEL }} DEBUG=${{ runner.debug }} testIntegration
 
       - name: Upload integration tests log files
         uses: actions/upload-artifact@v4
         if: (matrix.tests == 'integration' || matrix.tests == 'all') && always()
         with:
-          name: ${{ matrix.os }}-${{ matrix.cpu }}-${{ matrix.nim_version }}-integration-tests-logs
+          name: ${{ matrix.os }}-${{ matrix.cpu }}-${{ matrix.nim_version }}-${{ matrix.tests }}-tests-logs
           path: tests/integration/logs/
           retention-days: 1
 

.github/workflows/ci.yml

@@ -10,34 +10,46 @@ on:
 env:
   cache_nonce: 0 # Allows for easily busting actions/cache caches
   nim_version: v2.0.14
+  builder_integration_from_vars: true
+  builder_integration_linux: runner-node-01-linux-03-eu-hel1
+  builder_integration_macos: macos-14-large
+  builder_integration_windows: windows-latest-amd64-32vcpu
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref || github.run_id }}
   cancel-in-progress: true
 
 jobs:
   matrix:
+    name: Compute matrix
     runs-on: ubuntu-latest
     outputs:
       matrix: ${{ steps.matrix.outputs.matrix }}
       cache_nonce: ${{ env.cache_nonce }}
     steps:
+      - name: Compute builders
+        if: ${{ env.builder_integration_from_vars == 'true' }}
+        run: |
+          if [[ -n "${{ vars.builder_integration_linux }}" ]]; then echo "builder_integration_linux=${{ vars.builder_integration_linux }}"; fi >> $GITHUB_ENV
+          if [[ -n "${{ vars.builder_integration_macos }}" ]]; then echo "builder_integration_macos=${{ vars.builder_integration_macos }}"; fi >> $GITHUB_ENV
+          if [[ -n "${{ vars.builder_integration_windows }}" ]]; then echo "builder_integration_windows=${{ vars.builder_integration_windows }}"; fi >> $GITHUB_ENV
+
       - name: Compute matrix
         id: matrix
         uses: fabiocaccamo/create-matrix-action@v5
         with:
           matrix: |
             os {linux},   cpu {amd64}, builder {ubuntu-latest},   tests {unittest},    nim_version {${{ env.nim_version }}}, shell {bash --noprofile --norc -e -o pipefail}
             os {linux},   cpu {amd64}, builder {ubuntu-latest},   tests {contract},    nim_version {${{ env.nim_version }}}, shell {bash --noprofile --norc -e -o pipefail}
-            os {linux},   cpu {amd64}, builder {ubuntu-latest},   tests {integration}, nim_version {${{ env.nim_version }}}, shell {bash --noprofile --norc -e -o pipefail}
+            os {linux},   cpu {amd64}, builder {${{ env.builder_integration_linux }}},   tests {integration}, nim_version {${{ env.nim_version }}}, shell {bash --noprofile --norc -e -o pipefail}
             os {linux},   cpu {amd64}, builder {ubuntu-latest},   tests {tools},       nim_version {${{ env.nim_version }}}, shell {bash --noprofile --norc -e -o pipefail}
             os {macos},   cpu {amd64}, builder {macos-13},       tests {unittest},    nim_version {${{ env.nim_version }}}, shell {bash --noprofile --norc -e -o pipefail}
             os {macos},   cpu {amd64}, builder {macos-13},       tests {contract},    nim_version {${{ env.nim_version }}}, shell {bash --noprofile --norc -e -o pipefail}
-            os {macos},   cpu {amd64}, builder {macos-13},       tests {integration}, nim_version {${{ env.nim_version }}}, shell {bash --noprofile --norc -e -o pipefail}
+            os {macos},   cpu {amd64}, builder {${{ env.builder_integration_macos }}},   tests {integration}, nim_version {${{ env.nim_version }}}, shell {bash --noprofile --norc -e -o pipefail}
             os {macos},   cpu {amd64}, builder {macos-13},       tests {tools},       nim_version {${{ env.nim_version }}}, shell {bash --noprofile --norc -e -o pipefail}
             os {windows}, cpu {amd64}, builder {windows-latest}, tests {unittest},    nim_version {${{ env.nim_version }}}, shell {msys2}
             os {windows}, cpu {amd64}, builder {windows-latest}, tests {contract},    nim_version {${{ env.nim_version }}}, shell {msys2}
-            os {windows}, cpu {amd64}, builder {windows-latest}, tests {integration}, nim_version {${{ env.nim_version }}}, shell {msys2}
+            os {windows}, cpu {amd64}, builder {${{ env.builder_integration_windows }}}, tests {integration}, nim_version {${{ env.nim_version }}}, shell {msys2}
             os {windows}, cpu {amd64}, builder {windows-latest}, tests {tools},       nim_version {${{ env.nim_version }}}, shell {msys2}
 
   build:

Makefile

@@ -141,10 +141,24 @@ testContracts: | build deps
 	echo -e $(BUILD_MSG) "build/$@" && \
 		$(ENV_SCRIPT) nim testContracts $(NIM_PARAMS) build.nims
 
+TEST_PARAMS :=
+ifdef DEBUG
+	TEST_PARAMS := $(TEST_PARAMS) -d:DebugTestHarness=$(DEBUG)
+  TEST_PARAMS := $(TEST_PARAMS) -d:NoCodexLogFilters=$(DEBUG)
+  TEST_PARAMS := $(TEST_PARAMS) -d:ShowContinuousStatusUpdates=$(DEBUG)
+  TEST_PARAMS := $(TEST_PARAMS) -d:DebugHardhat=$(DEBUG)
+endif
+ifdef TEST_TIMEOUT
+  TEST_PARAMS := $(TEST_PARAMS) -d:TestTimeout=$(TEST_TIMEOUT)
+endif
+ifdef PARALLEL
+  TEST_PARAMS := $(TEST_PARAMS) -d:EnableParallelTests=$(PARALLEL)
+endif
+
 # Builds and runs the integration tests
 testIntegration: | build deps
 	echo -e $(BUILD_MSG) "build/$@" && \
-		$(ENV_SCRIPT) nim testIntegration $(NIM_PARAMS) build.nims
+		$(ENV_SCRIPT) nim testIntegration $(TEST_PARAMS) $(NIM_PARAMS) build.nims
 
 # Builds and runs all tests (except for Taiko L2 tests)
 testAll: | build deps

build.nims

@@ -1,8 +1,13 @@
 mode = ScriptMode.Verbose
 
 import std/os except commandLineParams
+import std/strutils
 
 ### Helper functions
+proc truthy(val: string): bool =
+  const truthySwitches = @["yes", "1", "on", "true"]
+  return val in truthySwitches
+
 proc buildBinary(name: string, srcDir = "./", params = "", lang = "c") =
   if not dirExists "build":
     mkDir "build"
@@ -45,11 +50,18 @@ task testContracts, "Build & run Codex Contract tests":
 task testIntegration, "Run integration tests":
   buildBinary "codex",
     params =
-      "-d:chronicles_runtime_filtering -d:chronicles_log_level=TRACE -d:codex_enable_proof_failures=true"
-  test "testIntegration"
+      "-d:chronicles_runtime_filtering -d:chronicles_log_level=TRACE -d:chronicles_disabled_topics=JSONRPC-HTTP-CLIENT,websock,libp2p,discv5 -d:codex_enable_proof_failures=true"
+  var sinks = @["textlines[nocolors,file]"]
+  for i in 2 ..< paramCount():
+    if "DebugTestHarness" in paramStr(i) and truthy paramStr(i).split('=')[1]:
+      sinks.add "textlines[stdout]"
+      break
+  var testParams =
+    "-d:chronicles_log_level=TRACE -d:chronicles_sinks=\"" & sinks.join(",") & "\""
+  test "testIntegration", params = testParams
   # use params to enable logging from the integration test executable
   # test "testIntegration", params = "-d:chronicles_sinks=textlines[notimestamps,stdout],textlines[dynamic] " &
-  #   "-d:chronicles_enabled_topics:integration:TRACE"  
+  #   "-d:chronicles_enabled_topics:integration:TRACE"
 
 task build, "build codex binary":
   codexTask()

codex/blockexchange/engine/engine.nim

@@ -68,6 +68,12 @@ const
   DefaultMaxPeersPerRequest* = 10
   DefaultTaskQueueSize = 100
   DefaultConcurrentTasks = 10
+    # DefaultMaxRetries = 3
+    # DefaultConcurrentDiscRequests = 10
+    # DefaultConcurrentAdvertRequests = 10
+    # DefaultDiscoveryTimeout = 1.minutes
+    # DefaultMaxQueriedBlocksCache = 1000
+    # DefaultMinPeersPerBlock = 3
 
 type
   TaskHandler* = proc(task: BlockExcPeerCtx): Future[void] {.gcsafe.}

codex/logutils.nim

@@ -92,6 +92,7 @@ import std/sugar
 import std/typetraits
 
 import pkg/chronicles except toJson, `%`
+from pkg/chronos import TransportAddress
 from pkg/libp2p import Cid, MultiAddress, `$`
 import pkg/questionable
 import pkg/questionable/results
@@ -255,3 +256,5 @@ formatIt(LogFormat.textLines, array[32, byte]):
   it.short0xHexLog
 formatIt(LogFormat.json, array[32, byte]):
   it.to0xHex
+formatIt(TransportAddress):
+  $it

tests/asynctest.nim

@@ -1,3 +1,13 @@
 import pkg/asynctest/chronos/unittest2
 
-export unittest2
+export unittest2 except eventually
+
+template eventuallySafe*(
+    expression: untyped, timeout = 5000, pollInterval = 1000
+): bool =
+  ## More sane defaults, for use with HTTP connections
+  eventually(expression, timeout, pollInterval)
+
+template eventually*(expression: untyped, timeout = 5000, pollInterval = 10): bool =
+  ## Fast defaults, do not use with HTTP connections!
+  eventually(expression, timeout, pollInterval)

tests/codex/slots/sampler/testutils.nim

@@ -77,15 +77,13 @@ asyncchecksuite "Test proof sampler utils":
       )
 
     proc getExpectedIndices(n: int): seq[Natural] =
-      return collect(
-        newSeq,
+      return collect(newSeq):
         (;
           for i in 1 .. n:
             cellIndex(
               proofInput.entropy, proofInput.slotRoot, proofInput.nCellsPerSlot, i
             )
-        ),
-      )
+        )
 
     check:
       slotCellIndices(3) == getExpectedIndices(3)

tests/ethertest.nim

@@ -5,6 +5,8 @@ import pkg/chronos
 import ./asynctest
 import ./checktest
 
+const HardhatPort {.intdefine.}: int = 8545
+
 ## Unit testing suite that sets up an Ethereum testing environment.
 ## Injects a `ethProvider` instance, and a list of `accounts`.
 ## Calls the `evm_snapshot` and `evm_revert` methods to ensure that any
@@ -17,7 +19,7 @@ template ethersuite*(name, body) =
 
     setup:
       ethProvider = JsonRpcProvider.new(
-        "http://127.0.0.1:8545", pollingInterval = chronos.milliseconds(100)
+        "http://127.0.0.1:" & $HardhatPort, pollingInterval = chronos.milliseconds(100)
       )
       snapshot = await send(ethProvider, "evm_snapshot")
       accounts = await ethProvider.listAccounts()

tests/helpers.nim

@@ -4,6 +4,8 @@ import helpers/templeveldb
 import std/times
 import std/sequtils, chronos
 
+import ./asynctest
+
 export multisetup, trackers, templeveldb
 
 ### taken from libp2p errorhelpers.nim

tests/integration/codexconfig.nim

@@ -169,7 +169,8 @@ proc withLogFile*(self: CodexConfigs): CodexConfigs {.raises: [CodexConfigError]
 
 proc withLogFile*(
     self: var CodexConfig, logFile: string
-) {.raises: [CodexConfigError].} = #: CodexConfigs =
+) {.raises: [CodexConfigError].} =
+  #: CodexConfigs =
   ## typically called internally from the test suite, sets a log file path to
   ## be created during the test run, for a specified node in the group
   # var config = self