Merge pull request #453 from openmina/develop

Merge `develop` into `main`

Author: tizoc

.drone.yml

@@ -55,18 +55,16 @@ steps:
   - name: build
     image: rust:1.77-bullseye
     commands:
-      - apt-get update && apt-get install -y libssl-dev libjemalloc-dev jq
+      - apt-get update && apt-get install -y libssl-dev libjemalloc-dev jq protobuf-compiler
       - rustup update 1.77 && rustup default 1.77
       - rustup component add rustfmt
       # just to be sure it builds without errors
       - cargo build
-      - cargo build --release --features p2p-libp2p --bin openmina
+      - cargo build --release --bin openmina
       # unit tests
-      - cargo test --features p2p-libp2p --release -p cli
+      - cargo test --release -p cli
       # build tester binaries
-      - cargo build --release --features scenario-generators --no-default-features -p openmina-node-testing && mv target/release/openmina-node-testing target/release/openmina-node-testing-mio
-      - cargo build --release --features scenario-generators,p2p-libp2p -p openmina-node-testing && mv target/release/openmina-node-testing target/release/openmina-node-testing-libp2p
-      - cargo build --release --features scenario-generators,p2p-libp2p,p2p-webrtc -p openmina-node-testing && mv target/release/openmina-node-testing target/release/openmina-node-testing-webrtc
+      - cargo build --release --features scenario-generators -p openmina-node-testing && mv target/release/openmina-node-testing target/release/openmina-node-testing-mio
       - cargo build --release --bin runner
 
   # o sole mio :)
@@ -333,6 +331,41 @@ steps:
     depends_on:
       - prepare
 
+  - name: block-production-overview-epoch-graphs
+    image: cypress/included:13.3.2
+    commands:
+      - cd frontend && cypress run --config-file "cypress.config.js" --spec "cypress/e2e/block-production/overview/epoch-graphs.cy.ts"
+    depends_on:
+      - prepare
+
+  - name: block-production-overview-slots
+    image: cypress/included:13.3.2
+    commands:
+      - cd frontend && cypress run --config-file "cypress.config.js" --spec "cypress/e2e/block-production/overview/slots.cy.ts"
+    depends_on:
+      - prepare
+
+  - name: block-production-overview-side-panel
+    image: cypress/included:13.3.2
+    commands:
+      - cd frontend && cypress run --config-file "cypress.config.js" --spec "cypress/e2e/block-production/overview/side-panel.cy.ts"
+    depends_on:
+      - prepare
+
+  - name: block-production-overview-toolbar
+    image: cypress/included:13.3.2
+    commands:
+      - cd frontend && cypress run --config-file "cypress.config.js" --spec "cypress/e2e/block-production/overview/toolbar.cy.ts"
+    depends_on:
+      - prepare
+
+  - name: block-production-overview-apis
+    image: cypress/included:13.3.2
+    commands:
+      - cd frontend && cypress run --config-file "cypress.config.js" --spec "cypress/e2e/block-production/overview/apis.cy.ts"
+    depends_on:
+      - prepare
+
   - name: nodes-overview-table
     image: cypress/included:13.3.2
     commands:
@@ -460,6 +493,11 @@ steps:
         from_secret: docker_hub_password
     depends_on:
       - dashboard-peers
+      - block-production-overview-epoch-graphs
+      - block-production-overview-slots
+      - block-production-overview-side-panel
+      - block-production-overview-toolbar
+      - block-production-overview-apis
       - nodes-overview-table
       - nodes-overview-side-panel
       - nodes-live-blocks-map

.github/workflows/ci.yaml

@@ -9,6 +9,9 @@ concurrency:
   group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
   cancel-in-progress: true
 
+env:
+  RUST_MIN_STACK: 10000000
+
 jobs:
   ledger-tests:
     runs-on: ubuntu-20.04
@@ -17,6 +20,10 @@ jobs:
     steps:
       - name: Git checkout
         uses: actions/checkout@v4
+      - name: Setup build dependencies
+        run: |
+          sudo apt update
+          sudo apt install -y protobuf-compiler
       - name: Setup Rust
         run: |
           # Nightly to be able to use `--report-time` below
@@ -41,6 +48,11 @@ jobs:
       - name: Git checkout
         uses: actions/checkout@v4
 
+      - name: Setup build dependencies
+        run: |
+          sudo apt update
+          sudo apt install -y protobuf-compiler
+
       - name: Setup Rust
         run: |
           rustup default 1.77
@@ -62,6 +74,11 @@ jobs:
       - name: Git checkout
         uses: actions/checkout@v4
 
+      - name: Setup build dependencies
+        run: |
+          sudo apt update
+          sudo apt install -y protobuf-compiler
+
       - name: Setup Rust
         run: |
           rustup default 1.77
@@ -88,6 +105,11 @@ jobs:
       - name: Git checkout
         uses: actions/checkout@v4
 
+      - name: Setup build dependencies
+        run: |
+          sudo apt update
+          sudo apt install -y protobuf-compiler
+
       - name: Setup Rust
         run: |
           rustup default 1.77
@@ -115,10 +137,16 @@ jobs:
 
   build-tests-webrtc:
     runs-on: ubuntu-20.04
+    if: false
     steps:
       - name: Git checkout
         uses: actions/checkout@v4
 
+      - name: Setup build dependencies
+        run: |
+          sudo apt update
+          sudo apt install -y protobuf-compiler
+
       - name: Setup Rust
         run: |
           rustup default 1.77
@@ -151,7 +179,7 @@ jobs:
       image: minaprotocol/mina-daemon:2.0.0berkeley-rc1-1551e2f-focal-berkeley
     strategy:
       matrix:
-        test: [p2p_basic_connections, p2p_basic_incoming, p2p_basic_outgoing]
+        test: [p2p_basic_connections, p2p_basic_incoming, p2p_basic_outgoing, p2p_pubsub]
       fail-fast: false
 
     steps:
@@ -168,19 +196,51 @@ jobs:
         run: |
           ./${{ matrix.test }} --test-threads=1
 
+  k8s-peers:
+    runs-on: ubuntu-20.04
+    # TODO: query cluster for actual addresses, or specify then on deployment
+    env:
+      PEERS: |
+        /ip4/135.181.217.23/tcp/31881/p2p/12D3KooWKiKQkUTTZ6u8HDk471ZQE2vStpEDnySiKDrYgZGfEcsF
+        /ip4/135.181.217.23/tcp/30386/p2p/12D3KooWBHknA3ty2tk4u3N6Rwnjs5YwYaxHT9dURf7zGeBJ4VJG
+        /ip4/135.181.217.23/tcp/32272/p2p/12D3KooWDZDH6ucaZY4bU2R8Xy9T1iy4PGtaZ8vEWeuGEeLaQEg5
+        /ip4/135.181.217.23/tcp/32218/p2p/12D3KooWMenq7gPcF7utkZao8WLtjMqsZkY7LeLD1Y9q9YyQZnkG
+        /ip4/135.181.217.23/tcp/30798/p2p/12D3KooWMMsF7dYbn4Szf7TvrRofuK53W3zj6TDKgxAFSWYypLsB
+        /ip4/135.181.217.23/tcp/31631/p2p/12D3KooWNeaaS6wpPSa41qkxQWaeauDR51WCNnLUyd5CcjBBiNTq
+        /ip4/135.181.217.23/tcp/30196/p2p/12D3KooWCBTaSVMxDT1SUoeSg3PUsVL1Df1hDxkSvN544i7tsE76
+        /ip4/135.181.217.23/tcp/30790/p2p/12D3KooWG7egucvX5sompG2euhaPL4QQ3BdHJrLS7HGWyiHyoD5y
+        /ip4/135.181.217.23/tcp/30070/p2p/12D3KooWFG5dhmgS6rb8NdoNbiK7Vf6TqTQDTYxq18DVkmPwsgfg
+    outputs:
+      peers: ${{ steps.peers.outputs.peers }}
+    steps:
+      - id: peers
+        name: Fetch k8s OCaml peers
+        run: |
+          {
+            echo 'peers<<EOF'
+            echo $PEERS
+            echo EOF
+          } >> "$GITHUB_OUTPUT"
+
   scenario-tests:
-    if: false
-    needs: [ build-tests, build-tests-webrtc ]
+    needs:
+      - k8s-peers
+      - build-tests
+    #  - build-tests-webrtc
     runs-on: ubuntu-20.04
     container:
       image: minaprotocol/mina-daemon:2.0.0berkeley-rc1-1551e2f-focal-berkeley
+    env:
+      # to allow local addrs discovery
+      OPENMINA_DISCOVERY_FILTER_ADDR: false
+      OPENMINA_SCENARIO_SEEDS: ${{ needs.k8s-peers.outputs.peers }}
     strategy:
       matrix:
         test:
           - single_node
           - multi_node
-          - webrtc_single_node
-          - webrtc_multi_node
+          # - webrtc_single_node
+          # - webrtc_multi_node
           # - connection_discovery
       fail-fast: false
 
@@ -201,41 +261,51 @@ jobs:
 
 
   bootstrap-test:
-    needs: [ build ]
+    needs: [ k8s-peers, build ]
     runs-on: ubuntu-20.04
+    env:
+      PEERS: ${{ needs.k8s-peers.outputs.peers }}
     steps:
       - name: Download binary
         uses: actions/download-artifact@v4
         with:
           name: bin
 
-      - name: Prepare peers list
-        run: |
-          cat > peers.txt <<EOF
-          /ip4/135.181.217.23/tcp/31881/p2p/12D3KooWL7TyJx6TZZVKqPKkjNXXovu4y7aRQaWEx3eaiY45Lqw8
-          /ip4/135.181.217.23/tcp/30386/p2p/12D3KooWQywRHUxh4bjvPSDKbrY8GeUzeGgHxKbesrJYpgqvaicS
-          /ip4/135.181.217.23/tcp/32272/p2p/12D3KooWA4Mcr2Kn6ivtqFxWcFdLxGW2gqdDVwS86kToUCYst2rj
-          /ip4/135.181.217.23/tcp/32218/p2p/12D3KooWKWRfqG8QdkpAVY3xUzxndPBgj5WR6fxBNAJKSzek3URZ
-          /ip4/135.181.217.23/tcp/30798/p2p/12D3KooWJXiYRpj1dCLPv2PMLyNTuWSgzavyKXyaVsmBRhnBVCNM
-          /ip4/135.181.217.23/tcp/31631/p2p/12D3KooWNeaaS6wpPSa41qkxQWaeauDR51WCNnLUyd5CcjBBiNTq
-          /ip4/135.181.217.23/tcp/30196/p2p/12D3KooWDdJEVxdrthmuBiEL2Yi6nnryo4UYxz1f98TiykyFBx7H
-          /ip4/135.181.217.23/tcp/30790/p2p/12D3KooWN8wmWdivGuUG7fTkV34hihgC6RnrfygzbZCuaQceaiak
-          /ip4/135.181.217.23/tcp/30070/p2p/12D3KooWNi6nauXV3bMtwkc3wqA7V79Um92muVJ5pKvC11wVo2xk
-          EOF
-          echo "::group::Peers for bottstrapping"
-          cat peers.txt
-          echo "::endgroup::"
-
       - name: Bootstrap node
         run: |
           set -eu
           chmod +x ./openmina
-          ./openmina node --no-peers-discovery --peers $(cat peers.txt) |& tee run.log &
+          ./openmina node --no-peers-discovery --peers $PEERS 2>&1 | tee run.log &
           PID=$!
           TIME=10
           SLEEP=10
           ATTEMPTS=$((TIME * 60 / SLEEP))
-          while ! curl -fs localhost:3000/readyz; do
+
+          healthy() {
+            curl -fs -m 5 localhost:3000/healthz
+          }
+
+          ready() {
+            curl -fs -m 5 localhost:3000/readyz
+          }
+
+          ready3() {
+            ready && sleep 10 && ready && sleep 10 && ready
+          }
+
+          while ! healthy; do
+              echo "::notice::The node is not healthy yet"
+              ATTEMPTS=$((ATTEMPTS-1))
+              if [ "${ATTEMPTS}" -eq 0 ]; then
+                  echo "::error::Cannot bootsrtap within $TIME minutes"
+                  exit 1
+              fi
+              sleep $SLEEP
+          done
+          echo "::notice::The node is healthy"
+
+          while ! ready3; do
+              echo "::notice::The node is not bootstrapped yet"
               ATTEMPTS=$((ATTEMPTS-1))
               if [ "${ATTEMPTS}" -eq 0 ]; then
                   echo "::error::Cannot bootsrtap within $TIME minutes"

.github/workflows/docker.yaml

@@ -46,6 +46,7 @@ jobs:
         uses: docker/build-push-action@v5
         with:
           context: .
+          platforms: linux/amd64,linux/arm64
           push: true
           tags: ${{ steps.meta.outputs.tags }}
           cache-from: type=gha

.github/workflows/lint.yml

@@ -11,6 +11,10 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
+      - name: Setup build dependencies
+        run: |
+          sudo apt update
+          sudo apt install -y protobuf-compiler
       - uses: actions-rs/toolchain@v1
         with:
           toolchain: 1.77
@@ -21,8 +25,15 @@ jobs:
         with:
           command: fmt
           args: --all -- --check
+      - uses: actions-rs/cargo@v1
+        name: check
+        with:
+          command: check
+          args: --all-targets
+        env:
+          RUSTFLAGS: -D warnings
       - uses: actions-rs/clippy-check@v1
         name: clippy
         with:
           token: ${{ secrets.GITHUB_TOKEN }}
-          args: --all-targets -- -A clippy::too_many_arguments -A clippy::doc_markdown
+          args: --all-targets -- -D warnings

CHANGELOG.md

@@ -7,6 +7,29 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+## [0.5.0] - 2024-05-31
+
+### Fixed
+
+- When applying blocks, use the `supercharge_coinbase` value from the block which was being ignored before.
+- Incorrect stream being used for RPC responses.
+- Allow multiple nodes running on the same host to connect to each other.
+- Invalid `delta_block_chain_proof` in block producer.
+- Various p2p layer fixes.
+
+### Added
+
+- Support for PubSub in the p2p layer.
+- Block producer dashboard, and simulator-based demo.
+- Support for parsing `daemon.json` files with custom genesis ledgers.
+- Chain ID computation (was hardcoded before).
+- Multiple RPC and p2p tests.
+- More limits to p2p messages, connections, and parsing.
+
+### Removed
+
+- Support for v1 messages in p2p layer.
+
 ## [0.4.0] - 2024-04-30
 
 ### Fixed