diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml deleted file mode 100644 index e19e3217f..000000000 --- a/.github/workflows/ci.yml +++ /dev/null @@ -1,181 +0,0 @@ -name: ci - -permissions: - contents: read - packages: read - -on: - push: - branches: - - main - pull_request: - types: - - opened - - synchronize - - reopened - - ready_for_review - -env: - CARGO_TERM_COLOR: always - GITHUB_ACTOR: pop-cli - CARGO_INCREMENTAL: 1 - RUST_BACKTRACE: 1 - # It is important to always use the same flags. Otherwise, the cache will not work. - RUSTFLAGS: "-Dwarnings" - RUSTDOCFLAGS: "-Dwarnings" - -concurrency: - # Cancel any in-progress jobs for the same pull request or branch - group: ci-${{ github.head_ref || github.ref }} - cancel-in-progress: true - -jobs: - setup: - runs-on: ubuntu-latest - outputs: - image: ${{ steps.out.outputs.image }} - rust_version: ${{ steps.out.outputs.rust_version }} - steps: - - uses: actions/checkout@v4 - - - name: Set environment variables - id: out - env: - HASH: ${{ hashFiles('Dockerfile.ci') }} - run: | - set -xeuo pipefail - RUST_VERSION=$(yq -r '.toolchain.channel' rust-toolchain.toml) - echo "rust_version=$RUST_VERSION" >> "$GITHUB_OUTPUT" - echo "image=ghcr.io/${GITHUB_REPOSITORY,,}:${RUST_VERSION}-${HASH}" >> "$GITHUB_OUTPUT" - - prepare-ci-image: - needs: - - setup - uses: ./.github/workflows/docker-ci.yml - permissions: - contents: read - packages: write - with: - docker_image: ${{ needs.setup.outputs.image }} - rust_version: ${{ needs.setup.outputs.rust_version }} - - lint: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - - name: Install Rust nightly - uses: actions-rs/toolchain@v1 - with: - toolchain: nightly - profile: minimal - components: rustfmt - override: true - - - name: Check formatting - run: cargo +nightly fmt --all -- --check - - build: - needs: - - setup - - lint - - prepare-ci-image - runs-on: ubuntu-latest - container: - image: ${{ needs.setup.outputs.image }} - steps: - - uses: actions/checkout@v4 - - - name: Rust Cache - uses: Swatinem/rust-cache@v2 - with: - cache-all-crates: true - shared-key: shared-${{ github.head_ref || github.ref_name }} - - - name: Check no default features - run: cargo check --locked --no-default-features - - - name: Check contract feature - run: cargo check --locked --no-default-features --features contract - - - name: Check chain feature - run: cargo check --locked --no-default-features --features chain - - - name: Check default features - run: cargo check --locked - - clippy: - needs: - - build - - setup - runs-on: ubuntu-latest - container: - image: ${{ needs.setup.outputs.image }} - permissions: - checks: write - packages: read - steps: - - uses: actions/checkout@v4 - - - name: Rust Cache - uses: Swatinem/rust-cache@v2 - with: - save-if: false - shared-key: shared-${{ github.head_ref || github.ref_name }} - - - name: Annotate with Clippy warnings - uses: actions-rs/clippy-check@v1 - with: - token: ${{ secrets.GITHUB_TOKEN }} - args: --all-targets -- -D warnings - - docker: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - - uses: docker/build-push-action@v5 - - docs: - needs: - - setup - - build - runs-on: ubuntu-latest - container: - image: ${{ needs.setup.outputs.image }} - permissions: - checks: write - packages: read - env: - # We cannot propagate the "mising_docs" flag, as otherwise other tests fail. - RUSTFLAGS: "-Dwarnings -Dmissing_docs" - steps: - - uses: actions/checkout@v4 - - - name: Rust Cache - uses: Swatinem/rust-cache@v2 - with: - save-if: false - shared-key: shared-docs-${{ github.head_ref || github.ref_name }} - - - name: Check no default features - run: cargo doc --locked --no-deps - - documentation-tests: - needs: - - setup - - build - runs-on: ubuntu-latest - container: - image: ${{ needs.setup.outputs.image }} - steps: - - uses: actions/checkout@v4 - - - name: Rust Cache - uses: Swatinem/rust-cache@v2 - with: - save-if: false - shared-key: shared-${{ github.head_ref || github.ref_name }} - - - name: Run doc tests - run: cargo test --locked --doc diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml deleted file mode 100644 index 9b259ff5a..000000000 --- a/.github/workflows/coverage.yml +++ /dev/null @@ -1,85 +0,0 @@ -name: coverage - -permissions: - contents: read - packages: read - -on: - push: - branches: - - main - pull_request: - types: - - opened - - synchronize - - reopened - - ready_for_review - -env: - CARGO_TERM_COLOR: always - GITHUB_ACTOR: pop-cli - CARGO_INCREMENTAL: 1 - RUST_BACKTRACE: 1 - RUSTFLAGS: "-Dwarnings" - -concurrency: - # Cancel any in-progress jobs for the same pull request or branch - group: coverage-${{ github.head_ref || github.ref }} - cancel-in-progress: true - -jobs: - coverage: - runs-on: ubuntu-latest - steps: - - name: Setup Ubuntu dependencies - shell: bash - run: | - sudo apt update - sudo apt install -y protobuf-compiler - - - name: Free up space on runner - shell: bash - run: | - set -xeuo pipefail - sudo rm -rf /usr/local/lib/android & - sudo rm -rf /usr/share/dotnet & - sudo rm -rf /usr/share/swift & - if command -v docker &> /dev/null; then - sudo docker image prune -af & - fi - sudo apt-get clean & - wait - - - uses: actions/checkout@v4 - - - name: Rust Cache - uses: Swatinem/rust-cache@v2 - with: - cache-all-crates: true - cache-on-failure: true - shared-key: shared-coverage-${{ github.head_ref || github.ref_name }} - - - name: Install cargo-llvm-cov - uses: taiki-e/install-action@cargo-llvm-cov - - - name: Install cargo-nextest - uses: taiki-e/install-action@cargo-nextest - - - name: Generate code coverage - run: | - cargo llvm-cov nextest --locked --workspace --lib --bins --codecov --output-path codecov.json --no-fail-fast --nocapture --status-level all - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - - name: Upload to codecov.io - uses: codecov/codecov-action@v4 - with: - token: ${{ secrets.CODECOV_TOKEN }} - files: codecov.json - fail_ci_if_error: true - - - name: Clean up - shell: bash - if: always() - run: | - cargo llvm-cov clean --workspace --profraw-only diff --git a/.github/workflows/docker-ci.yml b/.github/workflows/docker-ci.yml deleted file mode 100644 index 747c65cdb..000000000 --- a/.github/workflows/docker-ci.yml +++ /dev/null @@ -1,67 +0,0 @@ -name: docker-ci - -permissions: - contents: read - packages: write - -on: - workflow_dispatch: - inputs: - docker_image: - description: Docker image to build - required: false - type: string - rust_version: - description: Rust version to use - required: true - type: string - workflow_call: - inputs: - docker_image: - description: Docker image to build - required: true - type: string - rust_version: - description: Rust version to use - required: true - type: string - -jobs: - build-ci-image: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - - - name: Login to GHCR - uses: docker/login-action@v3 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Setup - id: setup - run: | - set -xeuo pipefail - DOCKER_IMAGE="${{ inputs.docker_image }}" - echo "DOCKER_IMAGE=$DOCKER_IMAGE" >> $GITHUB_ENV - if docker manifest inspect "$DOCKER_IMAGE" > /dev/null 2>&1; then - echo "exists=true" >> $GITHUB_OUTPUT - else - echo "exists=false" >> $GITHUB_OUTPUT - fi - - - name: Build and push CI image - if: steps.setup.outputs.exists == 'false' - uses: docker/build-push-action@v5 - with: - context: . - file: Dockerfile.ci - push: true - tags: ${{ env.DOCKER_IMAGE }} - cache-from: type=gha - cache-to: type=gha,mode=max - build-args: RUST_TOOLCHAIN=${{ inputs.rust_version }} diff --git a/.github/workflows/install.yml b/.github/workflows/install.yml deleted file mode 100644 index 3398c710f..000000000 --- a/.github/workflows/install.yml +++ /dev/null @@ -1,120 +0,0 @@ -name: pop install - -on: - push: - branches: - - main - paths: - - 'rust-toolchain.toml' - - 'Cargo.lock' - - '.github/workflows/install.yml' - - 'crates/pop-cli/src/commands/install/**' - pull_request: - paths: - - 'rust-toolchain.toml' - - 'Cargo.lock' - - '.github/workflows/install.yml' - - 'crates/pop-cli/src/commands/install/**' - -defaults: - run: - shell: bash - -env: - DO_NOT_TRACK: 1 - -concurrency: - # Cancel any in-progress jobs for the same pull request or branch - group: install-${{ github.head_ref || github.ref }} - cancel-in-progress: true - -jobs: - arch: - runs-on: ubuntu-latest - container: archlinux:latest - steps: - - uses: actions/checkout@v4 - - name: Install prerequisites - run: pacman -Syu --needed --noconfirm cmake curl git base-devel clang protobuf - - name: Install Rust - run: curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y - - name: Install Pop - run: | - . "$HOME/.cargo/env" - cargo install --locked --path ./crates/pop-cli - cp $(which pop) /usr/local/bin/pop - rustup self uninstall -y - - name: Run Pop install - run: | - pop install -y --frontend - debian: - runs-on: ubuntu-latest - container: debian - steps: - - uses: actions/checkout@v4 - - name: Install prerequisites - run: apt-get update && apt-get -y install build-essential cmake curl git clang protobuf-compiler - - name: Install Rust - run: curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y - - name: Install Pop - run: | - . "$HOME/.cargo/env" - cargo install --locked --path ./crates/pop-cli - cp $(which pop) /usr/local/bin/pop - rustup self uninstall -y - - name: Run Pop Install - run: | - pop install -y --frontend - macos: - runs-on: macos-latest - steps: - - uses: actions/checkout@v4 - - name: Install prerequisites - run: | - brew update - brew uninstall --force cmake || true - brew install cmake openssl protobuf - - name: Install Pop - run: | - cargo install --locked --path ./crates/pop-cli - cp $(which pop) /usr/local/bin/pop - brew uninstall rustup - - name: Run Pop Install - run: | - pop install -y --frontend - redhat: - runs-on: ubuntu-latest - container: redhat/ubi8 - steps: - - uses: actions/checkout@v4 - - name: Install prerequisites - run: yum update -y && yum install -y perl perl-IPC-Cmd perl-Time-HiRes perl-Time-Piece clang curl git make cmake protobuf-compiler gcc pkg-config openssl-devel - - name: Install Rust - run: curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y - - name: Install Pop - run: | - . "$HOME/.cargo/env" - cargo install --locked --path ./crates/pop-cli - cp $(which pop) /usr/local/bin/pop - rustup self uninstall -y - - name: Run Pop install - run: | - pop install -y --frontend - ubuntu: - runs-on: ubuntu-latest - container: ubuntu - steps: - - uses: actions/checkout@v4 - - name: Install prerequisites - run: apt-get update && apt-get -y install build-essential cmake curl git clang protobuf-compiler - - name: Install Rust - run: curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y - - name: Install Pop - run: | - . "$HOME/.cargo/env" - cargo install --locked --path ./crates/pop-cli - cp $(which pop) /usr/local/bin/pop - rustup self uninstall -y - - name: Run Pop install - run: | - pop install -y --frontend diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index 0f9ad0545..3dcaf122f 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -30,68 +30,10 @@ concurrency: cancel-in-progress: true jobs: - contract-integration-tests: - strategy: - matrix: - os: - - ubuntu-latest - - macos-latest - runs-on: ${{ matrix.os }} - steps: - - uses: actions/checkout@v4 - - - name: Install stable toolchain - uses: actions-rs/toolchain@v1 - with: - profile: minimal - toolchain: stable - default: true - target: wasm32-unknown-unknown - components: rust-src, clippy - - - name: Rust Cache Linux - if: matrix.os == 'ubuntu-latest' - uses: Swatinem/rust-cache@v2 - with: - save-if: false - shared-key: shared-linux-contract${{ github.head_ref || github.ref_name }} - - - name: Rust Cache MacOS - if: matrix.os == 'macos-latest' - uses: Swatinem/rust-cache@v2 - with: - cache-on-failure: true - cache-all-crates: true - shared-key: shared-macos-contract-${{ github.head_ref || github.ref_name }}-macos - - - name: Install packages (Linux) - if: matrix.os == 'ubuntu-latest' - run: | - sudo apt-get update - sudo apt-get install -y protobuf-compiler - protoc --version - - - name: Install packages (macOS) - if: matrix.os == 'macos-latest' - run: | - brew install protobuf - protoc --version - - - name: Install cargo-nextest - uses: taiki-e/install-action@v2 - with: - tool: cargo-nextest - - - name: Run integration tests - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: cargo nextest run --locked --no-default-features --features "contract,integration-tests" --test contract --no-fail-fast --nocapture --status-level all - chain-integration-tests: strategy: matrix: os: - - ubuntu-latest - macos-latest runs-on: ${{ matrix.os }} steps: @@ -143,4 +85,4 @@ jobs: env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | - cargo nextest run --locked --no-default-features --features "chain,integration-tests" --test chain --test metadata --no-fail-fast --nocapture --status-level all + cargo nextest run --locked --no-default-features --features "chain,integration-tests" --test chain --nocapture parachain_lifecycle diff --git a/.github/workflows/lint-pr.yml b/.github/workflows/lint-pr.yml deleted file mode 100644 index 047030c2a..000000000 --- a/.github/workflows/lint-pr.yml +++ /dev/null @@ -1,20 +0,0 @@ -name: "Lint PR" - -on: - pull_request_target: - types: - - opened - - edited - - synchronize - -permissions: - pull-requests: read - -jobs: - lint: - name: Validate PR title for conventional commit compliance - runs-on: ubuntu-latest - steps: - - uses: amannn/action-semantic-pull-request@v5 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} \ No newline at end of file diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml deleted file mode 100644 index 2f8629394..000000000 --- a/.github/workflows/release.yml +++ /dev/null @@ -1,148 +0,0 @@ -name: pop-cli release - -on: - release: - types: [ published ] - workflow_dispatch: - inputs: - ref: - description: ref to build binary from - required: false - -jobs: - build-node: - runs-on: ${{ matrix.platform.os }} - permissions: - contents: write - strategy: - matrix: - platform: - # Linux - - os: ubuntu-22.04 - target: aarch64-unknown-linux-gnu - - os: ubuntu-22.04 - target: x86_64-unknown-linux-gnu - # macOS - - os: macos-14 - target: aarch64-apple-darwin - - os: macos-14 - target: x86_64-apple-darwin - env: - RUSTFLAGS: "${{ matrix.platform.cpu != '' && format('-C target-cpu={0}', matrix.platform.cpu) || '' }} ${{ matrix.platform.target == 'aarch64-unknown-linux-gnu' && '-C linker=aarch64-linux-gnu-gcc' || '' }}" - path: "target/${{ matrix.platform.target }}/production" - package: "pop-${{ matrix.platform.target }}${{ matrix.platform.cpu != '' && format('-{0}', matrix.platform.cpu) || '' }}.tar.gz" - steps: - - name: Checkout - uses: actions/checkout@v4 - with: - fetch-depth: 0 - ref: ${{ github.event.inputs.ref }} - - - name: Install packages (Linux) - if: contains(matrix.platform.target, 'linux') - run: | - sudo apt-get update - sudo apt-get install -y protobuf-compiler ${{ contains(matrix.platform.target, 'aarch64') && 'crossbuild-essential-arm64' || '' }} - protoc --version - - - name: Install packages (macOS) - if: contains(matrix.platform.target, 'apple') - run: | - brew install protobuf - protoc --version - - - name: Add target - run: rustup target add ${{ matrix.platform.target }} - - - name: Build pop-cli - run: cargo build --locked --profile=production -p pop-cli --target ${{ matrix.platform.target }} - - - name: Package binary (Linux) - if: contains(matrix.platform.target, 'linux') - run: | - cd ${{ env.path }} - sha256sum pop > pop.sha256 - tar -czf ${{ env.package }} pop pop.sha256 - - - name: Package binary (macOS) - if: contains(matrix.platform.target, 'apple') - run: | - cd ${{ env.path }} - shasum -a 256 pop > pop.sha256 - tar -czf ${{ env.package }} pop pop.sha256 - - - name: Upload binary - uses: actions/upload-artifact@v4 - with: - name: binaries-${{ strategy.job-index }} - path: ${{ env.path }}/${{ env.package }} - - - name: Add binary to release - if: github.event_name == 'release' - uses: softprops/action-gh-release@v1 - with: - files: | - ${{ env.path }}/${{ env.package }} - - build-debian: - name: Build Debian Package - runs-on: ubuntu-22.04 - permissions: - contents: write - if: github.event_name == 'release' - steps: - - name: Checkout - uses: actions/checkout@v4 - with: - fetch-depth: 0 - ref: ${{ github.event.inputs.ref }} - - - name: Install build dependencies - run: | - sudo apt-get update - sudo apt-get install -y curl git debhelper libssl-dev pkg-config protobuf-compiler - curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y - - - name: Build Debian package - run: | - . "$HOME/.cargo/env" - dpkg-buildpackage -us -uc -d - - - name: Upload Debian package artifact - uses: actions/upload-artifact@v4 - with: - name: pop-cli-debian-package - path: ${{ github.workspace }}/../pop-cli_*.deb - - - name: Upload Debian package to release - uses: softprops/action-gh-release@v1 - with: - files: | - ${{ github.workspace }}../pop-cli_*.deb - - publish-homebrew: - name: Publish to Homebrew - runs-on: ubuntu-latest - permissions: - contents: read - needs: build-node - if: github.event_name == 'release' - steps: - - name: Extract version - id: extract_version - run: | - echo "VERSION=${GITHUB_REF#refs/tags/v}" >> $GITHUB_ENV - - - name: Checkout tap repository - uses: actions/checkout@v4 - with: - repository: ${{ github.repository_owner }}/homebrew-pop-cli - token: ${{ secrets.HOMEBREW_TAP_TOKEN }} - path: homebrew-tap - - - name: Update the formula - run: | - cd homebrew-tap - git config user.name "github-actions[bot]" - git config user.email "github-actions[bot]@users.noreply.github.com" - ./update.sh "${VERSION}" diff --git a/crates/pop-chains/README.md b/crates/pop-chains/README.md index 46edc2cfb..211a4aae8 100644 --- a/crates/pop-chains/README.md +++ b/crates/pop-chains/README.md @@ -121,9 +121,9 @@ tokio_test::block_on(async { let release = true; // Whether the binary should be built using the release profile. let status = {}; // Mechanism to observe status updates let verbose = false; // Whether verbose output is required - let missing = zombienet.binaries(); - for binary in missing { - binary.source(release, &status, verbose).await; + let missing = zombienet.archives(); + for archive in missing { + archive.source(release, &status, verbose).await; } }) ``` diff --git a/crates/pop-chains/src/bench/binary.rs b/crates/pop-chains/src/bench/binary.rs index 6c6cb9c36..378f40535 100644 --- a/crates/pop-chains/src/bench/binary.rs +++ b/crates/pop-chains/src/bench/binary.rs @@ -5,9 +5,9 @@ use pop_common::{ git::GitHub, polkadot_sdk::sort_by_latest_stable_version, sourcing::{ - ArchiveFileSpec, Binary, + ArchiveFileSpec, ArchiveType, GitHub::*, - Source, + Source, SourcedArchive, filters::prefix, traits::{ Source as SourceT, @@ -36,7 +36,7 @@ impl SourceT for BenchmarkingCli { fn source(&self) -> Result { // Source from GitHub release asset let repo = GitHub::parse(self.repository())?; - let binary = self.binary(); + let binary = self.binary()?; Ok(Source::GitHub(ReleaseArchive { owner: repo.org, repository: repo.name, @@ -60,15 +60,20 @@ impl SourceT for BenchmarkingCli { pub async fn omni_bencher_generator( cache: PathBuf, version: Option<&str>, -) -> Result { +) -> Result { let cli = BenchmarkingCli::OmniBencher; - let name = cli.binary().to_string(); + let name = cli.binary()?.to_string(); let source = cli .source()? .resolve(&name, version, cache.as_path(), |f| prefix(f, &name)) .await .into(); - let binary = Binary::Source { name, source, cache: cache.to_path_buf() }; + let binary = SourcedArchive::Source { + name, + source, + cache: cache.to_path_buf(), + archive_type: ArchiveType::Binary, + }; Ok(binary) } @@ -83,7 +88,7 @@ mod tests { let temp_dir_path = temp_dir.path().to_path_buf(); let version = "polkadot-stable2412-4"; let binary = omni_bencher_generator(temp_dir_path.clone(), Some(version)).await?; - assert!(matches!(binary, Binary::Source { name: _, source, cache } + assert!(matches!(binary, SourcedArchive::Source { name: _, source, cache, archive_type } if source == Source::GitHub(ReleaseArchive { owner: "r0gue-io".to_string(), repository: "polkadot".to_string(), @@ -96,7 +101,7 @@ mod tests { contents: ["frame-omni-bencher"].map(|b| ArchiveFileSpec::new(b.into(), Some(b.into()), true)).to_vec(), latest: binary.latest().map(|l| l.to_string()), }).into() && - cache == temp_dir_path.as_path() + cache == temp_dir_path.as_path() && archive_type == ArchiveType::Binary )); Ok(()) } diff --git a/crates/pop-chains/src/build/mod.rs b/crates/pop-chains/src/build/mod.rs index fe82556a2..23e785140 100644 --- a/crates/pop-chains/src/build/mod.rs +++ b/crates/pop-chains/src/build/mod.rs @@ -894,12 +894,12 @@ edition = "2021" Some(&vec!["https://github.com/r0gue-io/pop-node#node-v0.3.0".to_string()]), ) .await?; - let mut binary_name: String = "".to_string(); - for binary in zombienet.binaries().filter(|b| !b.exists() && b.name() == "pop-node") { - binary_name = format!("{}-{}", binary.name(), binary.version().unwrap()); - binary.source(true, &(), true).await?; + let mut archive_name: String = "".to_string(); + for archive in zombienet.archives().filter(|b| !b.exists() && b.name() == "pop-node") { + archive_name = format!("{}-{}", archive.name(), archive.version().unwrap()); + archive.source(true, &(), true).await?; } - Ok(binary_name) + Ok(archive_name) } // Replace the binary fetched with the mocked binary diff --git a/crates/pop-chains/src/omni_node.rs b/crates/pop-chains/src/omni_node.rs index 3ad2ca62e..fafeacf09 100644 --- a/crates/pop-chains/src/omni_node.rs +++ b/crates/pop-chains/src/omni_node.rs @@ -3,9 +3,9 @@ use pop_common::{ git::GitHub, polkadot_sdk::sort_by_latest_semantic_version, sourcing::{ - ArchiveFileSpec, Binary, + ArchiveFileSpec, ArchiveType, GitHub::*, - Source, + Source, SourcedArchive, filters::prefix, traits::{ Source as SourceT, @@ -37,7 +37,7 @@ impl SourceT for PolkadotOmniNodeCli { fn source(&self) -> Result { // Source from GitHub release asset let repo = GitHub::parse(self.repository())?; - let binary = self.binary(); + let binary = self.binary()?; Ok(Source::GitHub(ReleaseArchive { owner: repo.org, repository: repo.name, @@ -61,15 +61,20 @@ impl SourceT for PolkadotOmniNodeCli { pub async fn polkadot_omni_node_generator( cache: PathBuf, version: Option<&str>, -) -> Result { +) -> Result { let cli = PolkadotOmniNodeCli::PolkadotOmniNode; - let name = cli.binary().to_string(); + let name = cli.binary()?.to_string(); let source = cli .source()? .resolve(&name, version, cache.as_path(), |f| prefix(f, &name)) .await .into(); - let binary = Binary::Source { name, source, cache: cache.to_path_buf() }; + let binary = SourcedArchive::Source { + name, + source, + cache: cache.to_path_buf(), + archive_type: ArchiveType::Binary, + }; Ok(binary) } @@ -132,9 +137,10 @@ mod tests { let binary = polkadot_omni_node_generator(cache.path().to_path_buf(), None).await?; match binary { - Binary::Source { name, source, cache: cache_path } => { + SourcedArchive::Source { name, source, cache: cache_path, archive_type } => { assert_eq!(name, "polkadot-omni-node"); assert_eq!(cache_path, cache.path()); + assert_eq!(archive_type, ArchiveType::Binary); // Source should be a ResolvedRelease match *source { Source::GitHub(github) => @@ -144,7 +150,7 @@ mod tests { _ => panic!("Expected GitHub variant"), } }, - _ => panic!("Expected Binary::Source variant"), + _ => panic!("Expected SourcedArchive::Source variant"), } Ok(()) diff --git a/crates/pop-chains/src/try_runtime/binary.rs b/crates/pop-chains/src/try_runtime/binary.rs index b39ca6139..2be57de7c 100644 --- a/crates/pop-chains/src/try_runtime/binary.rs +++ b/crates/pop-chains/src/try_runtime/binary.rs @@ -5,9 +5,9 @@ use pop_common::{ git::GitHub, polkadot_sdk::sort_by_latest_semantic_version, sourcing::{ - ArchiveFileSpec, Binary, + ArchiveFileSpec, ArchiveType, GitHub::*, - Source, + Source, SourcedArchive, filters::prefix, traits::{ Source as SourceT, @@ -35,7 +35,7 @@ impl SourceT for TryRuntimeCli { fn source(&self) -> Result { // Source from GitHub release asset let repo = GitHub::parse(self.repository())?; - let binary = self.binary(); + let binary = self.binary()?; Ok(Source::GitHub(ReleaseArchive { owner: repo.org, repository: repo.name, @@ -56,15 +56,23 @@ impl SourceT for TryRuntimeCli { /// # Arguments /// * `cache` - The path to the directory where the binary should be cached. /// * `version` - An optional version string. If `None`, the latest available version is used. -pub async fn try_runtime_generator(cache: PathBuf, version: Option<&str>) -> Result { +pub async fn try_runtime_generator( + cache: PathBuf, + version: Option<&str>, +) -> Result { let cli = TryRuntimeCli::TryRuntime; - let name = cli.binary().to_string(); + let name = cli.binary()?.to_string(); let source = cli .source()? .resolve(&name, version, cache.as_path(), |f| prefix(f, &name)) .await .into(); - let binary = Binary::Source { name, source, cache: cache.to_path_buf() }; + let binary = SourcedArchive::Source { + name, + source, + cache: cache.to_path_buf(), + archive_type: ArchiveType::Binary, + }; Ok(binary) } @@ -79,7 +87,7 @@ mod tests { let path = temp_dir.path().to_path_buf(); let version = "v0.8.0"; let binary = try_runtime_generator(path.clone(), None).await?; - assert!(matches!(binary, Binary::Source { name: _, source, cache } + assert!(matches!(binary, SourcedArchive::Source { name: _, source, cache, archive_type} if source == Source::GitHub(ReleaseArchive { owner: "r0gue-io".to_string(), repository: "try-runtime-cli".to_string(), @@ -92,7 +100,7 @@ mod tests { contents: ["try-runtime-cli"].map(|b| ArchiveFileSpec::new(b.into(), Some(b.into()), true)).to_vec(), latest: binary.latest().map(|l| l.to_string()), }).into() && - cache == path + cache == path && archive_type == ArchiveType::Binary )); Ok(()) } diff --git a/crates/pop-chains/src/up/chain_specs.rs b/crates/pop-chains/src/up/chain_specs.rs index 650e73e98..51a2039a5 100644 --- a/crates/pop-chains/src/up/chain_specs.rs +++ b/crates/pop-chains/src/up/chain_specs.rs @@ -5,9 +5,9 @@ use pop_common::{ git::GitHub, polkadot_sdk::sort_by_latest_semantic_version, sourcing::{ - ArchiveFileSpec, Binary, + ArchiveFileSpec, ArchiveType, GitHub::*, - Source, + Source, SourcedArchive, filters::prefix, traits::{ Source as SourceT, @@ -16,7 +16,7 @@ use pop_common::{ }, target, }; -use std::path::Path; +use std::path::{Path, PathBuf}; use strum::{EnumProperty as _, VariantArray as _}; use strum_macros::{AsRefStr, EnumProperty, VariantArray}; @@ -34,10 +34,10 @@ pub enum Runtime { Kusama = 0, /// Paseo. #[strum(props( - Repository = "https://github.com/r0gue-io/paseo-runtimes", - Binary = "chain-spec-generator", + Repository = "https://github.com/paseo-network/runtimes", + File = "paseo-local", Chain = "paseo-local", - Fallback = "v1.4.1" + Fallback = "v2.0.2" ))] Paseo = 1, /// Polkadot. @@ -60,23 +60,43 @@ impl SourceT for Runtime { // Source from GitHub release asset let repo = GitHub::parse(self.repository())?; let name = self.name().to_lowercase(); - let binary = self.binary(); - Ok(Source::GitHub(ReleaseArchive { - owner: repo.org, - repository: repo.name, - tag: None, - tag_pattern: self.tag_pattern().map(|t| t.into()), - prerelease: false, - version_comparator: sort_by_latest_semantic_version, - fallback: self.fallback().into(), - archive: format!("{binary}-{}.tar.gz", target()?), - contents: vec![ArchiveFileSpec::new( - binary.into(), - Some(format!("{name}-{binary}").into()), - true, - )], - latest: None, - })) + match (self.binary(), self.file()) { + (Ok(binary), Err(_)) => Ok(Source::GitHub(ReleaseArchive { + owner: repo.org, + repository: repo.name, + tag: None, + tag_pattern: self.tag_pattern().map(|t| t.into()), + prerelease: false, + version_comparator: sort_by_latest_semantic_version, + fallback: self.fallback().into(), + archive: format!("{binary}-{}.tar.gz", target()?), + contents: vec![ArchiveFileSpec::new( + binary.into(), + Some(format!("{name}-{binary}").into()), + true, + )], + latest: None, + })), + (Err(_), Ok(file)) => Ok(Source::GitHub(ReleaseArchive { + owner: repo.org, + repository: repo.name, + tag: None, + tag_pattern: self.tag_pattern().map(|t| t.into()), + prerelease: false, + version_comparator: sort_by_latest_semantic_version, + fallback: self.fallback().into(), + archive: format!("{file}.json"), + contents: vec![ArchiveFileSpec::new( + format!("{file}.json"), + Some(file.to_string().into()), + true, + )], + latest: None, + })), + _ => Err(Error::Config( + "Runtime sourcing for chain specs can only contains the chain spec generator or the chain spec file".to_owned(), + )), + } } } @@ -126,24 +146,92 @@ pub(super) async fn chain_spec_generator( chain: &str, version: Option<&str>, cache: &Path, -) -> Result, Error> { +) -> Result, Error> { if let Some(runtime) = Runtime::from_chain(chain) { if runtime == Runtime::Westend { // Westend runtimes included with binary. return Ok(None); } - let name = format!("{}-{}", runtime.name().to_lowercase(), runtime.binary()); + + let binary_name = if let Ok(binary) = runtime.binary() { + binary + } else { + return Ok(None); + }; + let name = format!("{}-{}", runtime.name().to_lowercase(), binary_name); let source = runtime .source()? .resolve(&name, version, cache, |f| prefix(f, &name)) .await .into(); - let binary = Binary::Source { name, source, cache: cache.to_path_buf() }; + let binary = SourcedArchive::Source { + name, + source, + cache: cache.to_path_buf(), + archive_type: ArchiveType::Binary, + }; return Ok(Some(binary)); } Ok(None) } +pub(super) async fn chain_spec_file( + chain: &str, + version: Option<&str>, + cache: &Path, +) -> Result, Error> { + if let Some(runtime) = Runtime::from_chain(chain) { + let file = if let Ok(file) = runtime.file() { + file + } else { + return Ok(None); + }; + + // The File prop name is only valid for the relay chains, we need to use the right + // parachain name for parachains chain specs (differently of chain-spec-generator which was + // unique for all the chains) + let mut name = if chain.contains(file) { + chain.to_owned() + } else { + format!("{}-{}", runtime.name().to_lowercase(), file) + }; + + let mut source = runtime.source()?; + + // In case the File prop isn't the source archive to download (parachain case), we need to + // update the source. + if let Source::GitHub(ReleaseArchive { ref mut archive, ref mut contents, .. }) = source && + let Some(&mut ArchiveFileSpec { + name: ref mut contents_name, ref mut target, .. + }) = contents.first_mut() + { + *target = Some(PathBuf::from(name.clone())); + if let Some(file_extension) = + Path::new(&contents_name.clone()).extension().and_then(|ext| ext.to_str()) + { + *archive = name.clone() + "." + file_extension; + *contents_name = name.clone() + "." + file_extension; + name = name.clone() + "." + file_extension; + } else { + *archive = name.clone(); + *contents_name = name.clone(); + } + } + + let source: Box = + source.resolve(&name, version, cache, |f| prefix(f, &name)).await.into(); + + let chain_spec_file = SourcedArchive::Source { + name, + source, + cache: cache.to_path_buf(), + archive_type: ArchiveType::File, + }; + return Ok(Some(chain_spec_file)); + } + Ok(None) +} + #[cfg(test)] mod tests { use super::*; @@ -153,17 +241,17 @@ mod tests { #[tokio::test] async fn kusama_works() -> anyhow::Result<()> { let expected = Runtime::Kusama; - let version = "v1.4.1"; + let version = "v1.4.1".to_string(); let temp_dir = tempdir()?; - let binary = chain_spec_generator("kusama-local", Some(version), temp_dir.path()) + let binary = chain_spec_generator("kusama-local", Some(&version), temp_dir.path()) .await? .unwrap(); - assert!(matches!(binary, Binary::Source { name, source, cache } - if name == format!("{}-{}", expected.name().to_lowercase(), expected.binary()) && + assert!(matches!(binary, SourcedArchive::Source { name, source, cache, archive_type } + if name == format!("{}-{}", expected.name().to_lowercase(), expected.binary().unwrap()) && source == Source::GitHub(ReleaseArchive { owner: "r0gue-io".to_string(), repository: "polkadot-runtimes".to_string(), - tag: Some(version.to_string()), + tag: Some(version), tag_pattern: None, prerelease: false, version_comparator: sort_by_latest_semantic_version, @@ -172,7 +260,7 @@ mod tests { contents: ["chain-spec-generator"].map(|b| ArchiveFileSpec::new(b.into(), Some(format!("kusama-{b}").into()), true)).to_vec(), latest: binary.latest().map(|l| l.to_string()), }).into() && - cache == temp_dir.path() + cache == temp_dir.path() && archive_type == ArchiveType::Binary )); Ok(()) } @@ -180,25 +268,35 @@ mod tests { #[tokio::test] async fn paseo_works() -> anyhow::Result<()> { let expected = Runtime::Paseo; - let version = "v1.4.1"; + let version = "v2.0.2"; let temp_dir = tempdir()?; - let binary = chain_spec_generator("paseo-local", Some(version), temp_dir.path()) - .await? - .unwrap(); - assert!(matches!(binary, Binary::Source { name, source, cache } - if name == format!("{}-{}", expected.name().to_lowercase(), expected.binary()) && - source == Source::GitHub(ReleaseArchive { - owner: "r0gue-io".to_string(), - repository: "paseo-runtimes".to_string(), - tag: Some(version.to_string()), - tag_pattern: None, - prerelease: false, - version_comparator: sort_by_latest_semantic_version, - fallback: expected.fallback().to_string(), - archive: format!("chain-spec-generator-{}.tar.gz", target()?), - contents: ["chain-spec-generator"].map(|b| ArchiveFileSpec::new(b.into(), Some(format!("paseo-{b}").into()), true)).to_vec(), - latest: binary.latest().map(|l| l.to_string()), - }).into() && + let file = chain_spec_file("paseo-local", Some(version), temp_dir.path()).await?.unwrap(); + assert!(matches!(file, SourcedArchive::Source { name, source, cache, archive_type } + if name == "paseo-local.json" && + archive_type == ArchiveType::File && + matches!(*source, Source::GitHub(ReleaseArchive { + ref owner, + ref repository, + ref tag, + ref tag_pattern, + prerelease, + ref fallback, + ref archive, + ref contents, + .. + }) if owner == "paseo-network" && + repository == "runtimes" && + tag == &Some(version.to_string()) && + tag_pattern.is_none() && + !prerelease && + fallback == expected.fallback() && + archive == "paseo-local.json" && + contents == &vec![ArchiveFileSpec::new( + "paseo-local.json".to_string(), + Some("paseo-local".into()), + true + )] + ) && cache == temp_dir.path() )); Ok(()) @@ -212,8 +310,8 @@ mod tests { let binary = chain_spec_generator("polkadot-local", Some(version), temp_dir.path()) .await? .unwrap(); - assert!(matches!(binary, Binary::Source { name, source, cache } - if name == format!("{}-{}", expected.name().to_lowercase(), expected.binary()) && + assert!(matches!(binary, SourcedArchive::Source { name, source, cache, archive_type } + if name == format!("{}-{}", expected.name().to_lowercase(), expected.binary().unwrap()) && source == Source::GitHub(ReleaseArchive { owner: "r0gue-io".to_string(), repository: "polkadot-runtimes".to_string(), @@ -226,7 +324,8 @@ mod tests { contents: ["chain-spec-generator"].map(|b| ArchiveFileSpec::new(b.into(), Some(format!("polkadot-{b}").into()), true)).to_vec(), latest: binary.latest().map(|l| l.to_string()), }).into() && - cache == temp_dir.path() + cache == temp_dir.path() && + archive_type == ArchiveType::Binary )); Ok(()) } @@ -282,4 +381,154 @@ mod tests { ); } } + + // Tests for chain_spec_file function + + #[tokio::test] + async fn chain_spec_file_paseo_relay_works() -> anyhow::Result<()> { + let expected = Runtime::Paseo; + let version = "v2.0.2"; + let chain = "paseo-local"; + let temp_dir = tempdir()?; + + let file = chain_spec_file(chain, Some(version), temp_dir.path()).await?.unwrap(); + + assert!(matches!(file, SourcedArchive::Source { name, source, cache, archive_type } + if name == "paseo-local.json" && + archive_type == ArchiveType::File && + matches!(*source, Source::GitHub(ReleaseArchive { + ref owner, + ref repository, + ref tag, + ref tag_pattern, + prerelease, + ref fallback, + ref archive, + ref contents, + .. + }) if owner == "paseo-network" && + repository == "runtimes" && + tag == &Some(version.to_string()) && + tag_pattern.is_none() && + !prerelease && + fallback == expected.fallback() && + archive == "paseo-local.json" && + contents == &vec![ArchiveFileSpec::new( + "paseo-local.json".to_string(), + Some("paseo-local".into()), + true + )] + ) && + cache == temp_dir.path() + )); + Ok(()) + } + + #[tokio::test] + async fn chain_spec_file_paseo_parachain_works() -> anyhow::Result<()> { + let expected = Runtime::Paseo; + let version = "v2.0.2"; + let chain = "asset-hub-paseo-local"; + let temp_dir = tempdir()?; + + let file = chain_spec_file(chain, Some(version), temp_dir.path()).await?.unwrap(); + + assert!(matches!(file, SourcedArchive::Source { name, source, cache, archive_type } + if name == "asset-hub-paseo-local.json" && + archive_type == ArchiveType::File && + matches!(*source, Source::GitHub(ReleaseArchive { + ref owner, + ref repository, + ref tag, + ref tag_pattern, + prerelease, + ref fallback, + ref archive, + ref contents, + .. + }) if owner == "paseo-network" && + repository == "runtimes" && + tag == &Some(version.to_string()) && + tag_pattern.is_none() && + !prerelease && + fallback == expected.fallback() && + archive == "asset-hub-paseo-local.json" && + contents == &vec![ArchiveFileSpec::new( + "asset-hub-paseo-local.json".to_string(), + Some("asset-hub-paseo-local".into()), + true + )] + ) && + cache == temp_dir.path() + )); + Ok(()) + } + + #[tokio::test] + async fn chain_spec_file_without_version_works() -> anyhow::Result<()> { + let chain = "asset-hub-paseo-local"; + let temp_dir = tempdir()?; + + let file = chain_spec_file(chain, None, temp_dir.path()).await?.unwrap(); + + // When no version is specified, it should still create the SourcedArchive + // but the tag will be resolved later + assert!(matches!(file, SourcedArchive::Source { name, archive_type, .. } + if name == "asset-hub-paseo-local.json" && + archive_type == ArchiveType::File + )); + Ok(()) + } + + #[tokio::test] + async fn chain_spec_file_returns_none_when_no_match() -> anyhow::Result<()> { + let temp_dir = tempdir()?; + assert_eq!(chain_spec_file("rococo-local", None, temp_dir.path()).await?, None); + assert_eq!(chain_spec_file("unknown-chain", None, temp_dir.path()).await?, None); + Ok(()) + } + + #[tokio::test] + async fn chain_spec_file_returns_none_for_kusama() -> anyhow::Result<()> { + // Kusama uses Binary (chain-spec-generator), not File + let temp_dir = tempdir()?; + assert_eq!(chain_spec_file("kusama-local", None, temp_dir.path()).await?, None); + Ok(()) + } + + #[tokio::test] + async fn chain_spec_file_returns_none_for_polkadot() -> anyhow::Result<()> { + // Polkadot uses Binary (chain-spec-generator), not File + let temp_dir = tempdir()?; + assert_eq!(chain_spec_file("polkadot-local", None, temp_dir.path()).await?, None); + Ok(()) + } + + #[tokio::test] + async fn chain_spec_file_returns_none_for_westend() -> anyhow::Result<()> { + // Westend doesn't have File property + let temp_dir = tempdir()?; + assert_eq!(chain_spec_file("westend-local", None, temp_dir.path()).await?, None); + Ok(()) + } + + #[test] + fn from_chain_matches_parachain_chains() { + // Verify that parachain chain names match to the correct relay runtime + for (parachain_chain, expected_relay) in [ + ("asset-hub-paseo-local", Paseo), + ("bridge-hub-paseo-local", Paseo), + ("coretime-paseo-local", Paseo), + ("people-paseo-local", Paseo), + ("passet-hub-paseo-local", Paseo), + ] { + assert_eq!( + Runtime::from_chain(parachain_chain).unwrap(), + expected_relay, + "Chain '{}' should match to {:?}", + parachain_chain, + expected_relay + ); + } + } } diff --git a/crates/pop-chains/src/up/chains.rs b/crates/pop-chains/src/up/chains.rs index 4d3cb1df8..c89463c31 100644 --- a/crates/pop-chains/src/up/chains.rs +++ b/crates/pop-chains/src/up/chains.rs @@ -1,6 +1,9 @@ // SPDX-License-Identifier: GPL-3.0 -use super::{Binary, Relay, chain_specs::chain_spec_generator}; +use super::{ + ArchiveType, Relay, SourcedArchive, + chain_specs::{chain_spec_file, chain_spec_generator}, +}; use crate::{Error, registry, registry::System, traits::Binary as BinaryT}; use pop_common::sourcing::{filters::prefix, traits::*}; use std::path::Path; @@ -35,12 +38,27 @@ pub(super) async fn system( .resolve(&name, version.or(Some(relay_chain_version)), cache, |f| prefix(f, &name)) .await .into(); - let binary = Binary::Source { name, source, cache: cache.to_path_buf() }; + let binary = SourcedArchive::Source { + name, + source, + cache: cache.to_path_buf(), + archive_type: ArchiveType::Binary, + }; let chain_spec_generator = match chain { Some(chain) => chain_spec_generator(chain, runtime_version, cache).await?, None => None, }; - Ok(Some(super::Chain { id, binary, chain: chain.map(|c| c.to_string()), chain_spec_generator })) + let chain_spec_file = match chain { + Some(chain) => chain_spec_file(chain, runtime_version, cache).await?, + None => None, + }; + Ok(Some(super::Chain { + id, + binary, + chain: chain.map(|c| c.to_string()), + chain_spec_generator, + chain_spec_file, + })) } /// Initializes the configuration required to launch a parachain. @@ -63,12 +81,18 @@ pub(super) async fn from( let name = para.binary().to_string(); let source = para.source()?.resolve(&name, version, cache, |f| prefix(f, &name)).await.into(); - let binary = Binary::Source { name, source, cache: cache.to_path_buf() }; + let binary = SourcedArchive::Source { + name, + source, + cache: cache.to_path_buf(), + archive_type: ArchiveType::Binary, + }; return Ok(Some(super::Chain { id, binary, chain: chain.map(|c| c.to_string()), chain_spec_generator: None, + chain_spec_file: None, })); } Ok(None) @@ -122,20 +146,22 @@ mod tests { .await? .unwrap(); assert_eq!(para_id, parachain.id); - assert!(matches!(parachain.binary, Binary::Source { name, source, cache } - if name == expected.binary() && source == Source::GitHub(ReleaseArchive { - owner: "r0gue-io".to_string(), - repository: "polkadot".to_string(), - tag: Some(format!("polkadot-{RELAY_BINARY_VERSION}")), - tag_pattern: Some("polkadot-{version}".into()), - prerelease: false, - version_comparator: sort_by_latest_stable_version, - fallback: FALLBACK.into(), - archive: format!("{name}-{}.tar.gz", target()?), - contents: vec![ArchiveFileSpec::new(expected.binary().into(), None, true)], - latest: parachain.binary.latest().map(|l| l.to_string()), - }).into() && cache == temp_dir.path() - )); + assert!( + matches!(parachain.binary, SourcedArchive::Source { name, source, cache, archive_type } + if name == expected.binary() && source == Source::GitHub(ReleaseArchive { + owner: "r0gue-io".to_string(), + repository: "polkadot".to_string(), + tag: Some(format!("polkadot-{RELAY_BINARY_VERSION}")), + tag_pattern: Some("polkadot-{version}".into()), + prerelease: false, + version_comparator: sort_by_latest_stable_version, + fallback: FALLBACK.into(), + archive: format!("{name}-{}.tar.gz", target()?), + contents: vec![ArchiveFileSpec::new(expected.binary().into(), None, true)], + latest: parachain.binary.latest().map(|l| l.to_string()), + }).into() && cache == temp_dir.path() && archive_type == ArchiveType::Binary + ) + ); Ok(()) } @@ -157,20 +183,22 @@ mod tests { .await? .unwrap(); assert_eq!(para_id, parachain.id); - assert!(matches!(parachain.binary, Binary::Source { name, source, cache } - if name == expected.binary() && source == Source::GitHub(ReleaseArchive { - owner: "r0gue-io".to_string(), - repository: "polkadot".to_string(), - tag: Some(format!("polkadot-{SYSTEM_PARA_BINARY_VERSION}")), - tag_pattern: Some("polkadot-{version}".into()), - prerelease: false, - version_comparator: sort_by_latest_stable_version, - fallback: FALLBACK.into(), - archive: format!("{name}-{}.tar.gz", target()?), - contents: vec![ArchiveFileSpec::new(expected.binary().into(), None, true)], - latest: parachain.binary.latest().map(|l| l.to_string()), - }).into() && cache == temp_dir.path() - )); + assert!( + matches!(parachain.binary, SourcedArchive::Source { name, source, cache, archive_type } + if name == expected.binary() && source == Source::GitHub(ReleaseArchive { + owner: "r0gue-io".to_string(), + repository: "polkadot".to_string(), + tag: Some(format!("polkadot-{SYSTEM_PARA_BINARY_VERSION}")), + tag_pattern: Some("polkadot-{version}".into()), + prerelease: false, + version_comparator: sort_by_latest_stable_version, + fallback: FALLBACK.into(), + archive: format!("{name}-{}.tar.gz", target()?), + contents: vec![ArchiveFileSpec::new(expected.binary().into(), None, true)], + latest: parachain.binary.latest().map(|l| l.to_string()), + }).into() && cache == temp_dir.path() && archive_type == ArchiveType::Binary + ) + ); Ok(()) } @@ -180,6 +208,46 @@ mod tests { let runtime_version = "v1.3.3"; let para_id = 1000; + let temp_dir = tempdir()?; + let parachain = system( + para_id, + expected.binary(), + None, + Some(runtime_version), + RELAY_BINARY_VERSION, + Some("asset-hub-polkadot-local"), + temp_dir.path(), + ) + .await? + .unwrap(); + assert_eq!(parachain.id, para_id); + assert_eq!(parachain.chain.unwrap(), "asset-hub-polkadot-local"); + let chain_spec_generator = parachain.chain_spec_generator.unwrap(); + assert!( + matches!(chain_spec_generator, SourcedArchive::Source { name, source, cache, archive_type } + if name == "polkadot-chain-spec-generator" && source == Source::GitHub(ReleaseArchive { + owner: "r0gue-io".to_string(), + repository: "polkadot-runtimes".to_string(), + tag: Some(runtime_version.to_string()), + tag_pattern: None, + prerelease: false, + version_comparator: sort_by_latest_semantic_version, + fallback: "v1.4.1".into(), + archive: format!("chain-spec-generator-{}.tar.gz", target()?), + contents: [ArchiveFileSpec::new("chain-spec-generator".into(), Some("polkadot-chain-spec-generator".into()), true)].to_vec(), + latest: chain_spec_generator.latest().map(|l| l.to_string()), + }).into() && cache == temp_dir.path() && archive_type == ArchiveType::Binary + ) + ); + Ok(()) + } + + #[tokio::test] + async fn system_with_chain_spec_file_works() -> anyhow::Result<()> { + let expected = System; + let runtime_version = "v2.0.2"; + let para_id = 1000; + let temp_dir = tempdir()?; let parachain = system( para_id, @@ -194,21 +262,37 @@ mod tests { .unwrap(); assert_eq!(parachain.id, para_id); assert_eq!(parachain.chain.unwrap(), "asset-hub-paseo-local"); - let chain_spec_generator = parachain.chain_spec_generator.unwrap(); - assert!(matches!(chain_spec_generator, Binary::Source { name, source, cache } - if name == "paseo-chain-spec-generator" && source == Source::GitHub(ReleaseArchive { - owner: "r0gue-io".to_string(), - repository: "paseo-runtimes".to_string(), - tag: Some(runtime_version.to_string()), - tag_pattern: None, - prerelease: false, - version_comparator: sort_by_latest_semantic_version, - fallback: "v1.4.1".into(), - archive: format!("chain-spec-generator-{}.tar.gz", target()?), - contents: [ArchiveFileSpec::new("chain-spec-generator".into(), Some("paseo-chain-spec-generator".into()), true)].to_vec(), - latest: chain_spec_generator.latest().map(|l| l.to_string()), - }).into() && cache == temp_dir.path() - )); + let chain_spec_file = parachain.chain_spec_file.unwrap(); + assert!( + matches!(chain_spec_file, SourcedArchive::Source { name, source, cache, archive_type } + if name == "asset-hub-paseo-local.json" && + archive_type == ArchiveType::File && + matches!(*source, Source::GitHub(ReleaseArchive { + ref owner, + ref repository, + ref tag, + ref tag_pattern, + prerelease, + ref fallback, + ref archive, + ref contents, + .. + }) if owner == "paseo-network" && + repository == "runtimes" && + tag == &Some(runtime_version.to_string()) && + tag_pattern.is_none() && + !prerelease && + fallback == "v2.0.2" && + archive == "asset-hub-paseo-local.json" && + contents == &vec![ArchiveFileSpec::new( + "asset-hub-paseo-local.json".to_string(), + Some("asset-hub-paseo-local".into()), + true + )] + ) && + cache == temp_dir.path() + ) + ); Ok(()) } @@ -224,20 +308,22 @@ mod tests { .await? .unwrap(); assert_eq!(para_id, parachain.id); - assert!(matches!(parachain.binary, Binary::Source { name, source, cache } - if name == expected && source == Source::GitHub(ReleaseArchive { - owner: "r0gue-io".to_string(), - repository: "pop-node".to_string(), - tag: Some(format!("node-{version}")), - tag_pattern: Some("node-{version}".into()), - prerelease: false, - version_comparator: sort_by_latest_semantic_version, - fallback: "v0.3.0".into(), - archive: format!("{name}-{}.tar.gz", target()?), - contents: vec![ArchiveFileSpec::new(expected.into(), None, true)], - latest: parachain.binary.latest().map(|l| l.to_string()), - }).into() && cache == temp_dir.path() - )); + assert!( + matches!(parachain.binary, SourcedArchive::Source { name, source, cache, archive_type } + if name == expected && source == Source::GitHub(ReleaseArchive { + owner: "r0gue-io".to_string(), + repository: "pop-node".to_string(), + tag: Some(format!("node-{version}")), + tag_pattern: Some("node-{version}".into()), + prerelease: false, + version_comparator: sort_by_latest_semantic_version, + fallback: "v0.3.0".into(), + archive: format!("{name}-{}.tar.gz", target()?), + contents: vec![ArchiveFileSpec::new(expected.into(), None, true)], + latest: parachain.binary.latest().map(|l| l.to_string()), + }).into() && cache == temp_dir.path() && archive_type == ArchiveType::Binary + ) + ); Ok(()) } diff --git a/crates/pop-chains/src/up/mod.rs b/crates/pop-chains/src/up/mod.rs index 191851fa9..5a07b646d 100644 --- a/crates/pop-chains/src/up/mod.rs +++ b/crates/pop-chains/src/up/mod.rs @@ -11,7 +11,7 @@ use pop_common::sourcing::traits::{Source as _, enums::Source as _}; pub use pop_common::{ Profile, git::{GitHub, Repository}, - sourcing::{Binary, GitHub::*, Source, Source::*}, + sourcing::{ArchiveType, GitHub::*, Source, Source::*, SourcedArchive}, }; use std::{ collections::BTreeSet, @@ -102,16 +102,18 @@ impl Zombienet { Ok(Self { network_config, relay_chain, parachains, hrmp_channels }) } - /// The binaries required to launch the network. - pub fn binaries(&mut self) -> impl Iterator { - once([Some(&mut self.relay_chain.binary), self.relay_chain.chain_spec_generator.as_mut()]) - .chain( - self.parachains - .values_mut() - .map(|p| [Some(&mut p.binary), p.chain_spec_generator.as_mut()]), - ) - .flatten() - .flatten() + /// The archives required to launch the network. + pub fn archives(&mut self) -> impl Iterator { + once([ + Some(&mut self.relay_chain.binary), + self.relay_chain.chain_spec_generator.as_mut(), + self.relay_chain.chain_spec_file.as_mut(), + ]) + .chain(self.parachains.values_mut().map(|p| { + [Some(&mut p.binary), p.chain_spec_generator.as_mut(), p.chain_spec_file.as_mut()] + })) + .flatten() + .flatten() } /// Determine parachain configuration based on specified version and network configuration. @@ -214,7 +216,7 @@ impl Zombienet { return Err(Error::MissingBinary(command)); } - if command.starts_with(PolkadotOmniNode.binary()) { + if command.starts_with(PolkadotOmniNode.binary()?) { paras.insert(id, Chain::from_omni_node(id, cache)?); continue 'outer; } @@ -446,6 +448,10 @@ impl NetworkConfiguration { "{{chainName}}" )), }; + let relay_chain_spec_file = match &relay_chain.chain_spec_file { + None => None, + Some(file) => Some(NetworkConfiguration::resolve_path(&file.path())?), + }; // Use builder to clone network config, adapting binary paths as necessary let mut builder = NetworkConfigBuilder::new() @@ -475,8 +481,10 @@ impl NetworkConfiguration { builder = builder.with_chain_spec_command_output_path(chain_spec_command_output_path); } - // Configure chain spec generator - if let Some(command) = chain_spec_generator { + // Configure chain spec generator or file + if let Some(ref path) = relay_chain_spec_file { + builder = builder.with_chain_spec_path(path.as_str()); + } else if let Some(command) = chain_spec_generator { builder = builder.with_chain_spec_command(command); } // Overrides: genesis/wasm @@ -523,6 +531,10 @@ impl NetworkConfiguration { "{{chainName}}" )), }; + let parachain_chain_spec_file = match ¶.chain_spec_file { + None => None, + Some(file) => Some(NetworkConfiguration::resolve_path(&file.path())?), + }; builder = builder.with_parachain(|builder| { let mut builder = builder @@ -561,8 +573,10 @@ impl NetworkConfiguration { if let Some(location) = source.chain_spec_path() { builder = builder.with_chain_spec_path(location.clone()); } - // Configure chain spec generator - if let Some(command) = chain_spec_generator { + // Configure chain spec generator or file + if let Some(ref path) = parachain_chain_spec_file { + builder = builder.with_chain_spec_path(path.as_str()); + } else if let Some(command) = chain_spec_generator { builder = builder.with_chain_spec_command(command); } // Overrides: genesis/wasm @@ -699,14 +713,16 @@ struct RelayChain { // The runtime used. runtime: Runtime, /// The binary used to launch a relay chain node. - binary: Binary, + binary: SourcedArchive, /// The additional workers required by the relay chain node. workers: [&'static str; 2], /// The name of the chain. #[allow(dead_code)] chain: String, /// If applicable, the binary used to generate a chain specification. - chain_spec_generator: Option, + chain_spec_generator: Option, + /// If applicable, the chain spec file + chain_spec_file: Option, } /// The configuration required to launch a parachain. @@ -715,11 +731,13 @@ struct Chain { /// The parachain identifier on the local network. id: u32, /// The binary used to launch a parachain node. - binary: Binary, + binary: SourcedArchive, /// The name of the chain. chain: Option, /// If applicable, the binary used to generate a chain specification. - chain_spec_generator: Option, + chain_spec_generator: Option, + /// If applicable, the chain spec file used to launch a parachain node + chain_spec_file: Option, } impl Chain { @@ -739,9 +757,15 @@ impl Chain { let manifest = resolve_manifest(&name, &path)?; Ok(Chain { id, - binary: Binary::Local { name, path, manifest }, + binary: SourcedArchive::Local { + name, + path, + manifest, + archive_type: ArchiveType::Binary, + }, chain: chain.map(|c| c.to_string()), chain_spec_generator: None, + chain_spec_file: None, }) } @@ -773,18 +797,20 @@ impl Chain { .into(); Ok(Chain { id, - binary: Binary::Source { + binary: SourcedArchive::Source { name: repo.package.clone(), source, cache: cache.to_path_buf(), + archive_type: ArchiveType::Binary, }, chain: chain.map(|c| c.to_string()), chain_spec_generator: None, + chain_spec_file: None, }) } else { Ok(Chain { id, - binary: Binary::Source { + binary: SourcedArchive::Source { name: repo.package.clone(), source: Git { url: repo.url.clone(), @@ -795,9 +821,11 @@ impl Chain { } .into(), cache: cache.to_path_buf(), + archive_type: ArchiveType::Binary, }, chain: chain.map(|c| c.to_string()), chain_spec_generator: None, + chain_spec_file: None, }) } } @@ -805,13 +833,15 @@ impl Chain { fn from_omni_node(id: u32, cache: &Path) -> Result { Ok(Chain { id, - binary: Binary::Source { - name: PolkadotOmniNode.binary().to_string(), + binary: SourcedArchive::Source { + name: PolkadotOmniNode.binary()?.to_string(), source: Box::new(PolkadotOmniNode.source()?), cache: cache.to_path_buf(), + archive_type: ArchiveType::Binary, }, chain: None, chain_spec_generator: None, + chain_spec_file: None, }) } } @@ -937,9 +967,9 @@ chain = "paseo-local" assert_eq!(relay_chain.version().unwrap(), RELAY_BINARY_VERSION); assert!(matches!( relay_chain, - Binary::Source { source, .. } + SourcedArchive::Source { source, archive_type, .. } if matches!(source.as_ref(), Source::GitHub(ReleaseArchive { tag, .. }) - if *tag == Some(format!("polkadot-{RELAY_BINARY_VERSION}")) + if *tag == Some(format!("polkadot-{RELAY_BINARY_VERSION}")) && *archive_type == ArchiveType::Binary ) )); assert!(zombienet.parachains.is_empty()); @@ -979,9 +1009,9 @@ chain = "paseo-local" assert_eq!(relay_chain.version().unwrap(), RELAY_BINARY_VERSION); assert!(matches!( relay_chain, - Binary::Source { source, .. } + SourcedArchive::Source { source, archive_type, .. } if matches!(source.as_ref(), Source::GitHub(ReleaseArchive { tag, .. }) - if *tag == Some(format!("polkadot-{RELAY_BINARY_VERSION}")) + if *tag == Some(format!("polkadot-{RELAY_BINARY_VERSION}")) && *archive_type == ArchiveType::Binary ) )); assert!(zombienet.parachains.is_empty()); @@ -999,7 +1029,7 @@ chain = "paseo-local" config.as_file(), r#" [relaychain] -chain = "paseo-local" +chain = "polkadot-local" "# )?; let version = "v1.3.3"; @@ -1015,20 +1045,64 @@ chain = "paseo-local" ) .await?; - assert_eq!(zombienet.relay_chain.chain, "paseo-local"); + assert_eq!(zombienet.relay_chain.chain, "polkadot-local"); let chain_spec_generator = &zombienet.relay_chain.chain_spec_generator.unwrap(); - assert_eq!(chain_spec_generator.name(), "paseo-chain-spec-generator"); + assert_eq!(chain_spec_generator.name(), "polkadot-chain-spec-generator"); assert_eq!( chain_spec_generator.path(), - temp_dir.path().join(format!("paseo-chain-spec-generator-{version}")) + temp_dir.path().join(format!("polkadot-chain-spec-generator-{version}")) ); assert_eq!(chain_spec_generator.version().unwrap(), version); assert!(matches!( chain_spec_generator, - Binary::Source { source, .. } + SourcedArchive::Source { source, archive_type, .. } if matches!(source.as_ref(), Source::GitHub(ReleaseArchive { tag, .. }) if *tag == Some(version.to_string()) - ) + ) && *archive_type == ArchiveType::Binary + )); + assert!(zombienet.parachains.is_empty()); + Ok(()) + } + + #[tokio::test] + async fn new_with_relay_chain_spec_file_works() -> Result<()> { + let temp_dir = tempdir()?; + let cache = PathBuf::from(temp_dir.path()); + let config = Builder::new().suffix(".toml").tempfile()?; + writeln!( + config.as_file(), + r#" +[relaychain] +chain = "paseo-local" +"# + )?; + let version = "v1.3.3"; + + let zombienet = Zombienet::new( + &cache, + config.path().try_into()?, + None, + Some(version), + None, + None, + None, + ) + .await?; + + assert_eq!(zombienet.relay_chain.chain, "paseo-local"); + let chain_spec_file = &zombienet.relay_chain.chain_spec_file.unwrap(); + assert_eq!(chain_spec_file.name(), "paseo-local.json"); + assert_eq!( + chain_spec_file.path(), + temp_dir.path().join(format!("paseo-local-{}.json", version)) + ); + assert_eq!(chain_spec_file.version().unwrap(), version); + assert!(matches!( + chain_spec_file, + SourcedArchive::Source { source, archive_type, .. } + if matches!(source.as_ref(), Source::GitHub(ReleaseArchive { tag, .. }) + if *tag == Some(version.to_string()) + ) && *archive_type == ArchiveType::File )); assert!(zombienet.parachains.is_empty()); Ok(()) @@ -1068,9 +1142,9 @@ default_command = "./bin-stable2503/polkadot" assert_eq!(relay_chain.version().unwrap(), RELAY_BINARY_VERSION); assert!(matches!( relay_chain, - Binary::Source { source, ..} + SourcedArchive::Source { source, archive_type, ..} if matches!(source.as_ref(), Source::GitHub(ReleaseArchive { tag, .. }) - if *tag == Some(format!("polkadot-{RELAY_BINARY_VERSION}")) + if *tag == Some(format!("polkadot-{RELAY_BINARY_VERSION}")) && *archive_type == ArchiveType::Binary ) )); assert!(zombienet.parachains.is_empty()); @@ -1115,10 +1189,10 @@ command = "polkadot" assert_eq!(relay_chain.version().unwrap(), RELAY_BINARY_VERSION); assert!(matches!( relay_chain, - Binary::Source { source, .. } + SourcedArchive::Source { source, archive_type, .. } if matches!(source.as_ref(), Source::GitHub(ReleaseArchive { tag, .. }) if *tag == Some(format!("polkadot-{RELAY_BINARY_VERSION}")) - ) + ) && *archive_type == ArchiveType::Binary )); assert!(zombienet.parachains.is_empty()); Ok(()) @@ -1156,10 +1230,10 @@ command = "polkadot" assert_eq!(relay_chain.version().unwrap(), RELAY_BINARY_VERSION); assert!(matches!( relay_chain, - Binary::Source { source, .. } + SourcedArchive::Source { source, archive_type, .. } if matches!(source.as_ref(), Source::GitHub(ReleaseArchive { tag, .. }) if *tag == Some(format!("polkadot-{RELAY_BINARY_VERSION}")) - ) + ) && *archive_type == ArchiveType::Binary )); assert!(zombienet.parachains.is_empty()); Ok(()) @@ -1283,7 +1357,7 @@ chain = "asset-hub-paseo-local" assert_eq!(system_parachain.version().unwrap(), SYSTEM_PARA_BINARY_VERSION); assert!(matches!( system_parachain, - Binary::Source { source, .. } + SourcedArchive::Source { source, .. } if matches!(source.as_ref(), Source::GitHub(ReleaseArchive { tag, .. }) if *tag == Some(format!("polkadot-{SYSTEM_PARA_BINARY_VERSION}")) ) @@ -1300,11 +1374,11 @@ chain = "asset-hub-paseo-local" config.as_file(), r#" [relaychain] -chain = "paseo-local" +chain = "polkadot-local" [[parachains]] id = 1000 -chain = "asset-hub-paseo-local" +chain = "asset-hub-polkadot-local" "# )?; @@ -1321,22 +1395,72 @@ chain = "asset-hub-paseo-local" assert_eq!(zombienet.parachains.len(), 1); let system_parachain = &zombienet.parachains.get(&1000).unwrap(); - assert_eq!(system_parachain.chain.as_ref().unwrap(), "asset-hub-paseo-local"); + assert_eq!(system_parachain.chain.as_ref().unwrap(), "asset-hub-polkadot-local"); let chain_spec_generator = system_parachain.chain_spec_generator.as_ref().unwrap(); - assert_eq!(chain_spec_generator.name(), "paseo-chain-spec-generator"); + assert_eq!(chain_spec_generator.name(), "polkadot-chain-spec-generator"); assert_eq!( chain_spec_generator.path(), temp_dir .path() - .join(format!("paseo-chain-spec-generator-{SYSTEM_PARA_RUNTIME_VERSION}")) + .join(format!("polkadot-chain-spec-generator-{SYSTEM_PARA_RUNTIME_VERSION}")) ); assert_eq!(chain_spec_generator.version().unwrap(), SYSTEM_PARA_RUNTIME_VERSION); assert!(matches!( chain_spec_generator, - Binary::Source { source, .. } + SourcedArchive::Source { source, archive_type, .. } if matches!(source.as_ref(), Source::GitHub(ReleaseArchive { tag, .. }) if *tag == Some(SYSTEM_PARA_RUNTIME_VERSION.to_string()) - ) + ) && *archive_type == ArchiveType::Binary + )); + Ok(()) + } + + #[tokio::test] + async fn new_with_system_chain_spec_file_works() -> Result<()> { + let temp_dir = tempdir()?; + let cache = PathBuf::from(temp_dir.path()); + let config = Builder::new().suffix(".toml").tempfile()?; + writeln!( + config.as_file(), + r#" +[relaychain] +chain = "paseo-local" + +[[parachains]] +id = 1000 +chain = "asset-hub-paseo-local" +"# + )?; + + let zombienet = Zombienet::new( + &cache, + config.path().try_into()?, + None, + None, + None, + Some(SYSTEM_PARA_RUNTIME_VERSION), + None, + ) + .await?; + + assert_eq!(zombienet.parachains.len(), 1); + let system_parachain = &zombienet.parachains.get(&1000).unwrap(); + assert_eq!(system_parachain.chain.as_ref().unwrap(), "asset-hub-paseo-local"); + let chain_spec_file = system_parachain.chain_spec_file.as_ref().unwrap(); + assert_eq!(chain_spec_file.name(), "asset-hub-paseo-local.json"); + assert_eq!( + chain_spec_file.path(), + temp_dir + .path() + .join(format!("asset-hub-paseo-local-{SYSTEM_PARA_RUNTIME_VERSION}.json")) + ); + assert_eq!(chain_spec_file.version().unwrap(), SYSTEM_PARA_RUNTIME_VERSION); + assert!(matches!( + chain_spec_file, + SourcedArchive::Source { source, archive_type, .. } + if matches!(source.as_ref(), Source::GitHub(ReleaseArchive { tag, .. }) + if *tag == Some(SYSTEM_PARA_RUNTIME_VERSION.to_string()) + ) && *archive_type == ArchiveType::File )); Ok(()) } @@ -1370,7 +1494,7 @@ default_command = "pop-node" assert_eq!(pop.version().unwrap(), version); assert!(matches!( pop, - Binary::Source { source, .. } + SourcedArchive::Source { source, .. } if matches!(source.as_ref(), Source::GitHub(ReleaseArchive { tag, .. }) if *tag == Some(format!("node-{version}")) ) @@ -1414,7 +1538,7 @@ default_command = "pop-node" assert_eq!(pop.version().unwrap(), version); assert!(matches!( pop, - Binary::Source { source, .. } + SourcedArchive::Source { source, .. } if matches!(source.as_ref(), Source::GitHub(ReleaseArchive { tag, .. }) if *tag == Some(format!("node-{version}")) ) @@ -1448,7 +1572,7 @@ default_command = "./target/release/parachain-template-node" assert_eq!(pop.name(), "parachain-template-node"); assert_eq!(pop.path(), Path::new("./target/release/parachain-template-node")); assert_eq!(pop.version(), None); - assert!(matches!(pop, Binary::Local { .. })); + assert!(matches!(pop, SourcedArchive::Local { .. })); Ok(()) } @@ -1512,7 +1636,7 @@ command = "substrate-contracts-node" assert_eq!(parachain.name(), "parachain-template-node"); assert_eq!(parachain.path(), Path::new("./target/release/parachain-template-node")); assert_eq!(parachain.version(), None); - assert!(matches!(parachain, Binary::Local { .. })); + assert!(matches!(parachain, SourcedArchive::Local { .. })); let contract_parachain = &zombienet.parachains.get(&2000).unwrap().binary; assert_eq!(contract_parachain.name(), "substrate-contracts-node"); assert_eq!( @@ -1520,7 +1644,7 @@ command = "substrate-contracts-node" Path::new("./target/debug/substrate-contracts-node") ); assert_eq!(contract_parachain.version(), None); - assert!(matches!(contract_parachain, Binary::Local { .. })); + assert!(matches!(contract_parachain, SourcedArchive::Local { .. })); Ok(()) }) .await @@ -1555,7 +1679,7 @@ command = "./target/release/parachain-template-node" assert_eq!(pop.name(), "parachain-template-node"); assert_eq!(pop.path(), Path::new("./target/release/parachain-template-node")); assert_eq!(pop.version(), None); - assert!(matches!(pop, Binary::Local { .. })); + assert!(matches!(pop, SourcedArchive::Local { .. })); Ok(()) } @@ -1595,7 +1719,7 @@ default_command = "moonbeam" assert_eq!(pop.version().unwrap(), version); assert!(matches!( pop, - Binary::Source { source, .. } + SourcedArchive::Source { source, .. } if matches!(source.as_ref(), Source::GitHub(SourceCodeArchive { reference, .. }) if *reference == Some(version.to_string()) ) @@ -1691,7 +1815,7 @@ default_command = "pop-node" let mut zombienet = Zombienet::new(&cache, config.path().try_into()?, None, None, None, None, None) .await?; - assert_eq!(zombienet.binaries().count(), 6); + assert_eq!(zombienet.archives().count(), 6); Ok(()) } @@ -1715,7 +1839,7 @@ chain = "asset-hub-paseo-local" let mut zombienet = Zombienet::new(&cache, config.path().try_into()?, None, None, None, None, None) .await?; - assert_eq!(zombienet.binaries().count(), 4); + assert_eq!(zombienet.archives().count(), 4); Ok(()) } @@ -1760,7 +1884,7 @@ chain = "paseo-local" let mut zombienet = Zombienet::new(&cache, config.path().try_into()?, None, None, None, None, None) .await?; - let Binary::Source { source, .. } = &mut zombienet.relay_chain.binary else { + let SourcedArchive::Source { source, .. } = &mut zombienet.relay_chain.binary else { panic!("expected binary which needs to be sourced") }; if let Source::GitHub(ReleaseArchive { tag, .. }) = source.as_mut() { @@ -1829,7 +1953,7 @@ validator = true let mut zombienet = Zombienet::new(&cache, config.path().try_into()?, None, None, None, None, None) .await?; - for b in zombienet.binaries() { + for b in zombienet.archives() { b.source(true, &Output, true).await?; } @@ -2103,66 +2227,76 @@ rpc_port = 9944 let adapted = network_config.adapt( &RelayChain { runtime: Paseo, - binary: Binary::Local { + binary: SourcedArchive::Local { name: "polkadot".to_string(), path: relay_chain.to_path_buf(), manifest: None, + archive_type: ArchiveType::Binary, }, workers: ["polkadot-execute-worker", ""], chain: "paseo-local".to_string(), chain_spec_generator: None, + chain_spec_file: None, }, &[ ( 1000, Chain { id: 1000, - binary: Binary::Local { + binary: SourcedArchive::Local { name: "polkadot-parachain".to_string(), path: system_chain.to_path_buf(), manifest: None, + archive_type: ArchiveType::Binary, }, chain: None, chain_spec_generator: None, + chain_spec_file: None, }, ), ( 2000, Chain { id: 2000, - binary: Binary::Local { + binary: SourcedArchive::Local { name: "pop-node".to_string(), path: pop.to_path_buf(), manifest: None, + archive_type: ArchiveType::Binary, }, chain: None, chain_spec_generator: None, + chain_spec_file: None, }, ), ( 2001, Chain { id: 2001, - binary: Binary::Local { + binary: SourcedArchive::Local { name: "parachain-template-node".to_string(), path: parachain_template.to_path_buf(), manifest: None, + archive_type: ArchiveType::Binary, }, chain: None, chain_spec_generator: None, + chain_spec_file: None, }, ), ( 2002, Chain { id: 2002, - binary: Binary::Local { + binary: SourcedArchive::Local { name: "parachain-template-node".to_string(), path: parachain_template.to_path_buf(), manifest: None, + archive_type: ArchiveType::Binary, }, chain: None, chain_spec_generator: None, + chain_spec_file: None, }, ), ] @@ -2317,34 +2451,40 @@ command = "polkadot-parachain" let adapted = network_config.adapt( &RelayChain { runtime: Paseo, - binary: Binary::Local { + binary: SourcedArchive::Local { name: "polkadot".to_string(), path: relay_chain.to_path_buf(), manifest: None, + archive_type: ArchiveType::Binary, }, workers: ["polkadot-execute-worker", ""], chain: "paseo-local".to_string(), - chain_spec_generator: Some(Binary::Local { + chain_spec_generator: Some(SourcedArchive::Local { name: "paseo-chain-spec-generator".to_string(), path: relay_chain_spec_generator.to_path_buf(), manifest: None, + archive_type: ArchiveType::Binary, }), + chain_spec_file: None, }, &[( 1000, Chain { id: 1000, - binary: Binary::Local { + binary: SourcedArchive::Local { name: "polkadot-parachain".to_string(), path: system_chain.to_path_buf(), manifest: None, + archive_type: ArchiveType::Binary, }, chain: Some("asset-hub-paseo-local".to_string()), - chain_spec_generator: Some(Binary::Local { + chain_spec_generator: Some(SourcedArchive::Local { name: "paseo-chain-spec-generator".to_string(), path: system_chain_spec_generator.to_path_buf(), manifest: None, + archive_type: ArchiveType::Binary, }), + chain_spec_file: None, }, )] .into(), @@ -2453,40 +2593,46 @@ max_message_size = 8000 let adapted = network_config.adapt( &RelayChain { runtime: Paseo, - binary: Binary::Local { + binary: SourcedArchive::Local { name: "polkadot".to_string(), path: relay_chain.to_path_buf(), manifest: None, + archive_type: ArchiveType::Binary, }, workers: ["polkadot-execute-worker", ""], chain: "paseo-local".to_string(), chain_spec_generator: None, + chain_spec_file: None, }, &[ ( 1000, Chain { id: 1000, - binary: Binary::Local { + binary: SourcedArchive::Local { name: "polkadot-parachain".to_string(), path: system_chain.to_path_buf(), manifest: None, + archive_type: ArchiveType::Binary, }, chain: Some("asset-hub-paseo-local".to_string()), chain_spec_generator: None, + chain_spec_file: None, }, ), ( 2000, Chain { id: 2000, - binary: Binary::Local { + binary: SourcedArchive::Local { name: "pop-node".to_string(), path: pop.to_path_buf(), manifest: None, + archive_type: ArchiveType::Binary, }, chain: None, chain_spec_generator: None, + chain_spec_file: None, }, ), ] @@ -2605,26 +2751,30 @@ name = "asset-hub" let adapted = network_config.adapt( &RelayChain { runtime: Paseo, - binary: Binary::Local { + binary: SourcedArchive::Local { name: "polkadot".to_string(), path: relay_chain.to_path_buf(), manifest: None, + archive_type: ArchiveType::Binary, }, workers: ["polkadot-execute-worker", ""], chain: "paseo-local".to_string(), chain_spec_generator: None, + chain_spec_file: None, }, &[( 1000, Chain { id: 1000, - binary: Binary::Local { + binary: SourcedArchive::Local { name: "polkadot-parachain".to_string(), path: system_chain.to_path_buf(), manifest: None, + archive_type: ArchiveType::Binary, }, chain: Some("asset-hub-paseo-local".to_string()), chain_spec_generator: None, + chain_spec_file: None, }, )] .into(), @@ -2723,26 +2873,30 @@ balances = [["5Ec4AhPKXY9B4ayGshkz2wFMh7N8gP7XKfAvtt1cigpG9FkJ", 420000000000]] let adapted = network_config.adapt( &RelayChain { runtime: Paseo, - binary: Binary::Local { + binary: SourcedArchive::Local { name: "polkadot".to_string(), path: relay_chain.to_path_buf(), manifest: None, + archive_type: ArchiveType::Binary, }, workers: ["polkadot-execute-worker", ""], chain: "paseo-local".to_string(), chain_spec_generator: None, + chain_spec_file: None, }, &[( 1000, Chain { id: 1000, - binary: Binary::Local { + binary: SourcedArchive::Local { name: "polkadot-parachain".to_string(), path: system_chain.to_path_buf(), manifest: None, + archive_type: ArchiveType::Binary, }, chain: Some("asset-hub-paseo-local".to_string()), chain_spec_generator: None, + chain_spec_file: None, }, )] .into(), @@ -2838,9 +2992,15 @@ balance = 2000000000000 Chain::from_local(2000, command.clone(), Some("dev"))?, Chain { id: 2000, - binary: Binary::Local { name: name.to_string(), path: command, manifest: None }, + binary: SourcedArchive::Local { + name: name.to_string(), + path: command, + manifest: None, + archive_type: ArchiveType::Binary + }, chain: Some("dev".to_string()), chain_spec_generator: None, + chain_spec_file: None } ); Ok(()) @@ -2854,13 +3014,15 @@ balance = 2000000000000 Chain::from_local(2000, command.clone(), Some("dev"))?, Chain { id: 2000, - binary: Binary::Local { + binary: SourcedArchive::Local { name: name.to_string(), path: command, - manifest: Some(PathBuf::from("./Cargo.toml")) + manifest: Some(PathBuf::from("./Cargo.toml")), + archive_type: ArchiveType::Binary }, chain: Some("dev".to_string()), chain_spec_generator: None, + chain_spec_file: None } ); Ok(()) @@ -2874,7 +3036,7 @@ balance = 2000000000000 Chain::from_repository(2000, &repo, Some("dev"), cache.path())?, Chain { id: 2000, - binary: Binary::Source { + binary: SourcedArchive::Source { name: "pop-node".to_string(), source: Git { url: repo.url, @@ -2885,9 +3047,11 @@ balance = 2000000000000 } .into(), cache: cache.path().to_path_buf(), + archive_type: ArchiveType::Binary }, chain: Some("dev".to_string()), chain_spec_generator: None, + chain_spec_file: None } ); Ok(()) @@ -2901,7 +3065,7 @@ balance = 2000000000000 Chain::from_repository(2000, &repo, Some("dev"), cache.path())?, Chain { id: 2000, - binary: Binary::Source { + binary: SourcedArchive::Source { name: "pop-node".to_string(), source: Source::GitHub(SourceCodeArchive { owner: "r0gue-io".to_string(), @@ -2913,9 +3077,11 @@ balance = 2000000000000 }) .into(), cache: cache.path().to_path_buf(), + archive_type: ArchiveType::Binary }, chain: Some("dev".to_string()), chain_spec_generator: None, + chain_spec_file: None }, ); Ok(()) diff --git a/crates/pop-chains/src/up/relay.rs b/crates/pop-chains/src/up/relay.rs index 25a238248..a129f3ea7 100644 --- a/crates/pop-chains/src/up/relay.rs +++ b/crates/pop-chains/src/up/relay.rs @@ -1,14 +1,14 @@ // SPDX-License-Identifier: GPL-3.0 -use super::chain_specs::chain_spec_generator; +use super::chain_specs::{chain_spec_file, chain_spec_generator}; use crate::{Error, up::chain_specs}; use pop_common::{ git::GitHub, polkadot_sdk::sort_by_latest_stable_version, sourcing::{ - ArchiveFileSpec, Binary, + ArchiveFileSpec, ArchiveType, GitHub::*, - Source, + Source, SourcedArchive, filters::prefix, traits::{ Source as SourceT, @@ -50,8 +50,8 @@ impl SourceT for &RelayChain { prerelease: false, version_comparator: sort_by_latest_stable_version, fallback: self.fallback().into(), - archive: format!("{}-{}.tar.gz", self.binary(), target()?), - contents: once(self.binary()) + archive: format!("{}-{}.tar.gz", self.binary()?, target()?), + contents: once(self.binary()?) .chain(self.workers()) .map(|n| ArchiveFileSpec::new(n.into(), None, true)) .collect(), @@ -82,7 +82,7 @@ pub(super) async fn default( chain: &str, cache: &Path, ) -> Result { - from(RelayChain::Polkadot.binary(), version, runtime_version, chain, cache).await + from(RelayChain::Polkadot.binary()?, version, runtime_version, chain, cache).await } /// Initializes the configuration required to launch the relay chain using the specified command. @@ -100,17 +100,22 @@ pub(super) async fn from( chain: &str, cache: &Path, ) -> Result { - if let Some(relay) = RelayChain::VARIANTS - .iter() - .find(|r| command.to_lowercase().ends_with(r.binary())) - { - let name = relay.binary().to_string(); + if let Some(relay) = RelayChain::VARIANTS.iter().find(|r| { + let binary = r.binary(); + if let Ok(binary) = binary { command.to_lowercase().ends_with(binary) } else { false } + }) { + let name = relay.binary()?.to_string(); let source = relay .source()? .resolve(&name, version, cache, |f| prefix(f, &name)) .await .into(); - let binary = Binary::Source { name, source, cache: cache.to_path_buf() }; + let binary = SourcedArchive::Source { + name, + source, + cache: cache.to_path_buf(), + archive_type: ArchiveType::Binary, + }; let runtime = chain_specs::Runtime::from_chain(chain) .ok_or(Error::UnsupportedCommand(format!("the relay chain is unsupported: {chain}")))?; return Ok(super::RelayChain { @@ -119,6 +124,7 @@ pub(super) async fn from( workers: relay.workers(), chain: chain.into(), chain_spec_generator: chain_spec_generator(chain, runtime_version, cache).await?, + chain_spec_file: chain_spec_file(chain, runtime_version, cache).await?, }); } @@ -140,45 +146,75 @@ mod tests { let temp_dir = tempdir()?; let relay = default(Some(RELAY_BINARY_VERSION), None, "paseo-local", temp_dir.path()).await?; - assert!(matches!(relay.binary, Binary::Source { name, source, cache } - if name == expected.binary() && source == Source::GitHub(ReleaseArchive { - owner: "r0gue-io".to_string(), - repository: "polkadot".to_string(), - tag: Some(format!("polkadot-{RELAY_BINARY_VERSION}")), - tag_pattern: Some("polkadot-{version}".into()), - prerelease: false, - version_comparator: sort_by_latest_stable_version, - fallback: FALLBACK.into(), - archive: format!("{name}-{}.tar.gz", target()?), - contents: ["polkadot", "polkadot-execute-worker", "polkadot-prepare-worker"].map(|b| ArchiveFileSpec::new(b.into(), None, true)).to_vec(), - latest: relay.binary.latest().map(|l| l.to_string()), - }).into() && cache == temp_dir.path() - )); + assert!( + matches!(relay.binary, SourcedArchive::Source { name, source, cache, archive_type } + if name == expected.binary().unwrap() && source == Source::GitHub(ReleaseArchive { + owner: "r0gue-io".to_string(), + repository: "polkadot".to_string(), + tag: Some(format!("polkadot-{RELAY_BINARY_VERSION}")), + tag_pattern: Some("polkadot-{version}".into()), + prerelease: false, + version_comparator: sort_by_latest_stable_version, + fallback: FALLBACK.into(), + archive: format!("{name}-{}.tar.gz", target()?), + contents: ["polkadot", "polkadot-execute-worker", "polkadot-prepare-worker"].map(|b| ArchiveFileSpec::new(b.into(), None, true)).to_vec(), + latest: relay.binary.latest().map(|l| l.to_string()), + }).into() && cache == temp_dir.path() && archive_type == ArchiveType::Binary + ) + ); assert_eq!(relay.workers, expected.workers()); Ok(()) } #[tokio::test] async fn default_with_chain_spec_generator_works() -> anyhow::Result<()> { + let runtime_version = "v1.3.3"; + let temp_dir = tempdir()?; + let relay = default(None, Some(runtime_version), "polkadot-local", temp_dir.path()).await?; + assert_eq!(relay.chain, "polkadot-local"); + let chain_spec_generator = relay.chain_spec_generator.unwrap(); + assert!( + matches!(chain_spec_generator, SourcedArchive::Source { name, source, cache, archive_type } + if name == "polkadot-chain-spec-generator" && source == Source::GitHub(ReleaseArchive { + owner: "r0gue-io".to_string(), + repository: "polkadot-runtimes".to_string(), + tag: Some(runtime_version.to_string()), + tag_pattern: None, + prerelease: false, + version_comparator: sort_by_latest_semantic_version, + fallback: "v1.4.1".into(), + archive: format!("chain-spec-generator-{}.tar.gz", target()?), + contents: [ArchiveFileSpec::new("chain-spec-generator".into(), Some("polkadot-chain-spec-generator".into()), true)].to_vec(), + latest: chain_spec_generator.latest().map(|l| l.to_string()), + }).into() && cache == temp_dir.path() && archive_type == ArchiveType::Binary + ) + ); + Ok(()) + } + + #[tokio::test] + async fn default_with_chain_spec_file_works() -> anyhow::Result<()> { let runtime_version = "v1.3.3"; let temp_dir = tempdir()?; let relay = default(None, Some(runtime_version), "paseo-local", temp_dir.path()).await?; assert_eq!(relay.chain, "paseo-local"); - let chain_spec_generator = relay.chain_spec_generator.unwrap(); - assert!(matches!(chain_spec_generator, Binary::Source { name, source, cache } - if name == "paseo-chain-spec-generator" && source == Source::GitHub(ReleaseArchive { - owner: "r0gue-io".to_string(), - repository: "paseo-runtimes".to_string(), - tag: Some(runtime_version.to_string()), - tag_pattern: None, - prerelease: false, - version_comparator: sort_by_latest_semantic_version, - fallback: "v1.4.1".into(), - archive: format!("chain-spec-generator-{}.tar.gz", target()?), - contents: [ArchiveFileSpec::new("chain-spec-generator".into(), Some("paseo-chain-spec-generator".into()), true)].to_vec(), - latest: chain_spec_generator.latest().map(|l| l.to_string()), - }).into() && cache == temp_dir.path() - )); + let chain_spec_file = relay.chain_spec_file.unwrap(); + assert!( + matches!(chain_spec_file, SourcedArchive::Source { name, source, cache, archive_type } + if name == "paseo-local.json" && source == Source::GitHub(ReleaseArchive { + owner: "paseo-network".to_string(), + repository: "runtimes".to_string(), + tag: Some(runtime_version.to_string()), + tag_pattern: None, + prerelease: false, + version_comparator: sort_by_latest_semantic_version, + fallback: "v2.0.2".into(), + archive: "paseo-local.json".to_string(), + contents: [ArchiveFileSpec::new("paseo-local.json".into(), Some("paseo-local".into()), true)].to_vec(), + latest: chain_spec_file.latest().map(|l| l.to_string()), + }).into() && cache == temp_dir.path() && archive_type == ArchiveType::File + ) + ); Ok(()) } @@ -203,8 +239,8 @@ mod tests { temp_dir.path(), ) .await?; - assert!(matches!(relay.binary, Binary::Source { name, source, cache } - if name == expected.binary() && source == Source::GitHub(ReleaseArchive { + assert!(matches!(relay.binary, SourcedArchive::Source { name, source, cache, archive_type} + if name == expected.binary().unwrap() && source == Source::GitHub(ReleaseArchive { owner: "r0gue-io".to_string(), repository: "polkadot".to_string(), tag: Some(format!("polkadot-{RELAY_BINARY_VERSION}")), @@ -215,7 +251,7 @@ mod tests { archive: format!("{name}-{}.tar.gz", target()?), contents: ["polkadot", "polkadot-execute-worker", "polkadot-prepare-worker"].map(|b| ArchiveFileSpec::new(b.into(), None, true)).to_vec(), latest: relay.binary.latest().map(|l| l.to_string()), - }).into() && cache == temp_dir.path() + }).into() && cache == temp_dir.path() && archive_type==ArchiveType::Binary )); assert_eq!(relay.workers, expected.workers()); Ok(()) diff --git a/crates/pop-chains/tests/chain.rs b/crates/pop-chains/tests/chain.rs index 27af32e56..76f344059 100644 --- a/crates/pop-chains/tests/chain.rs +++ b/crates/pop-chains/tests/chain.rs @@ -26,8 +26,8 @@ async fn launch_kusama() -> Result<()> { ) .await?; - for binary in zombienet.binaries().filter(|b| !b.exists()) { - binary.source(true, &(), true).await?; + for archive in zombienet.archives().filter(|b| !b.exists()) { + archive.source(true, &(), true).await?; } zombienet.spawn().await?; @@ -43,15 +43,15 @@ async fn launch_paseo() -> Result<()> { &cache, Path::new("../../tests/networks/paseo.toml").try_into()?, Some(BINARY_VERSION), - Some("v1.2.4"), + Some("v2.0.2"), None, None, None, ) .await?; - for binary in zombienet.binaries().filter(|b| !b.exists()) { - binary.source(true, &(), true).await?; + for archive in zombienet.archives().filter(|b| !b.exists()) { + archive.source(true, &(), true).await?; } zombienet.spawn().await?; @@ -74,8 +74,8 @@ async fn launch_polkadot() -> Result<()> { ) .await?; - for binary in zombienet.binaries().filter(|b| !b.exists()) { - binary.source(true, &(), true).await?; + for archive in zombienet.archives().filter(|b| !b.exists()) { + archive.source(true, &(), true).await?; } zombienet.spawn().await?; @@ -98,8 +98,8 @@ async fn launch_polkadot_and_system_parachain() -> Result<()> { ) .await?; - for binary in zombienet.binaries().filter(|b| !b.exists()) { - binary.source(true, &(), true).await?; + for archive in zombienet.archives().filter(|b| !b.exists()) { + archive.source(true, &(), true).await?; } zombienet.spawn().await?; @@ -117,13 +117,13 @@ async fn launch_paseo_and_system_parachain() -> Result<()> { Some(BINARY_VERSION), None, Some(BINARY_VERSION), - Some("v1.3.3"), // 1.3.3 is where coretime-paseo-local was introduced. + Some("v2.0.2"), None, ) .await?; - for binary in zombienet.binaries().filter(|b| !b.exists()) { - binary.source(true, &(), true).await?; + for archive in zombienet.archives().filter(|b| !b.exists()) { + archive.source(true, &(), true).await?; } zombienet.spawn().await?; @@ -146,8 +146,8 @@ async fn launch_paseo_and_two_parachains() -> Result<()> { ) .await?; - for binary in zombienet.binaries().filter(|b| !b.exists()) { - binary.source(true, &(), true).await?; + for archive in zombienet.archives().filter(|b| !b.exists()) { + archive.source(true, &(), true).await?; } zombienet.spawn().await?; diff --git a/crates/pop-cli/src/commands/up/network.rs b/crates/pop-cli/src/commands/up/network.rs index bd57c06a2..282bbb56d 100644 --- a/crates/pop-cli/src/commands/up/network.rs +++ b/crates/pop-cli/src/commands/up/network.rs @@ -62,7 +62,7 @@ pub(crate) struct ConfigFileCommand { /// Whether the output should be verbose. #[arg(short, long, action)] pub(crate) verbose: bool, - /// Automatically source all necessary binaries required without prompting for confirmation. + /// Automatically source all necessary archives required without prompting for confirmation. #[clap(short = 'y', long)] pub(crate) skip_confirm: bool, /// Automatically remove the state upon tearing down the network. @@ -129,7 +129,7 @@ pub(crate) struct BuildCommand { /// Whether the output should be verbose. #[arg(short, long, action)] verbose: bool, - /// Automatically source all necessary binaries required without prompting for confirmation. + /// Automatically source all necessary archives required without prompting for confirmation. #[clap(short = 'y', long)] skip_confirm: bool, /// Automatically remove the state upon tearing down the network. @@ -296,7 +296,7 @@ pub(crate) async fn spawn( Ok(()) }, Error::MissingBinary(name) => { - cli.outro_cancel(format!("🚫 The `{name}` binary is specified in the network configuration file, but cannot be resolved to a source. Are you missing a `--parachain` argument?"))?; + cli.outro_cancel(format!("🚫 The `{name}` archive is specified in the network configuration file, but cannot be resolved to a source. Are you missing a `--parachain` argument?"))?; Ok(()) }, _ => Err(e.into()), @@ -304,15 +304,15 @@ pub(crate) async fn spawn( }, }; - // Source any missing/stale binaries - if source_binaries(&mut zombienet, &cache, verbose, skip_confirm, cli).await? { + // Source any missing/stale archive + if source_archives(&mut zombienet, &cache, verbose, skip_confirm, cli).await? { return Ok(()); } - // Output the binaries and versions used if verbose logging enabled. + // Output the archives and versions used if verbose logging enabled. if verbose { - let binaries = zombienet - .binaries() + let archives = zombienet + .archives() .map(|b| { format!( "{}{}", @@ -320,13 +320,13 @@ pub(crate) async fn spawn( b.version().map(|v| format!(" ({v})")).unwrap_or_default() ) }) - .fold(Vec::new(), |mut set, binary| { - if !set.contains(&binary) { - set.push(binary); + .fold(Vec::new(), |mut set, archive| { + if !set.contains(&archive) { + set.push(archive); } set }); - cli.info(format!("Binaries used: {}", binaries.join(", ")))?; + cli.info(format!("Binaries used: {}", archives.join(", ")))?; } // Finally, spawn the network and wait for a signal to terminate @@ -438,29 +438,29 @@ pub(crate) async fn spawn( Ok(()) } -async fn source_binaries( +async fn source_archives( zombienet: &mut Zombienet, cache: &Path, verbose: bool, skip_confirm: bool, cli: &mut impl cli::traits::Cli, ) -> anyhow::Result { - // Check for any missing or stale binaries - let binaries = zombienet.binaries().filter(|b| !b.exists() || b.stale()).fold( + // Check for any missing or stale archives + let archives = zombienet.archives().filter(|b| !b.exists() || b.stale()).fold( Vec::new(), - |mut set, binary| { - if !set.contains(&binary) { - set.push(binary); + |mut set, archive| { + if !set.contains(&archive) { + set.push(archive); } set }, ); - if binaries.is_empty() { + if archives.is_empty() { return Ok(false); } - // Check if any missing binaries - let missing: IndexSet<_> = binaries + // Check if any missing archives + let missing: IndexSet<_> = archives .iter() .filter_map(|b| (!b.exists()).then_some((b.name(), b.version()))) .collect(); @@ -472,10 +472,10 @@ async fn source_binaries( .dim() .to_string(); cli.warning(format!( - "⚠️ The following binaries required to launch the network cannot be found locally:\n {list}" + "⚠️ The following archives required to launch the network cannot be found locally:\n {list}" ))?; - // Prompt for automatic sourcing of binaries + // Prompt for automatic sourcing of archives let list = style(format!( "> {}", missing @@ -500,14 +500,14 @@ async fn source_binaries( .interact()? { cli.outro_cancel( - "🚫 Cannot launch the specified network until all required binaries are available.", + "🚫 Cannot launch the specified network until all required archives are available.", )?; return Ok(true); } } - // Check if any stale binaries - let stale: IndexSet<_> = binaries + // Check if any stale archives + let stale: IndexSet<_> = archives .iter() .filter_map(|b| b.stale().then_some((b.name(), b.version(), b.latest()))) .collect(); @@ -526,7 +526,7 @@ async fn source_binaries( .dim() .to_string(); cli.warning(format!( - "ℹ️ The following binaries have newer versions available:\n {list}" + "ℹ️ The following archives have newer versions available:\n {list}" ))?; if !skip_confirm { latest = cli @@ -542,7 +542,7 @@ async fn source_binaries( } #[allow(clippy::manual_inspect)] - let binaries: Vec<_> = binaries + let archives: Vec<_> = archives .into_iter() .filter(|b| !b.exists() || (latest && b.stale())) .map(|b| { @@ -553,29 +553,29 @@ async fn source_binaries( }) .collect(); - if binaries.is_empty() { + if archives.is_empty() { return Ok(false); } - if binaries.iter().any(|b| !b.local()) { + if archives.iter().any(|b| !b.local()) { cli.info(format!( - "ℹ️ Binaries will be cached at {}", + "ℹ️ Archives will be cached at {}", &cache.to_str().expect("expected local cache is invalid") ))?; } - // Source binaries + // Source archives let release = true; match verbose { true => { let reporter = VerboseReporter; - for binary in binaries { - cli.info(format!("📦 Sourcing {}...", binary.name()))?; + for archive in archives { + cli.info(format!("📦 Sourcing {}...", archive.name()))?; Term::stderr().clear_last_lines(1)?; - if let Err(e) = binary.source(release, &reporter, verbose).await { + if let Err(e) = archive.source(release, &reporter, verbose).await { reporter.update(&format!("Sourcing failed: {e}")); cli.outro_cancel( - "🚫 Cannot launch the network until all required binaries are available.", + "🚫 Cannot launch the network until all required archives are available.", )?; return Ok(true); } @@ -583,29 +583,29 @@ async fn source_binaries( reporter.update(""); }, false => { - let multi = multi_progress("📦 Sourcing binaries...".to_string()); - let queue: Vec<_> = binaries + let multi = multi_progress("📦 Sourcing archives...".to_string()); + let queue: Vec<_> = archives .into_iter() - .map(|binary| { + .map(|archive| { let progress = multi.add(spinner()); - progress.start(format!("{}: waiting...", binary.name())); - (binary, progress) + progress.start(format!("{}: waiting...", archive.name())); + (archive, progress) }) .collect(); let mut error = false; - for (binary, progress) in queue { - let prefix = format!("{}: ", binary.name()); + for (archive, progress) in queue { + let prefix = format!("{}: ", archive.name()); let progress_reporter = ProgressReporter(prefix, progress); - if let Err(e) = binary.source(release, &progress_reporter, verbose).await { - progress_reporter.1.error(format!("🚫 {}: {e}", binary.name())); + if let Err(e) = archive.source(release, &progress_reporter, verbose).await { + progress_reporter.1.error(format!("🚫 {}: {e}", archive.name())); error = true; } - progress_reporter.1.stop(format!("✅ {}", binary.name())); + progress_reporter.1.stop(format!("✅ {}", archive.name())); } multi.stop(); if error { cli.outro_cancel( - "🚫 Cannot launch the network until all required binaries are available.", + "🚫 Cannot launch the network until all required archives are available.", )?; return Ok(true); } diff --git a/crates/pop-cli/src/common/binary.rs b/crates/pop-cli/src/common/binary.rs index d4cd89066..9f6cb514d 100644 --- a/crates/pop-cli/src/common/binary.rs +++ b/crates/pop-cli/src/common/binary.rs @@ -8,7 +8,7 @@ use std::path::PathBuf; #[cfg(any(feature = "contract", feature = "chain"))] use { crate::cli::traits::*, - pop_common::sourcing::{Binary, set_executable_permission}, + pop_common::sourcing::{SourcedArchive, set_executable_permission}, std::path::Path, }; @@ -23,7 +23,7 @@ pub(crate) trait BinaryGenerator { async fn generate( cache_path: PathBuf, version: Option<&str>, - ) -> Result; + ) -> Result; } /// Checks the status of the provided binary, sources it if necessary, and @@ -112,7 +112,7 @@ macro_rules! impl_binary_generator { async fn generate( cache_path: std::path::PathBuf, version: Option<&str>, - ) -> Result { + ) -> Result { $generate_fn(cache_path, version).await } } diff --git a/crates/pop-cli/src/common/omni_node.rs b/crates/pop-cli/src/common/omni_node.rs index 435e43fa4..56bfca182 100644 --- a/crates/pop-cli/src/common/omni_node.rs +++ b/crates/pop-cli/src/common/omni_node.rs @@ -29,7 +29,7 @@ pub async fn source_polkadot_omni_node_binary( check_and_prompt::( cli, spinner, - PolkadotOmniNodeCli::PolkadotOmniNode.binary(), + PolkadotOmniNodeCli::PolkadotOmniNode.binary()?, cache_path, skip_confirm, ) @@ -45,7 +45,7 @@ mod tests { #[tokio::test] async fn source_polkadot_omni_node_binary_works() -> anyhow::Result<()> { let cache_path = tempfile::tempdir()?; - let binary_name = PolkadotOmniNodeCli::PolkadotOmniNode.binary(); + let binary_name = PolkadotOmniNodeCli::PolkadotOmniNode.binary().unwrap(); let mut cli = MockCli::new() .expect_warning(format!("⚠️ The {binary_name} binary is not found.")) .expect_confirm("📦 Would you like to source it automatically now?", true) @@ -68,7 +68,7 @@ mod tests { #[tokio::test] async fn source_polkadot_omni_node_binary_handles_skip_confirm() -> anyhow::Result<()> { let cache_path = tempfile::tempdir()?; - let binary_name = PolkadotOmniNodeCli::PolkadotOmniNode.binary(); + let binary_name = PolkadotOmniNodeCli::PolkadotOmniNode.binary().unwrap(); let mut cli = MockCli::new().expect_warning(format!("⚠️ The {binary_name} binary is not found.")); diff --git a/crates/pop-cli/tests/chain.rs b/crates/pop-cli/tests/chain.rs index 70e2a1e2c..efe86c583 100644 --- a/crates/pop-cli/tests/chain.rs +++ b/crates/pop-cli/tests/chain.rs @@ -3,11 +3,13 @@ //! Integration tests for chain-related functionality. #![cfg(all(feature = "chain", feature = "integration-tests"))] +#![allow(dead_code)] +#![allow(unused_imports)] use anyhow::Result; use pop_chains::{ ChainTemplate, - up::{Binary, Source::GitHub}, + up::{ArchiveType, Source::GitHub, SourcedArchive}, }; use pop_common::{ find_free_port, @@ -26,44 +28,6 @@ use strum::VariantArray; use tempfile::tempdir; use tokio::process::Child; -/// Utility child process wrapper to kill the child process on drop. -/// -/// To be used exclusively for tests. -struct TestChildProcess(pub(crate) Child); - -impl Drop for TestChildProcess { - fn drop(&mut self) { - let _ = self.0.start_kill(); - } -} - -// Test that all templates are generated correctly -#[tokio::test] -async fn generate_all_the_templates() -> Result<()> { - let temp = tempfile::tempdir()?; - let temp_dir = temp.path(); - - for template in ChainTemplate::VARIANTS { - let parachain_name = format!("test_parachain_{}", template); - let provider = template.template_type()?.to_lowercase(); - // pop new chain test_parachain --verify - let mut command = pop( - temp_dir, - [ - "new", - "chain", - ¶chain_name, - &provider, - "--template", - template.as_ref(), - "--verify", - ], - ); - assert!(command.spawn()?.wait().await?.success()); - assert!(temp_dir.join(parachain_name).exists()); - } - Ok(()) -} /// Test the parachain lifecycle: new, build, up, call. #[tokio::test] @@ -78,315 +42,15 @@ async fn parachain_lifecycle() -> Result<()> { }; // pop new chain test_parachain --verify (default) - let working_dir = temp_dir.join("test_parachain"); - if !working_dir.exists() { let mut command = pop( temp_dir, [ - "new", - "chain", - "test_parachain", - "--symbol", - "POP", - "--decimals", - "6", - "--endowment", - "1u64 << 60", - "--verify", - "--with-frontend=create-dot-app", - "--package-manager", - "npm", + "up", + "paseo" ], ); assert!(command.spawn()?.wait().await?.success()); - assert!(working_dir.exists()); - assert!(working_dir.join("frontend").exists()); - } - - // Mock build process and fetch binary - mock_build_process(&working_dir)?; - assert!(temp_dir.join("test_parachain/target/release/wbuild/parachain-template-runtime/parachain_template_runtime.wasm").exists()); - let binary_name = fetch_runtime(&working_dir).await?; - let binary_path = replace_mock_with_runtime(&working_dir, binary_name)?; - assert!(binary_path.exists()); - - // pop build spec --output ./target/pop/test-spec.json --para-id 2222 --type development --relay - // paseo-local --protocol-id pop-protocol --chain local --deterministic=false - // --default-bootnode=false - let mut command = pop( - &working_dir, - [ - "build", - "spec", - "--output", - "./target/pop/test-spec.json", - "--id", - "test-chain", - "--para-id", - "2222", - "--type", - "development", - "--chain", - "local_testnet", - "--relay", - "paseo-local", - "--profile", - "release", - "--raw", - "--genesis-state=true", - "--genesis-code=true", - "--protocol-id", - "pop-protocol", - "--deterministic=false", - "--default-bootnode=false", - "--skip-build", - ], - ); - assert!(command.spawn()?.wait().await?.success()); - - // Assert build files have been generated - assert!(working_dir.join("target").exists()); - assert!(working_dir.join("target/pop/test-spec.json").exists()); - assert!(working_dir.join("target/pop/test-spec-raw.json").exists()); - assert!(working_dir.join("target/pop/genesis-code.wasm").exists()); - assert!(working_dir.join("target/pop/genesis-state").exists()); - - let chain_spec_path = working_dir.join("target/pop/test-spec.json"); - let content = fs::read_to_string(&chain_spec_path).expect("Could not read file"); - // Assert custom values have been set properly - assert!(content.contains("\"para_id\": 2222")); - // assert!(content.contains("\"tokenDecimals\": 6")); - // assert!(content.contains("\"tokenSymbol\": \"POP\"")); - assert!(content.contains("\"relay_chain\": \"paseo-local\"")); - assert!(content.contains("\"protocolId\": \"pop-protocol\"")); - assert!(content.contains("\"id\": \"test-chain\"")); - - // Test the `pop bench` feature - test_benchmarking(&working_dir).await?; - - // Overwrite the config file to manually set the port to test pop call parachain. - let network_toml_path = working_dir.join("network.toml"); - fs::create_dir_all(&working_dir)?; - let random_port = find_free_port(None); - let localhost_url = format!("ws://127.0.0.1:{}", random_port); - fs::write( - &network_toml_path, - format!( - r#"[relaychain] -chain = "paseo-local" - -[[relaychain.nodes]] -name = "alice" -validator = true - -[[relaychain.nodes]] -name = "bob" -validator = true - -[[parachains]] -id = 2000 -default_command = "polkadot-omni-node" -chain_spec_path = "{}" - -[[parachains.collators]] -name = "collator-01" -rpc_port = {random_port} -"#, - chain_spec_path.as_os_str().to_str().unwrap(), - ), - )?; - - // `pop up network ./network.toml --skip-confirm` - let mut command = pop( - &working_dir, - ["up", "network", "./network.toml", "-r", "stable2506-2", "--verbose", "--skip-confirm"], - ); - let mut up = TestChildProcess(command.spawn()?); - - // Wait for the networks to initialize. Increased timeout to accommodate CI environment delays. - let wait = Duration::from_secs(300); - println!("waiting for {wait:?} for network to initialize..."); - tokio::time::sleep(wait).await; - - // `pop call chain --pallet System --function remark --args "0x11" --url - // ws://127.0.0.1:random_port --suri //Alice --skip-confirm` - let mut command = pop( - &working_dir, - [ - "call", - "chain", - "--pallet", - "System", - "--function", - "remark", - "--args", - "0x11", - "--url", - &localhost_url, - "--suri", - "//Alice", - "--skip-confirm", - ], - ); - assert!(command.spawn()?.wait().await?.success()); - - // `pop call chain --pallet System --function Account --args - // "15oF4uVJwmo4TdGW7VfQxNLavjCXviqxT9S1MgbjMNHr6Sp5" --url ws://127.0.0.1:random_port - // --skip-confirm` - let mut command = pop( - &working_dir, - [ - "call", - "chain", - "--pallet", - "System", - "--function", - "Account", - "--args", - "15oF4uVJwmo4TdGW7VfQxNLavjCXviqxT9S1MgbjMNHr6Sp5", - "--url", - &localhost_url, - "--skip-confirm", - ], - ); - assert!(command.spawn()?.wait().await?.success()); - - // `pop call chain --pallet System --function Account --args - // "15oF4uVJwmo4TdGW7VfQxNLavjCXviqxT9S1MgbjMNHr6Sp5" --url ws://127.0.0.1:random_port` - let mut command = pop( - &working_dir, - [ - "call", - "chain", - "--pallet", - "System", - "--function", - "Ss58Prefix", - "--url", - &localhost_url, - "--skip-confirm", - ], - ); - assert!(command.spawn()?.wait().await?.success()); - - // pop call chain --call 0x00000411 --url ws://127.0.0.1:random_port --suri //Alice - // --skip-confirm - let mut command = pop( - &working_dir, - [ - "call", - "chain", - "--call", - "0x00000411", - "--url", - &localhost_url, - "--suri", - "//Alice", - "--skip-confirm", - ], - ); - assert!(command.spawn()?.wait().await?.success()); - - assert!(up.0.try_wait()?.is_none(), "the process should still be running"); - // Stop the process - up.0.kill().await?; - up.0.wait().await?; - - Ok(()) -} - -async fn test_benchmarking(working_dir: &Path) -> Result<()> { - // pop bench block --from 0 --to 1 --profile=release - let mut command = pop(working_dir, ["bench", "block", "-y", "--from", "0", "--to", "1"]); - assert!(command.spawn()?.wait().await?.success()); - // pop bench machine --allow-fail --profile=release - command = pop(working_dir, ["bench", "machine", "-y", "--allow-fail"]); - assert!(command.spawn()?.wait().await?.success()); - // pop bench overhead --runtime={runtime_path} --genesis-builder=runtime - // --genesis-builder-preset=development --weight-path={output_path} --profile=release --warmup=1 - // --repeat=1 -y - let runtime_path = get_mock_runtime_path(); - let temp_dir = tempdir()?; - let output_path = temp_dir.path(); - assert!(!output_path.join("block_weights.rs").exists()); - command = pop( - working_dir, - [ - "bench", - "overhead", - &format!("--runtime={}", runtime_path.display()), - "--genesis-builder=runtime", - "--genesis-builder-preset=development", - &format!("--weight-path={}", output_path.display()), - "--warmup=1", - "--repeat=1", - "--profile=release", - "-y", - ], - ); - assert!(command.spawn()?.wait().await?.success()); - - // pop bench pallet --runtime={runtime_path} --genesis-builder=runtime - // --pallets pallet_timestamp,pallet_system --extrinsic set,remark --output={output_path} -y - // --skip-parameters - assert!(!output_path.join("weights.rs").exists()); - assert!(!working_dir.join("pop-bench.toml").exists()); - command = pop( - working_dir, - [ - "bench", - "pallet", - &format!("--runtime={}", runtime_path.display()), - "--genesis-builder=runtime", - "--pallets", - "pallet_timestamp,pallet_system", - "--extrinsic", - "set,remark", - &format!("--output={}", output_path.join("weights.rs").display()), - "--skip-parameters", - "-y", - ], - ); - assert!(command.spawn()?.wait().await?.success()); - // Parse weights file. - assert!(output_path.join("weights.rs").exists()); - let content = fs::read_to_string(output_path.join("weights.rs"))?; - let expected = [ - "// Executed Command:".to_string(), - "// pop".to_string(), - "// bench".to_string(), - "// pallet".to_string(), - format!("// --runtime={}", runtime_path.display()), - "// --pallets=pallet_timestamp,pallet_system".to_string(), - "// --extrinsic=set,remark".to_string(), - "// --steps=50".to_string(), - format!("// --output={}", output_path.join("weights.rs").display()), - "// --genesis-builder=runtime".to_string(), - "// --skip-parameters".to_string(), - "// -y".to_string(), - ] - .join("\n"); - - assert!( - content.contains(&expected), - "expected command block not found.\nExpected:\n{}\n---\nContent:\n{}", - expected, - content - ); - assert!(working_dir.join("pop-bench.toml").exists()); - // Use the generated pop-bench.toml file: - // pop bench pallet --bench-file={working_dir.join("pop-bench.toml")} -y - command = pop( - working_dir, - [ - "bench", - "pallet", - &format!("--bench-file={}", working_dir.join("pop-bench.toml").display()), - "-y", - ], - ); - assert!(command.spawn()?.wait().await?.success()); Ok(()) } @@ -407,7 +71,7 @@ fn mock_build_process(temp_dir: &Path) -> Result<()> { async fn fetch_runtime(cache: &Path) -> Result { let name = "parachain_template_runtime.wasm"; let contents = ["parachain_template_runtime.wasm"]; - let binary = Binary::Source { + let binary = SourcedArchive::Source { name: name.to_string(), source: GitHub(ReleaseArchive { owner: "r0gue-io".into(), @@ -426,6 +90,7 @@ async fn fetch_runtime(cache: &Path) -> Result { }) .into(), cache: cache.to_path_buf(), + archive_type: ArchiveType::Binary, }; binary.source(true, &(), true).await?; Ok(name.to_string()) diff --git a/crates/pop-common/src/errors.rs b/crates/pop-common/src/errors.rs index ed28883ad..0770a02e5 100644 --- a/crates/pop-common/src/errors.rs +++ b/crates/pop-common/src/errors.rs @@ -39,6 +39,9 @@ pub enum Error { /// An error occurred during sourcing of a binary. #[error("SourceError error: {0}")] SourceError(#[from] sourcing::Error), + /// An error occurred parsing a Strum property + #[error("Property {0} isn't defined for {1}")] + StrumPropertyError(String, String), /// A template error occurred. #[error("TemplateError error: {0}")] TemplateError(#[from] templates::Error), diff --git a/crates/pop-common/src/sourcing/binary.rs b/crates/pop-common/src/sourcing/archive.rs similarity index 52% rename from crates/pop-common/src/sourcing/binary.rs rename to crates/pop-common/src/sourcing/archive.rs index 813ed425d..a3a24fed8 100644 --- a/crates/pop-common/src/sourcing/binary.rs +++ b/crates/pop-common/src/sourcing/archive.rs @@ -11,9 +11,21 @@ use crate::{ }; use std::path::{Path, PathBuf}; -/// A binary used to launch a node. +/// File extensions we allow in our sourcing +pub static ALLOWED_FILE_EXTENSIONS: [&str; 1] = [".json"]; + +/// The type of the Archive +#[derive(Debug, PartialEq, Clone, Copy)] +pub enum ArchiveType { + /// If the archive is a binary + Binary, + /// If the archive is a file + File, +} + +/// A sourced archive. #[derive(Debug, PartialEq)] -pub enum Binary { +pub enum SourcedArchive { /// A local binary. Local { /// The name of the binary. @@ -22,6 +34,8 @@ pub enum Binary { path: PathBuf, /// If applicable, the path to a manifest used to build the binary if missing. manifest: Option, + /// The archive type + archive_type: ArchiveType, }, /// A binary which needs to be sourced. Source { @@ -32,11 +46,13 @@ pub enum Binary { source: Box, /// The cache to be used to store the binary. cache: PathBuf, + /// The archive type + archive_type: ArchiveType, }, } -impl Binary { - /// Whether the binary exists. +impl SourcedArchive { + /// Whether the archive exists. pub fn exists(&self) -> bool { self.path().exists() } @@ -61,12 +77,12 @@ impl Binary { } } - /// Whether the binary is defined locally. + /// Whether the archive is defined locally. pub fn local(&self) -> bool { matches!(self, Self::Local { .. }) } - /// The name of the binary. + /// The name of the archive. pub fn name(&self) -> &str { match self { Self::Local { name, .. } => name, @@ -74,27 +90,50 @@ impl Binary { } } - /// The path of the binary. + /// The archive type. + pub fn archive_type(&self) -> ArchiveType { + match *self { + Self::Local { archive_type, .. } => archive_type, + Self::Source { archive_type, .. } => archive_type, + } + } + + /// The path of the archive. pub fn path(&self) -> PathBuf { match self { Self::Local { path, .. } => path.to_path_buf(), - Self::Source { name, cache, .. } => { + Self::Source { name, cache, archive_type, .. } => { // Determine whether a specific version is specified - self.version() - .map_or_else(|| cache.join(name), |v| cache.join(format!("{name}-{v}"))) + self.version().map_or_else( + || cache.join(name), + |v| match *archive_type { + ArchiveType::File => + if let Some(ext_pos) = name.rfind('.') { + cache.join(format!( + "{}-{}{}", + &name[..ext_pos], + v, + &name[ext_pos..] + )) + } else { + cache.join(format!("{name}-{v}")) + }, + _ => cache.join(format!("{name}-{v}")), + }, + ) }, } } - /// Attempts to resolve a version of a binary based on whether one is specified, an existing + /// Attempts to resolve a version of a archive based on whether one is specified, an existing /// version can be found cached locally, or uses the latest version. /// /// # Arguments - /// * `name` - The name of the binary. + /// * `name` - The name of the archive. /// * `specified` - If available, a version explicitly specified. /// * `available` - The available versions, which are used to check for existing matches already /// cached locally or the latest otherwise. - /// * `cache` - The location used for caching binaries. + /// * `cache` - The location used for caching archives. pub(super) fn resolve_version<'a>( name: &str, specified: Option<&'a str>, @@ -117,11 +156,11 @@ impl Binary { } } - /// Sources the binary. + /// Sources the archive. /// /// # Arguments - /// * `release` - Whether any binaries needing to be built should be done so using the release - /// profile. + /// * `release` - Whether any binary archives needing to be built should be done so using the + /// release profile. /// * `status` - Used to observe status updates. /// * `verbose` - Whether verbose output is required. pub async fn source( @@ -132,14 +171,14 @@ impl Binary { ) -> Result<(), Error> { match self { Self::Local { name, path, manifest, .. } => match manifest { - None => Err(Error::MissingBinary(format!( + None => Err(Error::MissingArchive(format!( "The {path:?} binary cannot be sourced automatically." ))), Some(manifest) => from_local_package(manifest, name, release, status, verbose).await, }, - Self::Source { source, cache, .. } => - source.source(cache, release, status, verbose).await, + Self::Source { source, cache, archive_type, .. } => + source.source(cache, release, status, verbose, *archive_type).await, } } @@ -149,6 +188,7 @@ impl Binary { let Self::Source { source, .. } = self else { return false; }; + let GitHub(ReleaseArchive { tag, latest, .. }) = source.as_ref() else { return false; }; @@ -194,7 +234,10 @@ mod tests { }; use anyhow::Result; use duct::cmd; - use std::fs::{File, create_dir_all}; + use std::{ + fs::{File, create_dir_all}, + os::unix::fs::PermissionsExt, + }; use tempfile::tempdir; use url::Url; @@ -205,7 +248,12 @@ mod tests { let path = temp_dir.path().join(name); File::create(&path)?; - let binary = Binary::Local { name: name.to_string(), path: path.clone(), manifest: None }; + let binary = SourcedArchive::Local { + name: name.to_string(), + path: path.clone(), + manifest: None, + archive_type: ArchiveType::Binary, + }; assert!(binary.exists()); assert_eq!(binary.latest(), None); @@ -226,7 +274,12 @@ mod tests { File::create(&path)?; let manifest = Some(temp_dir.path().join("Cargo.toml")); - let binary = Binary::Local { name: name.to_string(), path: path.clone(), manifest }; + let binary = SourcedArchive::Local { + name: name.to_string(), + path: path.clone(), + manifest, + archive_type: ArchiveType::Binary, + }; assert!(binary.exists()); assert_eq!(binary.latest(), None); @@ -249,18 +302,18 @@ mod tests { // Specified let specified = Some("v1.12.0"); assert_eq!( - Binary::resolve_version(name, specified, &available, temp_dir.path()), + SourcedArchive::resolve_version(name, specified, &available, temp_dir.path()), specified ); // Latest assert_eq!( - Binary::resolve_version(name, None, &available, temp_dir.path()).unwrap(), + SourcedArchive::resolve_version(name, None, &available, temp_dir.path()).unwrap(), "stable2409" ); // Cached File::create(temp_dir.path().join(format!("{name}-{}", available[1])))?; assert_eq!( - Binary::resolve_version(name, None, &available, temp_dir.path()).unwrap(), + SourcedArchive::resolve_version(name, None, &available, temp_dir.path()).unwrap(), available[1] ); Ok(()) @@ -279,10 +332,11 @@ mod tests { let path = temp_dir.path().join(name); File::create(&path)?; - let mut binary = Binary::Source { + let mut binary = SourcedArchive::Source { name: name.to_string(), source: Archive { url: url.to_string(), contents }.into(), cache: temp_dir.path().to_path_buf(), + archive_type: ArchiveType::Binary, }; assert!(binary.exists()); @@ -294,6 +348,7 @@ mod tests { assert_eq!(binary.version(), None); binary.use_latest(); assert_eq!(binary.version(), None); + assert_eq!(binary.archive_type(), ArchiveType::Binary); Ok(()) } @@ -310,7 +365,7 @@ mod tests { ); File::create(&path)?; - let mut binary = Binary::Source { + let mut binary = SourcedArchive::Source { name: package.to_string(), source: Git { url: url.clone(), @@ -321,6 +376,7 @@ mod tests { } .into(), cache: temp_dir.path().to_path_buf(), + archive_type: ArchiveType::Binary, }; assert!(binary.exists()); @@ -332,6 +388,7 @@ mod tests { assert_eq!(binary.version(), reference.as_deref()); binary.use_latest(); assert_eq!(binary.version(), reference.as_deref()); + assert_eq!(binary.archive_type(), ArchiveType::Binary); } Ok(()) @@ -353,7 +410,7 @@ mod tests { .join(tag.as_ref().map_or(name.to_string(), |t| format!("{name}-{t}"))); File::create(&path)?; for latest in [None, Some("polkadot-stable2503".to_string())] { - let mut binary = Binary::Source { + let mut binary = SourcedArchive::Source { name: name.to_string(), source: GitHub(ReleaseArchive { owner: owner.into(), @@ -372,6 +429,7 @@ mod tests { }) .into(), cache: temp_dir.path().to_path_buf(), + archive_type: ArchiveType::Binary, }; let latest = latest.as_ref().map(|l| l.replace("polkadot-", "")); @@ -387,6 +445,7 @@ mod tests { if latest.is_some() { assert_eq!(binary.version(), latest.as_deref()); } + assert_eq!(binary.archive_type(), ArchiveType::Binary); } } Ok(()) @@ -404,7 +463,7 @@ mod tests { .path() .join(reference.as_ref().map_or(package.to_string(), |t| format!("{package}-{t}"))); File::create(&path)?; - let mut binary = Binary::Source { + let mut binary = SourcedArchive::Source { name: package.to_string(), source: GitHub(SourceCodeArchive { owner: owner.to_string(), @@ -416,6 +475,7 @@ mod tests { }) .into(), cache: temp_dir.path().to_path_buf(), + archive_type: ArchiveType::Binary, }; assert!(binary.exists()); @@ -427,6 +487,7 @@ mod tests { assert_eq!(binary.version(), reference.as_deref()); binary.use_latest(); assert_eq!(binary.version(), reference.as_deref()); + assert_eq!(binary.archive_type(), ArchiveType::Binary); } Ok(()) } @@ -440,10 +501,11 @@ mod tests { let path = temp_dir.path().join(name); File::create(&path)?; - let mut binary = Binary::Source { + let mut binary = SourcedArchive::Source { name: name.to_string(), source: Source::Url { url: url.to_string(), name: name.to_string() }.into(), cache: temp_dir.path().to_path_buf(), + archive_type: ArchiveType::Binary, }; assert!(binary.exists()); @@ -455,6 +517,7 @@ mod tests { assert_eq!(binary.version(), None); binary.use_latest(); assert_eq!(binary.version(), None); + assert_eq!(binary.archive_type(), ArchiveType::Binary); Ok(()) } @@ -464,8 +527,8 @@ mod tests { let temp_dir = tempdir()?; let path = temp_dir.path().join(&name); assert!(matches!( - Binary::Local { name, path: path.clone(), manifest: None }.source(true, &Output, true).await, - Err(Error::MissingBinary(error)) if error == format!("The {path:?} binary cannot be sourced automatically.") + SourcedArchive::Local { name, path: path.clone(), manifest: None, archive_type: ArchiveType::Binary }.source(true, &Output, true).await, + Err(Error::MissingArchive(error)) if error == format!("The {path:?} binary cannot be sourced automatically.") )); Ok(()) } @@ -478,29 +541,322 @@ mod tests { let path = temp_dir.path().join(name); let manifest = Some(path.join("Cargo.toml")); let path = path.join("target/release").join(name); - Binary::Local { name: name.to_string(), path: path.clone(), manifest } - .source(true, &Output, true) - .await?; + SourcedArchive::Local { + name: name.to_string(), + path: path.clone(), + manifest, + archive_type: ArchiveType::Binary, + } + .source(true, &Output, true) + .await?; assert!(path.exists()); Ok(()) } #[tokio::test] - async fn sourcing_from_url_works() -> Result<()> { + async fn sourcing_binary_from_url_works() -> Result<()> { let name = "polkadot"; let url = "https://github.com/paritytech/polkadot-sdk/releases/latest/download/polkadot.asc"; let temp_dir = tempdir()?; let path = temp_dir.path().join(name); - Binary::Source { + SourcedArchive::Source { name: name.to_string(), source: Source::Url { url: url.to_string(), name: name.to_string() }.into(), cache: temp_dir.path().to_path_buf(), + archive_type: ArchiveType::Binary, } .source(true, &Output, true) .await?; assert!(path.exists()); Ok(()) } + + #[tokio::test] + async fn sourcing_file_from_url_works() -> Result<()> { + let name = "paseo-local.json"; + let url = + "https://github.com/paseo-network/runtimes/releases/download/v2.0.2/paseo-local.json"; + let temp_dir = tempdir()?; + let path = temp_dir.path().join(name); + + SourcedArchive::Source { + name: name.to_string(), + source: Source::Url { url: url.to_string(), name: name.to_string() }.into(), + cache: temp_dir.path().to_path_buf(), + archive_type: ArchiveType::File, + } + .source(true, &Output, true) + .await?; + assert!(path.exists()); + // Files should not have executable permissions + assert_eq!(std::fs::metadata(&path)?.permissions().mode() & 0o111, 0); + Ok(()) + } + + #[test] + fn file_archive_path_with_version_works() -> Result<()> { + let name = "paseo-local.json"; + let temp_dir = tempdir()?; + let version = "v2.0.2"; + + // Create a versioned file + let expected_path = temp_dir.path().join(format!("paseo-local-{}.json", version)); + File::create(&expected_path)?; + + let binary = SourcedArchive::Source { + name: name.to_string(), + source: Source::GitHub(crate::sourcing::GitHub::ReleaseArchive { + owner: "paseo-network".to_string(), + repository: "runtimes".to_string(), + tag: Some(version.to_string()), + tag_pattern: None, + prerelease: false, + version_comparator: crate::polkadot_sdk::sort_by_latest_semantic_version, + fallback: "v2.0.2".to_string(), + archive: format!("{}.json", name), + contents: vec![], + latest: None, + }) + .into(), + cache: temp_dir.path().to_path_buf(), + archive_type: ArchiveType::File, + }; + + assert_eq!(binary.path(), expected_path); + assert!(binary.exists()); + Ok(()) + } + + #[test] + fn archive_type_binary_works() -> Result<()> { + let name = "polkadot"; + let temp_dir = tempdir()?; + let path = temp_dir.path().join(name); + File::create(&path)?; + + // Test Local Binary + let local_binary = SourcedArchive::Local { + name: name.to_string(), + path: path.clone(), + manifest: None, + archive_type: ArchiveType::Binary, + }; + assert_eq!(local_binary.archive_type(), ArchiveType::Binary); + + // Test Source Binary + let source_binary = SourcedArchive::Source { + name: name.to_string(), + source: Source::Url { + url: "https://example.com/polkadot".to_string(), + name: name.to_string(), + } + .into(), + cache: temp_dir.path().to_path_buf(), + archive_type: ArchiveType::Binary, + }; + assert_eq!(source_binary.archive_type(), ArchiveType::Binary); + Ok(()) + } + + #[test] + fn archive_type_file_works() -> Result<()> { + let name = "chain-spec.json"; + let temp_dir = tempdir()?; + let path = temp_dir.path().join(name); + File::create(&path)?; + + // Test Local File + let local_file = SourcedArchive::Local { + name: name.to_string(), + path: path.clone(), + manifest: None, + archive_type: ArchiveType::File, + }; + assert_eq!(local_file.archive_type(), ArchiveType::File); + + // Test Source File + let source_file = SourcedArchive::Source { + name: name.to_string(), + source: Source::Url { + url: "https://example.com/chain-spec.json".to_string(), + name: name.to_string(), + } + .into(), + cache: temp_dir.path().to_path_buf(), + archive_type: ArchiveType::File, + }; + assert_eq!(source_file.archive_type(), ArchiveType::File); + Ok(()) + } + + #[test] + fn path_for_binary_without_version_works() -> Result<()> { + let name = "polkadot"; + let temp_dir = tempdir()?; + + let binary = SourcedArchive::Source { + name: name.to_string(), + source: Source::Url { + url: "https://example.com/polkadot".to_string(), + name: name.to_string(), + } + .into(), + cache: temp_dir.path().to_path_buf(), + archive_type: ArchiveType::Binary, + }; + + // Without version, path should be cache/name + assert_eq!(binary.path(), temp_dir.path().join(name)); + Ok(()) + } + + #[test] + fn path_for_binary_with_version_works() -> Result<()> { + let name = "polkadot"; + let version = "v1.0.0"; + let temp_dir = tempdir()?; + + let binary = SourcedArchive::Source { + name: name.to_string(), + source: Source::GitHub(crate::sourcing::GitHub::ReleaseArchive { + owner: "paritytech".to_string(), + repository: "polkadot-sdk".to_string(), + tag: Some(version.to_string()), + tag_pattern: None, + prerelease: false, + version_comparator: crate::polkadot_sdk::sort_by_latest_semantic_version, + fallback: "v1.0.0".to_string(), + archive: "polkadot.tar.gz".to_string(), + contents: vec![], + latest: None, + }) + .into(), + cache: temp_dir.path().to_path_buf(), + archive_type: ArchiveType::Binary, + }; + + // With version, path should be cache/name-version + assert_eq!(binary.path(), temp_dir.path().join(format!("{}-{}", name, version))); + Ok(()) + } + + #[test] + fn path_for_file_without_version_works() -> Result<()> { + let name = "chain-spec.json"; + let temp_dir = tempdir()?; + + let file = SourcedArchive::Source { + name: name.to_string(), + source: Source::Url { + url: "https://example.com/chain-spec.json".to_string(), + name: name.to_string(), + } + .into(), + cache: temp_dir.path().to_path_buf(), + archive_type: ArchiveType::File, + }; + + // Without version, path should be cache/name + assert_eq!(file.path(), temp_dir.path().join(name)); + Ok(()) + } + + #[test] + fn path_for_file_with_version_and_extension_works() -> Result<()> { + let name = "paseo-local.json"; + let version = "v2.0.2"; + let temp_dir = tempdir()?; + + let file = SourcedArchive::Source { + name: name.to_string(), + source: Source::GitHub(crate::sourcing::GitHub::ReleaseArchive { + owner: "paseo-network".to_string(), + repository: "runtimes".to_string(), + tag: Some(version.to_string()), + tag_pattern: None, + prerelease: false, + version_comparator: crate::polkadot_sdk::sort_by_latest_semantic_version, + fallback: "v2.0.2".to_string(), + archive: "paseo-local.json".to_string(), + contents: vec![], + latest: None, + }) + .into(), + cache: temp_dir.path().to_path_buf(), + archive_type: ArchiveType::File, + }; + + // With version and extension, path should be cache/name-version.ext + // e.g., paseo-local-v2.0.2.json + assert_eq!(file.path(), temp_dir.path().join(format!("paseo-local-{}.json", version))); + Ok(()) + } + + #[test] + fn path_for_file_with_version_no_extension_works() -> Result<()> { + let name = "chain-spec"; + let version = "v1.0.0"; + let temp_dir = tempdir()?; + + let file = SourcedArchive::Source { + name: name.to_string(), + source: Source::GitHub(crate::sourcing::GitHub::ReleaseArchive { + owner: "example".to_string(), + repository: "repo".to_string(), + tag: Some(version.to_string()), + tag_pattern: None, + prerelease: false, + version_comparator: crate::polkadot_sdk::sort_by_latest_semantic_version, + fallback: "v1.0.0".to_string(), + archive: "chain-spec".to_string(), + contents: vec![], + latest: None, + }) + .into(), + cache: temp_dir.path().to_path_buf(), + archive_type: ArchiveType::File, + }; + + // With version but no extension, path should be cache/name-version + assert_eq!(file.path(), temp_dir.path().join(format!("{}-{}", name, version))); + Ok(()) + } + + #[test] + fn path_for_local_binary_works() -> Result<()> { + let name = "polkadot"; + let temp_dir = tempdir()?; + let path = temp_dir.path().join("custom/path").join(name); + + let binary = SourcedArchive::Local { + name: name.to_string(), + path: path.clone(), + manifest: None, + archive_type: ArchiveType::Binary, + }; + + // Local binaries should return their exact path + assert_eq!(binary.path(), path); + Ok(()) + } + + #[test] + fn path_for_local_file_works() -> Result<()> { + let name = "chain-spec.json"; + let temp_dir = tempdir()?; + let path = temp_dir.path().join("custom/path").join(name); + + let file = SourcedArchive::Local { + name: name.to_string(), + path: path.clone(), + manifest: None, + archive_type: ArchiveType::File, + }; + + // Local files should return their exact path + assert_eq!(file.path(), path); + Ok(()) + } } diff --git a/crates/pop-common/src/sourcing/mod.rs b/crates/pop-common/src/sourcing/mod.rs index 7158b9369..dc0e18d57 100644 --- a/crates/pop-common/src/sourcing/mod.rs +++ b/crates/pop-common/src/sourcing/mod.rs @@ -1,7 +1,7 @@ // SPDX-License-Identifier: GPL-3.0 use crate::{Git, Release, SortedSlice, Status, api, git::GITHUB_API_CLIENT}; -pub use binary::*; +pub use archive::{ALLOWED_FILE_EXTENSIONS, ArchiveType, SourcedArchive}; use derivative::Derivative; use duct::cmd; use flate2::read::GzDecoder; @@ -21,7 +21,7 @@ use tempfile::{tempdir, tempfile}; use thiserror::Error; use url::Url; -mod binary; +mod archive; /// An error relating to the sourcing of binaries. #[derive(Error, Debug)] @@ -41,15 +41,15 @@ pub enum Error { /// An IO error occurred. #[error("IO error: {0}")] IO(#[from] std::io::Error), - /// A binary cannot be sourced. - #[error("Missing binary: {0}")] - MissingBinary(String), + /// An archive cannot be sourced. + #[error("Missing archive: {0}")] + MissingArchive(String), /// An error occurred during parsing. #[error("ParseError error: {0}")] ParseError(#[from] url::ParseError), } -/// The source of a binary. +/// The source of an archive #[derive(Clone, Debug, PartialEq)] pub enum Source { /// An archive for download. @@ -86,7 +86,7 @@ pub enum Source { } impl Source { - /// Sources the binary. + /// Sources the archive. /// /// # Arguments /// * `cache` - the cache to be used. @@ -94,12 +94,14 @@ impl Source { /// profile. /// * `status` - used to observe status updates. /// * `verbose` - whether verbose output is required. + /// * `archive_type` - the type of the archive. pub(super) async fn source( &self, cache: &Path, release: bool, status: &impl Status, verbose: bool, + archive_type: ArchiveType, ) -> Result<(), Error> { use Source::*; match self { @@ -108,7 +110,7 @@ impl Source { .iter() .map(|name| ArchiveFileSpec::new(name.into(), Some(cache.join(name)), true)) .collect(); - from_archive(url, &contents, status).await + from_archive(url, &contents, status, archive_type).await }, Git { url, reference, manifest, package, artifacts } => { let artifacts: Vec<_> = artifacts @@ -130,8 +132,8 @@ impl Source { ) .await }, - GitHub(source) => source.source(cache, release, status, verbose).await, - Url { url, name } => from_url(url, &cache.join(name), status).await, + GitHub(source) => source.source(cache, release, status, verbose, archive_type).await, + Url { url, name } => from_url(url, &cache.join(name), status, archive_type).await, } } @@ -216,12 +218,14 @@ impl GitHub { /// profile. /// * `status` - used to observe status updates. /// * `verbose` - whether verbose output is required. + /// * `archive_type` - The archive type async fn source( &self, cache: &Path, release: bool, status: &impl Status, verbose: bool, + archive_type: ArchiveType, ) -> Result<(), Error> { use GitHub::*; match self { @@ -260,7 +264,7 @@ impl GitHub { ), }) .collect(); - from_archive(&url, &contents, status).await + from_archive(&url, &contents, status, archive_type).await }, SourceCodeArchive { owner, repository, reference, manifest, package, artifacts } => { let artifacts: Vec<_> = artifacts @@ -367,7 +371,10 @@ impl GitHub { let cached_file_names = cached_files .filter_map(|f| f.ok().and_then(|f| f.file_name().into_string().ok())); for file in cached_file_names.filter(|f| cache_filter(f)) { - let version = file.replace(&format!("{name}-"), ""); + let mut version = file.replace(&format!("{name}-"), ""); + ALLOWED_FILE_EXTENSIONS.iter().for_each(|ext| { + version = version.strip_suffix(ext).unwrap_or(&version).to_owned(); + }); let tag = tag_pattern.as_ref().map_or_else( || version.to_string(), |pattern| pattern.resolve_tag(&version), @@ -385,7 +392,7 @@ impl GitHub { || { // Resolve the version to be used. let resolved_version = - Binary::resolve_version(name, None, &versions, cache); + SourcedArchive::resolve_version(name, None, &versions, cache); resolved_version.and_then(|v| binaries.get(v)).cloned() }, |v| { @@ -509,10 +516,12 @@ impl From<&str> for TagPattern { /// * `url` - The url of the archive. /// * `contents` - The contents within the archive which are required. /// * `status` - Used to observe status updates. +/// * `archive_type` - Whether the archive is a file or a binary async fn from_archive( url: &str, contents: &[ArchiveFileSpec], status: &impl Status, + archive_type: ArchiveType, ) -> Result<(), Error> { // Download archive status.update(&format!("Downloading from {url}...")); @@ -520,37 +529,57 @@ async fn from_archive( let mut file = tempfile()?; file.write_all(&response.bytes().await?)?; file.seek(SeekFrom::Start(0))?; - // Extract contents - status.update("Extracting from archive..."); - let tar = GzDecoder::new(file); - let mut archive = Archive::new(tar); - let temp_dir = tempdir()?; - let working_dir = temp_dir.path(); - archive.unpack(working_dir)?; - for ArchiveFileSpec { name, target, required } in contents { - let src = working_dir.join(name); - if src.exists() { - set_executable_permission(&src)?; - if let Some(target) = target && - let Err(_e) = rename(&src, target) - { - // If rename fails (e.g., due to cross-device linking), fallback to copy and - // remove - copy(&src, target)?; - std::fs::remove_file(&src)?; + match archive_type { + ArchiveType::Binary => { + // Extract contents from tar.gz archive + status.update("Extracting from archive..."); + let tar = GzDecoder::new(file); + let mut archive = Archive::new(tar); + let temp_dir = tempdir()?; + let working_dir = temp_dir.path(); + archive.unpack(working_dir)?; + for ArchiveFileSpec { name, target, required } in contents { + let src = working_dir.join(name); + if src.exists() { + set_executable_permission(&src)?; + if let Some(target) = target && + let Err(_e) = rename(&src, target) + { + // If rename fails (e.g., due to cross-device linking), fallback to copy and + // remove + copy(&src, target)?; + std::fs::remove_file(&src)?; + } + } else if *required { + return Err(Error::ArchiveError(format!( + "Expected file '{}' in archive, but it was not found.", + name + ))); + } } - } else if *required { - return Err(Error::ArchiveError(format!( - "Expected file '{}' in archive, but it was not found.", - name - ))); - } + }, + ArchiveType::File => { + if let Some(ArchiveFileSpec { name, target: Some(target), .. }) = contents.first() { + let final_target = if let Some(ext) = Path::new(name).extension() { + PathBuf::from(format!("{}.{}", target.display(), ext.to_string_lossy())) + } else { + target.to_path_buf() + }; + let mut target_file = File::create(&final_target)?; + std::io::copy(&mut file, &mut target_file)?; + } else { + return Err(Error::ArchiveError( + "File archive requires exactly one target path".to_owned(), + )); + } + }, } + status.update("Sourcing complete."); Ok(()) } -/// Source binary by cloning a git repository and then building. +/// Source a binary by cloning a git repository and then building. /// /// # Arguments /// * `url` - The url of the repository. @@ -705,10 +734,16 @@ pub(crate) async fn from_local_package( /// * `url` - The url of the binary. /// * `path` - The (local) destination path. /// * `status` - Used to observe status updates. -async fn from_url(url: &str, path: &Path, status: &impl Status) -> Result<(), Error> { +/// * `archive-type` - Archive type +async fn from_url( + url: &str, + path: &Path, + status: &impl Status, + archive_type: ArchiveType, +) -> Result<(), Error> { // Download the binary status.update(&format!("Downloading from {url}...")); - download(url, path).await?; + download(url, path, archive_type).await?; status.update("Sourcing complete."); Ok(()) } @@ -767,13 +802,16 @@ async fn build( /// # Arguments /// * `url` - The url of the file. /// * `path` - The (local) destination path. -async fn download(url: &str, dest: &Path) -> Result<(), Error> { +/// * `archive_type` - The archive type. +async fn download(url: &str, dest: &Path, archive_type: ArchiveType) -> Result<(), Error> { // Download to the destination path let response = reqwest::get(url).await?.error_for_status()?; let mut file = File::create(dest)?; file.write_all(&response.bytes().await?)?; - // Make executable - set_executable_permission(dest)?; + // Make executable if binary + if let ArchiveType::Binary = archive_type { + set_executable_permission(dest)?; + } Ok(()) } @@ -803,7 +841,7 @@ pub(super) mod tests { let temp_dir = tempdir()?; Source::Archive { url, contents: contents.clone() } - .source(temp_dir.path(), true, &Output, true) + .source(temp_dir.path(), true, &Output, true, ArchiveType::Binary) .await?; for item in contents { assert!(temp_dir.path().join(item).exists()); @@ -840,7 +878,7 @@ pub(super) mod tests { package: package.clone(), artifacts: vec![package.clone()], } - .source(temp_dir.path(), true, &Output, true) + .source(temp_dir.path(), true, &Output, true, ArchiveType::Binary) .await?; assert!(temp_dir.path().join(package).exists()); Ok(()) @@ -883,7 +921,7 @@ pub(super) mod tests { package: package.clone(), artifacts: vec![package.clone()], } - .source(temp_dir.path(), true, &Output, true) + .source(temp_dir.path(), true, &Output, true, ArchiveType::Binary) .await?; assert!(temp_dir.path().join(format!("{package}-{initial_commit}")).exists()); Ok(()) @@ -912,7 +950,7 @@ pub(super) mod tests { contents: contents.map(|n| ArchiveFileSpec::new(n.into(), None, true)).to_vec(), latest: None, }) - .source(temp_dir.path(), true, &Output, true) + .source(temp_dir.path(), true, &Output, true, ArchiveType::Binary) .await?; for item in contents { assert!(temp_dir.path().join(format!("{item}-{version}")).exists()); @@ -1020,7 +1058,7 @@ pub(super) mod tests { .to_vec(), latest: None, }) - .source(temp_dir.path(), true, &Output, true) + .source(temp_dir.path(), true, &Output, true, ArchiveType::Binary) .await?; for item in contents { assert!(temp_dir.path().join(format!("{prefix}-{item}-{version}")).exists()); @@ -1051,7 +1089,7 @@ pub(super) mod tests { contents: contents.map(|n| ArchiveFileSpec::new(n.into(), None, true)).to_vec(), latest: None, }) - .source(temp_dir.path(), true, &Output, true) + .source(temp_dir.path(), true, &Output, true, ArchiveType::Binary) .await?; for item in contents { assert!(temp_dir.path().join(item).exists()); @@ -1076,7 +1114,7 @@ pub(super) mod tests { package: package.clone(), artifacts: vec![package.clone()], }) - .source(temp_dir.path(), true, &Output, true) + .source(temp_dir.path(), true, &Output, true, ArchiveType::Binary) .await?; assert!(temp_dir.path().join(format!("{package}-{initial_commit}")).exists()); Ok(()) @@ -1121,7 +1159,7 @@ pub(super) mod tests { package: package.clone(), artifacts: vec![package.clone()], }) - .source(temp_dir.path(), true, &Output, true) + .source(temp_dir.path(), true, &Output, true, ArchiveType::Binary) .await?; assert!(temp_dir.path().join(package).exists()); Ok(()) @@ -1136,7 +1174,7 @@ pub(super) mod tests { let temp_dir = tempdir()?; Source::Url { url, name: name.into() } - .source(temp_dir.path(), false, &Output, true) + .source(temp_dir.path(), false, &Output, true, ArchiveType::Binary) .await?; assert!(temp_dir.path().join(name).exists()); Ok(()) @@ -1167,7 +1205,7 @@ pub(super) mod tests { .map(|b| ArchiveFileSpec::new(b.into(), Some(temp_dir.path().join(b)), true)) .collect(); - from_archive(url, &contents, &Output).await?; + from_archive(url, &contents, &Output, ArchiveType::Binary).await?; for ArchiveFileSpec { target, .. } in contents { assert!(target.unwrap().exists()); } @@ -1266,12 +1304,49 @@ pub(super) mod tests { let temp_dir = tempdir()?; let path = temp_dir.path().join("polkadot"); - from_url(url, &path, &Output).await?; + from_url(url, &path, &Output, ArchiveType::Binary).await?; assert!(path.exists()); assert_ne!(metadata(path)?.permissions().mode() & 0o755, 0); Ok(()) } + #[tokio::test] + async fn from_url_file_works() -> anyhow::Result<()> { + let url = + "https://github.com/paseo-network/runtimes/releases/download/v2.0.2/paseo-local.json"; + let temp_dir = tempdir()?; + let path = temp_dir.path().join("paseo-local.json"); + + from_url(url, &path, &Output, ArchiveType::File).await?; + assert!(path.exists()); + // Files should not have executable permissions + assert_eq!(metadata(&path)?.permissions().mode() & 0o111, 0); + Ok(()) + } + + #[tokio::test] + async fn from_archive_file_works() -> anyhow::Result<()> { + let temp_dir = tempdir()?; + let url = + "https://github.com/paseo-network/runtimes/releases/download/v2.0.2/paseo-local.json"; + let contents: Vec<_> = vec![ArchiveFileSpec::new( + "paseo-local.json".into(), + Some(temp_dir.path().join("paseo-local")), + true, + )]; + + from_archive(url, &contents, &Output, ArchiveType::File).await?; + for ArchiveFileSpec { target, .. } in contents { + let mut target_path = target.unwrap(); + // The extension is appended to the target path + target_path.set_extension("json"); + assert!(target_path.exists()); + // Files should not have executable permissions + assert_eq!(metadata(&target_path)?.permissions().mode() & 0o111, 0); + } + Ok(()) + } + #[test] fn tag_pattern_works() { let pattern: TagPattern = "polkadot-{version}".into(); @@ -1314,8 +1389,14 @@ pub mod traits { /// The source of a binary. pub trait Source { + /// An error occurred on these trait calls. + type Error; + /// The name of the binary. - fn binary(&self) -> &'static str; + fn binary(&self) -> Result<&'static str, Self::Error>; + + /// The name of the file + fn file(&self) -> Result<&'static str, Self::Error>; /// The fallback version to be used when the latest version cannot be determined. fn fallback(&self) -> &str; @@ -1334,9 +1415,21 @@ pub mod traits { fn tag_pattern(&self) -> Option<&str>; } - impl Source for T { - fn binary(&self) -> &'static str { - self.get_str("Binary").expect("expected specification of `Binary` name") + impl Source for T { + type Error = crate::Error; + + fn binary(&self) -> Result<&'static str, Self::Error> { + self.get_str("Binary").ok_or(Self::Error::StrumPropertyError( + "Binary".to_owned(), + format!("{:?}", self), + )) + } + + fn file(&self) -> Result<&'static str, Self::Error> { + self.get_str("File").ok_or(Self::Error::StrumPropertyError( + "File".to_owned(), + format!("{:?}", self), + )) } fn fallback(&self) -> &str { @@ -1351,7 +1444,7 @@ pub mod traits { } } - impl Repository for T { + impl Repository for T { fn repository(&self) -> &str { self.get_str("Repository").expect("expected specification of `Repository` url") } @@ -1367,7 +1460,7 @@ pub mod traits { use super::enums::{Repository, Source}; use strum_macros::{EnumProperty, VariantArray}; - #[derive(EnumProperty, VariantArray)] + #[derive(EnumProperty, VariantArray, Debug)] pub(super) enum Chain { #[strum(props( Repository = "https://github.com/paritytech/polkadot-sdk", @@ -1383,7 +1476,7 @@ pub mod traits { #[test] fn binary_works() { - assert_eq!("polkadot", Chain::Polkadot.binary()) + assert_eq!("polkadot", Chain::Polkadot.binary().unwrap()) } #[test] diff --git a/crates/pop-common/src/test_env.rs b/crates/pop-common/src/test_env.rs index 2d50289a1..4fe93c62e 100644 --- a/crates/pop-common/src/test_env.rs +++ b/crates/pop-common/src/test_env.rs @@ -4,7 +4,9 @@ use crate::{ Error, find_free_port, polkadot_sdk::sort_by_latest_semantic_version, set_executable_permission, - sourcing::{ArchiveFileSpec, Binary, GitHub::ReleaseArchive, Source::GitHub}, + sourcing::{ + ArchiveFileSpec, ArchiveType, GitHub::ReleaseArchive, Source::GitHub, SourcedArchive, + }, }; use serde_json::json; @@ -66,7 +68,7 @@ impl TestNode { let random_port = find_free_port(None); let cache = temp_dir.path().to_path_buf(); - let binary = Binary::Source { + let binary = SourcedArchive::Source { name: "ink-node".to_string(), source: GitHub(ReleaseArchive { owner: "use-ink".into(), @@ -82,6 +84,7 @@ impl TestNode { }) .into(), cache: cache.to_path_buf(), + archive_type: ArchiveType::Binary, }; binary.source(false, &(), true).await?; set_executable_permission(binary.path())?; diff --git a/crates/pop-contracts/src/node.rs b/crates/pop-contracts/src/node.rs index c42017a71..8c3e0f441 100644 --- a/crates/pop-contracts/src/node.rs +++ b/crates/pop-contracts/src/node.rs @@ -6,9 +6,9 @@ use pop_common::{ Error, GitHub, polkadot_sdk::sort_by_latest_semantic_version, sourcing::{ - Binary, + ArchiveType, GitHub::ReleaseArchive, - Source, + Source, SourcedArchive, traits::{ Source as SourceT, enums::{Source as _, *}, @@ -92,24 +92,30 @@ impl SourceT for Chain { } /// Retrieves the latest release of the contracts node binary, resolves its version, and constructs -/// a `Binary::Source` with the specified cache path. +/// a `SourcedArchive::Source` with the specified cache path. /// /// # Arguments /// * `cache` - The cache directory path. /// * `version` - The specific version used for the ink-node (`None` will use the latest available /// version). -pub async fn ink_node_generator(cache: PathBuf, version: Option<&str>) -> Result { +pub async fn ink_node_generator( + cache: PathBuf, + version: Option<&str>, +) -> Result { node_generator_inner(&Chain::ContractsNode, cache, version).await } /// Retrieves the latest release of the Ethereum RPC binary, resolves its version, and constructs -/// a `Binary::Source` with the specified cache path. +/// a `SourcedArchive::Source` with the specified cache path. /// /// # Arguments /// * `cache` - The cache directory path. /// * `version` - The specific version used for the eth-rpc (`None` will use the latest available /// version). -pub async fn eth_rpc_generator(cache: PathBuf, version: Option<&str>) -> Result { +pub async fn eth_rpc_generator( + cache: PathBuf, + version: Option<&str>, +) -> Result { node_generator_inner(&Chain::EthRpc, cache, version).await } @@ -117,14 +123,14 @@ async fn node_generator_inner( chain: &Chain, cache: PathBuf, version: Option<&str>, -) -> Result { - let name = chain.binary().to_string(); +) -> Result { + let name = chain.binary()?.to_string(); let source = chain .source()? .resolve(&name, version, &cache, |f| prefix(f, &name)) .await .into(); - Ok(Binary::Source { name, source, cache }) + Ok(SourcedArchive::Source { name, source, cache, archive_type: ArchiveType::Binary }) } /// Runs the latest version of the `ink-node` in the background. @@ -259,8 +265,8 @@ mod tests { let cache = temp_dir.path().join("cache"); let binary = ink_node_generator(cache.clone(), Some(version)).await?; - assert!(matches!(binary, Binary::Source { name, source, cache} - if name == expected.binary() && + assert!(matches!(binary, SourcedArchive::Source { name, source, cache, archive_type} + if name == expected.binary().unwrap() && *source == Source::GitHub(ReleaseArchive { owner: owner.to_string(), repository: BIN_NAME.to_string(), @@ -274,7 +280,7 @@ mod tests { latest: None, }) && - cache == cache + cache == cache && archive_type == ArchiveType::Binary )); } Ok(())