From 47c97795689eab949b07bd4c929dd3064d76fad2 Mon Sep 17 00:00:00 2001 From: sanger-tolsoft <105875386+sanger-tolsoft@users.noreply.github.com> Date: Fri, 20 Jun 2025 09:37:34 +0000 Subject: [PATCH 1/9] Template update for nf-core/tools version 3.3.1 --- .editorconfig | 37 ----- .github/CONTRIBUTING.md | 2 +- .github/actions/get-shards/action.yml | 69 +++++++++ .github/actions/nf-test/action.yml | 113 ++++++++++++++ .github/workflows/ci.yml | 88 ----------- .github/workflows/clean-up.yml | 2 +- .github/workflows/download_pipeline.yml | 20 +-- .../{fix-linting.yml => fix_linting.yml} | 4 +- .github/workflows/linting.yml | 15 +- .github/workflows/linting_comment.yml | 4 +- .github/workflows/nf-test.yml | 139 ++++++++++++++++++ ...mment.yml => template-version-comment.yml} | 2 +- .nf-core.yml | 2 +- .pre-commit-config.yaml | 26 +++- .prettierrc.yml | 5 + README.md | 5 +- conf/base.config | 5 +- nextflow.config | 17 ++- nf-test.config | 24 +++ ro-crate-metadata.json | 34 ++--- .../utils_nfcore_readmapping_pipeline/main.nf | 1 - tests/.nftignore | 2 + tests/default.nf.test | 35 +++++ tests/nextflow.config | 12 ++ 24 files changed, 480 insertions(+), 183 deletions(-) delete mode 100644 .editorconfig create mode 100644 .github/actions/get-shards/action.yml create mode 100644 .github/actions/nf-test/action.yml delete mode 100644 .github/workflows/ci.yml rename .github/workflows/{fix-linting.yml => fix_linting.yml} (96%) create mode 100644 .github/workflows/nf-test.yml rename .github/workflows/{template_version_comment.yml => template-version-comment.yml} (95%) create mode 100644 nf-test.config create mode 100644 tests/.nftignore create mode 100644 tests/default.nf.test create mode 100644 tests/nextflow.config diff --git a/.editorconfig b/.editorconfig deleted file mode 100644 index 6d9b74cc..00000000 --- a/.editorconfig +++ /dev/null @@ -1,37 +0,0 @@ -root = true - -[*] -charset = utf-8 -end_of_line = lf -insert_final_newline = true -trim_trailing_whitespace = true -indent_size = 4 -indent_style = space - -[*.{md,yml,yaml,html,css,scss,js}] -indent_size = 2 - -# These files are edited and tested upstream in nf-core/modules -[/modules/nf-core/**] -charset = unset -end_of_line = unset -insert_final_newline = unset -trim_trailing_whitespace = unset -indent_style = unset -[/subworkflows/nf-core/**] -charset = unset -end_of_line = unset -insert_final_newline = unset -trim_trailing_whitespace = unset -indent_style = unset - -[/assets/email*] -indent_size = unset - -# ignore python and markdown -[*.{py,md}] -indent_style = unset - -# ignore ro-crate metadata files -[**/ro-crate-metadata.json] -insert_final_newline = unset diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md index 799d808f..05572888 100644 --- a/.github/CONTRIBUTING.md +++ b/.github/CONTRIBUTING.md @@ -71,7 +71,7 @@ If you wish to contribute a new step, please use the following coding standards: 5. Add any new parameters to `nextflow_schema.json` with help text (via the `nf-core pipelines schema build` tool). 6. Add sanity checks and validation for all relevant parameters. 7. Perform local tests to validate that the new code works as expected. -8. If applicable, add a new test command in `.github/workflow/ci.yml`. +8. If applicable, add a new test in the `tests` directory. ### Default values diff --git a/.github/actions/get-shards/action.yml b/.github/actions/get-shards/action.yml new file mode 100644 index 00000000..34085279 --- /dev/null +++ b/.github/actions/get-shards/action.yml @@ -0,0 +1,69 @@ +name: "Get number of shards" +description: "Get the number of nf-test shards for the current CI job" +inputs: + max_shards: + description: "Maximum number of shards allowed" + required: true + paths: + description: "Component paths to test" + required: false + tags: + description: "Tags to pass as argument for nf-test --tag parameter" + required: false +outputs: + shard: + description: "Array of shard numbers" + value: ${{ steps.shards.outputs.shard }} + total_shards: + description: "Total number of shards" + value: ${{ steps.shards.outputs.total_shards }} +runs: + using: "composite" + steps: + - name: Install nf-test + uses: nf-core/setup-nf-test@v1 + with: + version: ${{ env.NFT_VER }} + - name: Get number of shards + id: shards + shell: bash + run: | + # Run nf-test with dynamic parameter + nftest_output=$(nf-test test \ + --profile +docker \ + $(if [ -n "${{ inputs.tags }}" ]; then echo "--tag ${{ inputs.tags }}"; fi) \ + --dry-run \ + --ci \ + --changed-since HEAD^) || { + echo "nf-test command failed with exit code $?" + echo "Full output: $nftest_output" + exit 1 + } + echo "nf-test dry-run output: $nftest_output" + + # Default values for shard and total_shards + shard="[]" + total_shards=0 + + # Check if there are related tests + if echo "$nftest_output" | grep -q 'No tests to execute'; then + echo "No related tests found." + else + # Extract the number of related tests + number_of_shards=$(echo "$nftest_output" | sed -n 's|.*Executed \([0-9]*\) tests.*|\1|p') + if [[ -n "$number_of_shards" && "$number_of_shards" -gt 0 ]]; then + shards_to_run=$(( $number_of_shards < ${{ inputs.max_shards }} ? $number_of_shards : ${{ inputs.max_shards }} )) + shard=$(seq 1 "$shards_to_run" | jq -R . | jq -c -s .) + total_shards="$shards_to_run" + else + echo "Unexpected output format. Falling back to default values." + fi + fi + + # Write to GitHub Actions outputs + echo "shard=$shard" >> $GITHUB_OUTPUT + echo "total_shards=$total_shards" >> $GITHUB_OUTPUT + + # Debugging output + echo "Final shard array: $shard" + echo "Total number of shards: $total_shards" diff --git a/.github/actions/nf-test/action.yml b/.github/actions/nf-test/action.yml new file mode 100644 index 00000000..243e7823 --- /dev/null +++ b/.github/actions/nf-test/action.yml @@ -0,0 +1,113 @@ +name: "nf-test Action" +description: "Runs nf-test with common setup steps" +inputs: + profile: + description: "Profile to use" + required: true + shard: + description: "Shard number for this CI job" + required: true + total_shards: + description: "Total number of test shards(NOT the total number of matrix jobs)" + required: true + paths: + description: "Test paths" + required: true + tags: + description: "Tags to pass as argument for nf-test --tag parameter" + required: false +runs: + using: "composite" + steps: + - name: Setup Nextflow + uses: nf-core/setup-nextflow@v2 + with: + version: "${{ env.NXF_VERSION }}" + + - name: Set up Python + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 + with: + python-version: "3.13" + + - name: Install nf-test + uses: nf-core/setup-nf-test@v1 + with: + version: "${{ env.NFT_VER }}" + install-pdiff: true + + - name: Setup apptainer + if: contains(inputs.profile, 'singularity') + uses: eWaterCycle/setup-apptainer@main + + - name: Set up Singularity + if: contains(inputs.profile, 'singularity') + shell: bash + run: | + mkdir -p $NXF_SINGULARITY_CACHEDIR + mkdir -p $NXF_SINGULARITY_LIBRARYDIR + + - name: Conda setup + if: contains(inputs.profile, 'conda') + uses: conda-incubator/setup-miniconda@505e6394dae86d6a5c7fbb6e3fb8938e3e863830 # v3 + with: + auto-update-conda: true + conda-solver: libmamba + conda-remove-defaults: true + + # TODO Skip failing conda tests and document their failures + # https://github.com/nf-core/modules/issues/7017 + - name: Run nf-test + shell: bash + env: + NFT_DIFF: ${{ env.NFT_DIFF }} + NFT_DIFF_ARGS: ${{ env.NFT_DIFF_ARGS }} + NFT_WORKDIR: ${{ env.NFT_WORKDIR }} + run: | + nf-test test \ + --profile=+${{ inputs.profile }} \ + $(if [ -n "${{ inputs.tags }}" ]; then echo "--tag ${{ inputs.tags }}"; fi) \ + --ci \ + --changed-since HEAD^ \ + --verbose \ + --tap=test.tap \ + --shard ${{ inputs.shard }}/${{ inputs.total_shards }} + + # Save the absolute path of the test.tap file to the output + echo "tap_file_path=$(realpath test.tap)" >> $GITHUB_OUTPUT + + - name: Generate test summary + if: always() + shell: bash + run: | + # Add header if it doesn't exist (using a token file to track this) + if [ ! -f ".summary_header" ]; then + echo "# 🚀 nf-test results" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "| Status | Test Name | Profile | Shard |" >> $GITHUB_STEP_SUMMARY + echo "|:------:|-----------|---------|-------|" >> $GITHUB_STEP_SUMMARY + touch .summary_header + fi + + if [ -f test.tap ]; then + while IFS= read -r line; do + if [[ $line =~ ^ok ]]; then + test_name="${line#ok }" + # Remove the test number from the beginning + test_name="${test_name#* }" + echo "| ✅ | ${test_name} | ${{ inputs.profile }} | ${{ inputs.shard }}/${{ inputs.total_shards }} |" >> $GITHUB_STEP_SUMMARY + elif [[ $line =~ ^not\ ok ]]; then + test_name="${line#not ok }" + # Remove the test number from the beginning + test_name="${test_name#* }" + echo "| ❌ | ${test_name} | ${{ inputs.profile }} | ${{ inputs.shard }}/${{ inputs.total_shards }} |" >> $GITHUB_STEP_SUMMARY + fi + done < test.tap + else + echo "| ⚠️ | No test results found | ${{ inputs.profile }} | ${{ inputs.shard }}/${{ inputs.total_shards }} |" >> $GITHUB_STEP_SUMMARY + fi + + - name: Clean up + if: always() + shell: bash + run: | + sudo rm -rf /home/ubuntu/tests/ diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml deleted file mode 100644 index 3b44b6dc..00000000 --- a/.github/workflows/ci.yml +++ /dev/null @@ -1,88 +0,0 @@ -name: nf-core CI -# This workflow runs the pipeline with the minimal test dataset to check that it completes without any syntax errors -on: - push: - branches: - - dev - pull_request: - release: - types: [published] - workflow_dispatch: - -env: - NXF_ANSI_LOG: false - NXF_SINGULARITY_CACHEDIR: ${{ github.workspace }}/.singularity - NXF_SINGULARITY_LIBRARYDIR: ${{ github.workspace }}/.singularity - -concurrency: - group: "${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}" - cancel-in-progress: true - -jobs: - test: - name: "Run pipeline with test data (${{ matrix.NXF_VER }} | ${{ matrix.test_name }} | ${{ matrix.profile }})" - # Only run on push if this is the nf-core dev branch (merged PRs) - if: "${{ github.event_name != 'push' || (github.event_name == 'push' && github.repository == 'sanger-tol/readmapping') }}" - runs-on: ubuntu-latest - strategy: - matrix: - NXF_VER: - - "24.04.2" - - "latest-everything" - profile: - - "conda" - - "docker" - - "singularity" - test_name: - - "test" - isMaster: - - ${{ github.base_ref == 'master' }} - # Exclude conda and singularity on dev - exclude: - - isMaster: false - profile: "conda" - - isMaster: false - profile: "singularity" - steps: - - name: Check out pipeline code - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 - with: - fetch-depth: 0 - - - name: Set up Nextflow - uses: nf-core/setup-nextflow@v2 - with: - version: "${{ matrix.NXF_VER }}" - - - name: Set up Apptainer - if: matrix.profile == 'singularity' - uses: eWaterCycle/setup-apptainer@main - - - name: Set up Singularity - if: matrix.profile == 'singularity' - run: | - mkdir -p $NXF_SINGULARITY_CACHEDIR - mkdir -p $NXF_SINGULARITY_LIBRARYDIR - - - name: Set up Miniconda - if: matrix.profile == 'conda' - uses: conda-incubator/setup-miniconda@a4260408e20b96e80095f42ff7f1a15b27dd94ca # v3 - with: - miniconda-version: "latest" - auto-update-conda: true - conda-solver: libmamba - channels: conda-forge,bioconda - - - name: Set up Conda - if: matrix.profile == 'conda' - run: | - echo $(realpath $CONDA)/condabin >> $GITHUB_PATH - echo $(realpath python) >> $GITHUB_PATH - - - name: Clean up Disk space - uses: jlumbroso/free-disk-space@54081f138730dfa15788a46383842cd2f914a1be # v1.3.1 - - - name: "Run pipeline with test data ${{ matrix.NXF_VER }} | ${{ matrix.test_name }} | ${{ matrix.profile }}" - continue-on-error: ${{ matrix.NXF_VER == 'latest-everything' }} - run: | - nextflow run ${GITHUB_WORKSPACE} -profile ${{ matrix.test_name }},${{ matrix.profile }} --outdir ./results diff --git a/.github/workflows/clean-up.yml b/.github/workflows/clean-up.yml index 0b6b1f27..ac030fd5 100644 --- a/.github/workflows/clean-up.yml +++ b/.github/workflows/clean-up.yml @@ -10,7 +10,7 @@ jobs: issues: write pull-requests: write steps: - - uses: actions/stale@28ca1036281a5e5922ead5184a1bbf96e5fc984e # v9 + - uses: actions/stale@5bef64f19d7facfb25b37b414482c7164d639639 # v9 with: stale-issue-message: "This issue has been tagged as awaiting-changes or awaiting-feedback by an nf-core contributor. Remove stale label or add a comment otherwise this issue will be closed in 20 days." stale-pr-message: "This PR has been tagged as awaiting-changes or awaiting-feedback by an nf-core contributor. Remove stale label or add a comment if it is still useful." diff --git a/.github/workflows/download_pipeline.yml b/.github/workflows/download_pipeline.yml index ab06316e..999bcc38 100644 --- a/.github/workflows/download_pipeline.yml +++ b/.github/workflows/download_pipeline.yml @@ -12,14 +12,6 @@ on: required: true default: "dev" pull_request: - types: - - opened - - edited - - synchronize - branches: - - main - - master - pull_request_target: branches: - main - master @@ -52,9 +44,9 @@ jobs: - name: Disk space cleanup uses: jlumbroso/free-disk-space@54081f138730dfa15788a46383842cd2f914a1be # v1.3.1 - - uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5 + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 with: - python-version: "3.12" + python-version: "3.13" architecture: "x64" - name: Setup Apptainer @@ -120,6 +112,7 @@ jobs: echo "IMAGE_COUNT_AFTER=$image_count" >> "$GITHUB_OUTPUT" - name: Compare container image counts + id: count_comparison run: | if [ "${{ steps.count_initial.outputs.IMAGE_COUNT_INITIAL }}" -ne "${{ steps.count_afterwards.outputs.IMAGE_COUNT_AFTER }}" ]; then initial_count=${{ steps.count_initial.outputs.IMAGE_COUNT_INITIAL }} @@ -132,3 +125,10 @@ jobs: else echo "The pipeline can be downloaded successfully!" fi + + - name: Upload Nextflow logfile for debugging purposes + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: nextflow_logfile.txt + path: .nextflow.log* + include-hidden-files: true diff --git a/.github/workflows/fix-linting.yml b/.github/workflows/fix_linting.yml similarity index 96% rename from .github/workflows/fix-linting.yml rename to .github/workflows/fix_linting.yml index 856252bb..2e269b36 100644 --- a/.github/workflows/fix-linting.yml +++ b/.github/workflows/fix_linting.yml @@ -32,9 +32,9 @@ jobs: GITHUB_TOKEN: ${{ secrets.nf_core_bot_auth_token }} # Install and run pre-commit - - uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5 + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 with: - python-version: "3.12" + python-version: "3.13" - name: Install pre-commit run: pip install pre-commit diff --git a/.github/workflows/linting.yml b/.github/workflows/linting.yml index dbd52d5a..f2d7d1dd 100644 --- a/.github/workflows/linting.yml +++ b/.github/workflows/linting.yml @@ -3,9 +3,6 @@ name: nf-core linting # It runs the `nf-core pipelines lint` and markdown lint tests to ensure # that the code meets the nf-core guidelines. on: - push: - branches: - - dev pull_request: release: types: [published] @@ -17,9 +14,9 @@ jobs: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 - name: Set up Python 3.12 - uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5 + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 with: - python-version: "3.12" + python-version: "3.13" - name: Install pre-commit run: pip install pre-commit @@ -36,13 +33,13 @@ jobs: - name: Install Nextflow uses: nf-core/setup-nextflow@v2 - - uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5 + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 with: - python-version: "3.12" + python-version: "3.13" architecture: "x64" - name: read .nf-core.yml - uses: pietrobolcato/action-read-yaml@1.1.0 + uses: pietrobolcato/action-read-yaml@9f13718d61111b69f30ab4ac683e67a56d254e1d # 1.1.0 id: read_yml with: config: ${{ github.workspace }}/.nf-core.yml @@ -74,7 +71,7 @@ jobs: - name: Upload linting log file artifact if: ${{ always() }} - uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4 + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 with: name: linting-logs path: | diff --git a/.github/workflows/linting_comment.yml b/.github/workflows/linting_comment.yml index 95b6b6af..7e8050fb 100644 --- a/.github/workflows/linting_comment.yml +++ b/.github/workflows/linting_comment.yml @@ -11,7 +11,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Download lint results - uses: dawidd6/action-download-artifact@20319c5641d495c8a52e688b7dc5fada6c3a9fbc # v8 + uses: dawidd6/action-download-artifact@4c1e823582f43b179e2cbb49c3eade4e41f992e2 # v10 with: workflow: linting.yml workflow_conclusion: completed @@ -21,7 +21,7 @@ jobs: run: echo "pr_number=$(cat linting-logs/PR_number.txt)" >> $GITHUB_OUTPUT - name: Post PR comment - uses: marocchino/sticky-pull-request-comment@331f8f5b4215f0445d3c07b4967662a32a2d3e31 # v2 + uses: marocchino/sticky-pull-request-comment@52423e01640425a022ef5fd42c6fb5f633a02728 # v2 with: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} number: ${{ steps.pr_number.outputs.pr_number }} diff --git a/.github/workflows/nf-test.yml b/.github/workflows/nf-test.yml new file mode 100644 index 00000000..d62fd655 --- /dev/null +++ b/.github/workflows/nf-test.yml @@ -0,0 +1,139 @@ +name: Run nf-test +on: + push: + paths-ignore: + - "docs/**" + - "**/meta.yml" + - "**/*.md" + - "**/*.png" + - "**/*.svg" + pull_request: + paths-ignore: + - "docs/**" + - "**/meta.yml" + - "**/*.md" + - "**/*.png" + - "**/*.svg" + release: + types: [published] + workflow_dispatch: + +# Cancel if a newer run is started +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + NFT_VER: "0.9.2" + NFT_WORKDIR: "~" + NXF_ANSI_LOG: false + NXF_SINGULARITY_CACHEDIR: ${{ github.workspace }}/.singularity + NXF_SINGULARITY_LIBRARYDIR: ${{ github.workspace }}/.singularity + +jobs: + nf-test-changes: + name: nf-test-changes + runs-on: # use GitHub runners + - "ubuntu-latest" + outputs: + shard: ${{ steps.set-shards.outputs.shard }} + total_shards: ${{ steps.set-shards.outputs.total_shards }} + steps: + - name: Clean Workspace # Purge the workspace in case it's running on a self-hosted runner + run: | + ls -la ./ + rm -rf ./* || true + rm -rf ./.??* || true + ls -la ./ + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 + with: + fetch-depth: 0 + + - name: get number of shards + id: set-shards + uses: ./.github/actions/get-shards + env: + NFT_VER: ${{ env.NFT_VER }} + with: + max_shards: 7 + + - name: debug + run: | + echo ${{ steps.set-shards.outputs.shard }} + echo ${{ steps.set-shards.outputs.total_shards }} + + nf-test: + name: "${{ matrix.profile }} | ${{ matrix.NXF_VER }} | ${{ matrix.shard }}/${{ needs.nf-test-changes.outputs.total_shards }}" + needs: [nf-test-changes] + if: ${{ needs.nf-test-changes.outputs.total_shards != '0' }} + runs-on: # use GitHub runners + - "ubuntu-latest" + strategy: + fail-fast: false + matrix: + shard: ${{ fromJson(needs.nf-test-changes.outputs.shard) }} + profile: [conda, docker, singularity] + isMain: + - ${{ github.base_ref == 'master' || github.base_ref == 'main' }} + # Exclude conda and singularity on dev + exclude: + - isMain: false + profile: "conda" + - isMain: false + profile: "singularity" + NXF_VER: + - "24.04.2" + - "latest-everything" + env: + NXF_ANSI_LOG: false + TOTAL_SHARDS: ${{ needs.nf-test-changes.outputs.total_shards }} + + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 + with: + fetch-depth: 0 + + - name: Run nf-test + uses: ./.github/actions/nf-test + env: + NFT_DIFF: ${{ env.NFT_DIFF }} + NFT_DIFF_ARGS: ${{ env.NFT_DIFF_ARGS }} + NFT_WORKDIR: ${{ env.NFT_WORKDIR }} + with: + profile: ${{ matrix.profile }} + shard: ${{ matrix.shard }} + total_shards: ${{ env.TOTAL_SHARDS }} + confirm-pass: + needs: [nf-test] + if: always() + runs-on: # use GitHub runners + - "ubuntu-latest" + steps: + - name: One or more tests failed + if: ${{ contains(needs.*.result, 'failure') }} + run: exit 1 + + - name: One or more tests cancelled + if: ${{ contains(needs.*.result, 'cancelled') }} + run: exit 1 + + - name: All tests ok + if: ${{ contains(needs.*.result, 'success') }} + run: exit 0 + + - name: debug-print + if: always() + run: | + echo "::group::DEBUG: `needs` Contents" + echo "DEBUG: toJSON(needs) = ${{ toJSON(needs) }}" + echo "DEBUG: toJSON(needs.*.result) = ${{ toJSON(needs.*.result) }}" + echo "::endgroup::" + + - name: Clean Workspace # Purge the workspace in case it's running on a self-hosted runner + if: always() + run: | + ls -la ./ + rm -rf ./* || true + rm -rf ./.??* || true + ls -la ./ diff --git a/.github/workflows/template_version_comment.yml b/.github/workflows/template-version-comment.yml similarity index 95% rename from .github/workflows/template_version_comment.yml rename to .github/workflows/template-version-comment.yml index 537529bc..beb5c77f 100644 --- a/.github/workflows/template_version_comment.yml +++ b/.github/workflows/template-version-comment.yml @@ -14,7 +14,7 @@ jobs: ref: ${{ github.event.pull_request.head.sha }} - name: Read template version from .nf-core.yml - uses: nichmor/minimal-read-yaml@v0.0.2 + uses: nichmor/minimal-read-yaml@1f7205277e25e156e1f63815781db80a6d490b8f # v0.0.2 id: read_yml with: config: ${{ github.workspace }}/.nf-core.yml diff --git a/.nf-core.yml b/.nf-core.yml index e14c8c2d..ab8077a6 100644 --- a/.nf-core.yml +++ b/.nf-core.yml @@ -30,7 +30,7 @@ lint: - manifest.homePage - config_defaults: - params.vector_db -nf_core_version: 3.2.1 +nf_core_version: 3.3.1 repository_type: pipeline template: author: "@priyanka-surana" diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 1dec8650..9d0b248d 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -4,10 +4,24 @@ repos: hooks: - id: prettier additional_dependencies: - - prettier@3.2.5 - - - repo: https://github.com/editorconfig-checker/editorconfig-checker.python - rev: "3.1.2" + - prettier@3.5.0 + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v5.0.0 hooks: - - id: editorconfig-checker - alias: ec + - id: trailing-whitespace + args: [--markdown-linebreak-ext=md] + exclude: | + (?x)^( + .*ro-crate-metadata.json$| + modules/nf-core/.*| + subworkflows/nf-core/.*| + .*\.snap$ + )$ + - id: end-of-file-fixer + exclude: | + (?x)^( + .*ro-crate-metadata.json$| + modules/nf-core/.*| + subworkflows/nf-core/.*| + .*\.snap$ + )$ diff --git a/.prettierrc.yml b/.prettierrc.yml index c81f9a76..07dbd8bb 100644 --- a/.prettierrc.yml +++ b/.prettierrc.yml @@ -1 +1,6 @@ printWidth: 120 +tabWidth: 4 +overrides: + - files: "*.{md,yml,yaml,html,css,scss,js,cff}" + options: + tabWidth: 2 diff --git a/README.md b/README.md index 9995ecd4..f239327f 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,8 @@ [![GitHub Actions Linting Status](https://github.com/sanger-tol/readmapping/actions/workflows/linting.yml/badge.svg)](https://github.com/sanger-tol/readmapping/actions/workflows/linting.yml)[![Cite with Zenodo](http://img.shields.io/badge/DOI-10.5281/zenodo.XXXXXXX-1073c8?labelColor=000000)](https://doi.org/10.5281/zenodo.XXXXXXX) [![nf-test](https://img.shields.io/badge/unit_tests-nf--test-337ab7.svg)](https://www.nf-test.com) -[![Nextflow](https://img.shields.io/badge/nextflow%20DSL2-%E2%89%A524.04.2-23aa62.svg)](https://www.nextflow.io/) +[![Nextflow](https://img.shields.io/badge/version-%E2%89%A524.04.2-green?style=flat&logo=nextflow&logoColor=white&color=%230DC09D&link=https%3A%2F%2Fnextflow.io)](https://www.nextflow.io/) +[![nf-core template version](https://img.shields.io/badge/nf--core_template-3.3.1-green?style=flat&logo=nfcore&logoColor=white&color=%2324B064&link=https%3A%2F%2Fnf-co.re)](https://github.com/nf-core/tools/releases/tag/3.3.1) [![run with conda](http://img.shields.io/badge/run%20with-conda-3EB049?labelColor=000000&logo=anaconda)](https://docs.conda.io/en/latest/) [![run with docker](https://img.shields.io/badge/run%20with-docker-0db7ed?labelColor=000000&logo=docker)](https://www.docker.com/) [![run with singularity](https://img.shields.io/badge/run%20with-singularity-1d355c.svg?labelColor=000000)](https://sylabs.io/docs/) @@ -21,7 +22,7 @@ --> + workflows use the "tube map" design for that. See https://nf-co.re/docs/guidelines/graphic_design/workflow_diagrams#examples for examples. --> ## Usage diff --git a/conf/base.config b/conf/base.config index 6b2e835f..5bc52267 100644 --- a/conf/base.config +++ b/conf/base.config @@ -15,7 +15,7 @@ process { memory = { 6.GB * task.attempt } time = { 4.h * task.attempt } - errorStrategy = { task.exitStatus in ((130..145) + 104) ? 'retry' : 'finish' } + errorStrategy = { task.exitStatus in ((130..145) + 104 + 175) ? 'retry' : 'finish' } maxRetries = 1 maxErrors = '-1' @@ -59,4 +59,7 @@ process { errorStrategy = 'retry' maxRetries = 2 } + withLabel: process_gpu { + ext.use_gpu = { workflow.profile.contains('gpu') } + } } diff --git a/nextflow.config b/nextflow.config index 9831b30f..ffbb6896 100644 --- a/nextflow.config +++ b/nextflow.config @@ -148,16 +148,25 @@ profiles { ] } } + gpu { + docker.runOptions = '-u $(id -u):$(id -g) --gpus all' + apptainer.runOptions = '--nv' + singularity.runOptions = '--nv' + } test { includeConfig 'conf/test.config' } test_full { includeConfig 'conf/test_full.config' } } -// Load nf-core custom profiles from different Institutions -includeConfig !System.getenv('NXF_OFFLINE') && params.custom_config_base ? "${params.custom_config_base}/nfcore_custom.config" : "/dev/null" +// Load nf-core custom profiles from different institutions + +// If params.custom_config_base is set AND either the NXF_OFFLINE environment variable is not set or params.custom_config_base is a local path, the nfcore_custom.config file from the specified base path is included. +// Load sanger-tol/readmapping custom profiles from different institutions. +includeConfig params.custom_config_base && (!System.getenv('NXF_OFFLINE') || !params.custom_config_base.startsWith('http')) ? "${params.custom_config_base}/nfcore_custom.config" : "/dev/null" + // Load sanger-tol/readmapping custom profiles from different institutions. // TODO nf-core: Optionally, you can add a pipeline-specific nf-core config at https://github.com/nf-core/configs -// includeConfig !System.getenv('NXF_OFFLINE') && params.custom_config_base ? "${params.custom_config_base}/pipeline/readmapping.config" : "/dev/null" +// includeConfig params.custom_config_base && (!System.getenv('NXF_OFFLINE') || !params.custom_config_base.startsWith('http')) ? "${params.custom_config_base}/pipeline/readmapping.config" : "/dev/null" // Set default registry for Apptainer, Docker, Podman, Charliecloud and Singularity independent of -profile // Will not be used unless Apptainer / Docker / Podman / Charliecloud / Singularity are enabled @@ -236,7 +245,7 @@ manifest { // Nextflow plugins plugins { - id 'nf-schema@2.2.0' // Validation of pipeline parameters and creation of an input channel from a sample sheet + id 'nf-schema@2.3.0' // Validation of pipeline parameters and creation of an input channel from a sample sheet } validation { diff --git a/nf-test.config b/nf-test.config new file mode 100644 index 00000000..889df760 --- /dev/null +++ b/nf-test.config @@ -0,0 +1,24 @@ +config { + // location for all nf-test tests + testsDir "." + + // nf-test directory including temporary files for each test + workDir System.getenv("NFT_WORKDIR") ?: ".nf-test" + + // location of an optional nextflow.config file specific for executing tests + configFile "tests/nextflow.config" + + // ignore tests coming from the nf-core/modules repo + ignore 'modules/nf-core/**/*', 'subworkflows/nf-core/**/*' + + // run all test with defined profile(s) from the main nextflow.config + profile "test" + + // list of filenames or patterns that should be trigger a full test run + triggers 'nextflow.config', 'nf-test.config', 'conf/test.config', 'tests/nextflow.config', 'tests/.nftignore' + + // load the necessary plugins + plugins { + load "nft-utils@0.0.3" + } +} diff --git a/ro-crate-metadata.json b/ro-crate-metadata.json index b2357b85..e883224f 100644 --- a/ro-crate-metadata.json +++ b/ro-crate-metadata.json @@ -22,8 +22,8 @@ "@id": "./", "@type": "Dataset", "creativeWorkStatus": "InProgress", - "datePublished": "2025-05-19T08:29:53+00:00", - "description": "# sanger-tol/readmapping\n\n[![GitHub Actions CI Status](https://github.com/sanger-tol/readmapping/actions/workflows/ci.yml/badge.svg)](https://github.com/sanger-tol/readmapping/actions/workflows/ci.yml)\n[![GitHub Actions Linting Status](https://github.com/sanger-tol/readmapping/actions/workflows/linting.yml/badge.svg)](https://github.com/sanger-tol/readmapping/actions/workflows/linting.yml)[![Cite with Zenodo](http://img.shields.io/badge/DOI-10.5281/zenodo.XXXXXXX-1073c8?labelColor=000000)](https://doi.org/10.5281/zenodo.XXXXXXX)\n[![nf-test](https://img.shields.io/badge/unit_tests-nf--test-337ab7.svg)](https://www.nf-test.com)\n\n[![Nextflow](https://img.shields.io/badge/nextflow%20DSL2-%E2%89%A524.04.2-23aa62.svg)](https://www.nextflow.io/)\n[![run with conda](http://img.shields.io/badge/run%20with-conda-3EB049?labelColor=000000&logo=anaconda)](https://docs.conda.io/en/latest/)\n[![run with docker](https://img.shields.io/badge/run%20with-docker-0db7ed?labelColor=000000&logo=docker)](https://www.docker.com/)\n[![run with singularity](https://img.shields.io/badge/run%20with-singularity-1d355c.svg?labelColor=000000)](https://sylabs.io/docs/)\n[![Launch on Seqera Platform](https://img.shields.io/badge/Launch%20%F0%9F%9A%80-Seqera%20Platform-%234256e7)](https://cloud.seqera.io/launch?pipeline=https://github.com/sanger-tol/readmapping)\n\n## Introduction\n\n**sanger-tol/readmapping** is a bioinformatics pipeline that ...\n\n\n\n\n\n\n## Usage\n\n> [!NOTE]\n> If you are new to Nextflow and nf-core, please refer to [this page](https://nf-co.re/docs/usage/installation) on how to set-up Nextflow. Make sure to [test your setup](https://nf-co.re/docs/usage/introduction#how-to-run-a-pipeline) with `-profile test` before running the workflow on actual data.\n\n\n\nNow, you can run the pipeline using:\n\n\n\n```bash\nnextflow run sanger-tol/readmapping \\\n -profile \\\n --input samplesheet.csv \\\n --outdir \n```\n\n> [!WARNING]\n> Please provide pipeline parameters via the CLI or Nextflow `-params-file` option. Custom config files including those provided by the `-c` Nextflow option can be used to provide any configuration _**except for parameters**_; see [docs](https://nf-co.re/docs/usage/getting_started/configuration#custom-configuration-files).\n\n## Credits\n\nsanger-tol/readmapping was originally written by @priyanka-surana.\n\nWe thank the following people for their extensive assistance in the development of this pipeline:\n\n\n\n## Contributions and Support\n\nIf you would like to contribute to this pipeline, please see the [contributing guidelines](.github/CONTRIBUTING.md).\n\n## Citations\n\n\n\n\n\n\nAn extensive list of references for the tools used by the pipeline can be found in the [`CITATIONS.md`](CITATIONS.md) file.\n\nThis pipeline uses code and infrastructure developed and maintained by the [nf-core](https://nf-co.re) community, reused here under the [MIT license](https://github.com/nf-core/tools/blob/main/LICENSE).\n\n> **The nf-core framework for community-curated bioinformatics pipelines.**\n>\n> Philip Ewels, Alexander Peltzer, Sven Fillinger, Harshil Patel, Johannes Alneberg, Andreas Wilm, Maxime Ulysse Garcia, Paolo Di Tommaso & Sven Nahnsen.\n>\n> _Nat Biotechnol._ 2020 Feb 13. doi: [10.1038/s41587-020-0439-x](https://dx.doi.org/10.1038/s41587-020-0439-x).\n", + "datePublished": "2025-06-20T09:37:29+00:00", + "description": "# sanger-tol/readmapping\n\n[![GitHub Actions CI Status](https://github.com/sanger-tol/readmapping/actions/workflows/ci.yml/badge.svg)](https://github.com/sanger-tol/readmapping/actions/workflows/ci.yml)\n[![GitHub Actions Linting Status](https://github.com/sanger-tol/readmapping/actions/workflows/linting.yml/badge.svg)](https://github.com/sanger-tol/readmapping/actions/workflows/linting.yml)[![Cite with Zenodo](http://img.shields.io/badge/DOI-10.5281/zenodo.XXXXXXX-1073c8?labelColor=000000)](https://doi.org/10.5281/zenodo.XXXXXXX)\n[![nf-test](https://img.shields.io/badge/unit_tests-nf--test-337ab7.svg)](https://www.nf-test.com)\n\n[![Nextflow](https://img.shields.io/badge/version-%E2%89%A524.04.2-green?style=flat&logo=nextflow&logoColor=white&color=%230DC09D&link=https%3A%2F%2Fnextflow.io)](https://www.nextflow.io/)\n[![nf-core template version](https://img.shields.io/badge/nf--core_template-3.3.1-green?style=flat&logo=nfcore&logoColor=white&color=%2324B064&link=https%3A%2F%2Fnf-co.re)](https://github.com/nf-core/tools/releases/tag/3.3.1)\n[![run with conda](http://img.shields.io/badge/run%20with-conda-3EB049?labelColor=000000&logo=anaconda)](https://docs.conda.io/en/latest/)\n[![run with docker](https://img.shields.io/badge/run%20with-docker-0db7ed?labelColor=000000&logo=docker)](https://www.docker.com/)\n[![run with singularity](https://img.shields.io/badge/run%20with-singularity-1d355c.svg?labelColor=000000)](https://sylabs.io/docs/)\n[![Launch on Seqera Platform](https://img.shields.io/badge/Launch%20%F0%9F%9A%80-Seqera%20Platform-%234256e7)](https://cloud.seqera.io/launch?pipeline=https://github.com/sanger-tol/readmapping)\n\n## Introduction\n\n**sanger-tol/readmapping** is a bioinformatics pipeline that ...\n\n\n\n\n\n\n## Usage\n\n> [!NOTE]\n> If you are new to Nextflow and nf-core, please refer to [this page](https://nf-co.re/docs/usage/installation) on how to set-up Nextflow. Make sure to [test your setup](https://nf-co.re/docs/usage/introduction#how-to-run-a-pipeline) with `-profile test` before running the workflow on actual data.\n\n\n\nNow, you can run the pipeline using:\n\n\n\n```bash\nnextflow run sanger-tol/readmapping \\\n -profile \\\n --input samplesheet.csv \\\n --outdir \n```\n\n> [!WARNING]\n> Please provide pipeline parameters via the CLI or Nextflow `-params-file` option. Custom config files including those provided by the `-c` Nextflow option can be used to provide any configuration _**except for parameters**_; see [docs](https://nf-co.re/docs/usage/getting_started/configuration#custom-configuration-files).\n\n## Credits\n\nsanger-tol/readmapping was originally written by @priyanka-surana.\n\nWe thank the following people for their extensive assistance in the development of this pipeline:\n\n\n\n## Contributions and Support\n\nIf you would like to contribute to this pipeline, please see the [contributing guidelines](.github/CONTRIBUTING.md).\n\n## Citations\n\n\n\n\n\n\nAn extensive list of references for the tools used by the pipeline can be found in the [`CITATIONS.md`](CITATIONS.md) file.\n\nThis pipeline uses code and infrastructure developed and maintained by the [nf-core](https://nf-co.re) community, reused here under the [MIT license](https://github.com/nf-core/tools/blob/main/LICENSE).\n\n> **The nf-core framework for community-curated bioinformatics pipelines.**\n>\n> Philip Ewels, Alexander Peltzer, Sven Fillinger, Harshil Patel, Johannes Alneberg, Andreas Wilm, Maxime Ulysse Garcia, Paolo Di Tommaso & Sven Nahnsen.\n>\n> _Nat Biotechnol._ 2020 Feb 13. doi: [10.1038/s41587-020-0439-x](https://dx.doi.org/10.1038/s41587-020-0439-x).\n", "hasPart": [ { "@id": "main.nf" @@ -87,7 +87,7 @@ }, "mentions": [ { - "@id": "#1aeb1332-8ab7-4376-9334-68d4f4fd2c07" + "@id": "#e183c1fa-615e-4791-b30b-ed1fbdc37ab3" } ], "name": "sanger-tol/readmapping" @@ -112,17 +112,17 @@ "@type": ["File", "SoftwareSourceCode", "ComputationalWorkflow"], "creator": [ { - "@id": "https://orcid.org/0000-0002-7860-3560" + "@id": "https://orcid.org/0000-0001-8687-5905" }, { - "@id": "#ps22@sanger.ac.uk" + "@id": "https://orcid.org/0000-0002-7860-3560" }, { - "@id": "https://orcid.org/0000-0001-8687-5905" + "@id": "#ps22@sanger.ac.uk" } ], "dateCreated": "", - "dateModified": "2025-05-19T08:29:53Z", + "dateModified": "2025-06-20T09:37:29Z", "dct:conformsTo": "https://bioschemas.org/profiles/ComputationalWorkflow/1.0-RELEASE/", "keywords": ["nf-core", "nextflow"], "license": ["MIT"], @@ -149,11 +149,11 @@ "version": "!>=24.04.2" }, { - "@id": "#1aeb1332-8ab7-4376-9334-68d4f4fd2c07", + "@id": "#e183c1fa-615e-4791-b30b-ed1fbdc37ab3", "@type": "TestSuite", "instance": [ { - "@id": "#cd9b93f5-0cd4-4505-ab2f-55c996629114" + "@id": "#69337566-4611-455a-8c99-55d90c0264d7" } ], "mainEntity": { @@ -162,10 +162,10 @@ "name": "Test suite for sanger-tol/readmapping" }, { - "@id": "#cd9b93f5-0cd4-4505-ab2f-55c996629114", + "@id": "#69337566-4611-455a-8c99-55d90c0264d7", "@type": "TestInstance", "name": "GitHub Actions workflow for testing sanger-tol/readmapping", - "resource": "repos/sanger-tol/readmapping/actions/workflows/ci.yml", + "resource": "repos/sanger-tol/readmapping/actions/workflows/nf-test.yml", "runsOn": { "@id": "https://w3id.org/ro/terms/test#GithubService" }, @@ -270,6 +270,12 @@ "name": "nf-core", "url": "https://nf-co.re/" }, + { + "@id": "https://orcid.org/0000-0001-8687-5905", + "@type": "Person", + "email": "tc25@sanger.ac.uk", + "name": "Tyler Chafin" + }, { "@id": "https://orcid.org/0000-0002-7860-3560", "@type": "Person", @@ -281,12 +287,6 @@ "@type": "Person", "email": "ps22@sanger.ac.uk", "name": "Priyanka Surana" - }, - { - "@id": "https://orcid.org/0000-0001-8687-5905", - "@type": "Person", - "email": "tc25@sanger.ac.uk", - "name": "Tyler Chafin" } ] } diff --git a/subworkflows/local/utils_nfcore_readmapping_pipeline/main.nf b/subworkflows/local/utils_nfcore_readmapping_pipeline/main.nf index d818f6a6..6ceb5223 100644 --- a/subworkflows/local/utils_nfcore_readmapping_pipeline/main.nf +++ b/subworkflows/local/utils_nfcore_readmapping_pipeline/main.nf @@ -219,4 +219,3 @@ def methodsDescriptionText(mqc_methods_yaml) { return description_html.toString() } - diff --git a/tests/.nftignore b/tests/.nftignore new file mode 100644 index 00000000..73eb92f7 --- /dev/null +++ b/tests/.nftignore @@ -0,0 +1,2 @@ +.DS_Store +pipeline_info/*.{html,json,txt,yml} diff --git a/tests/default.nf.test b/tests/default.nf.test new file mode 100644 index 00000000..1627725d --- /dev/null +++ b/tests/default.nf.test @@ -0,0 +1,35 @@ +nextflow_pipeline { + + name "Test pipeline" + script "../main.nf" + tag "pipeline" + + test("-profile test") { + + when { + params { + outdir = "$outputDir" + } + } + + then { + // stable_name: All files + folders in ${params.outdir}/ with a stable name + def stable_name = getAllFilesFromDir(params.outdir, relative: true, includeDir: true, ignore: ['pipeline_info/*.{html,json,txt}']) + // stable_path: All files in ${params.outdir}/ with stable content + def stable_path = getAllFilesFromDir(params.outdir, ignoreFile: 'tests/.nftignore') + assertAll( + { assert workflow.success}, + { assert snapshot( + // Number of successful tasks + workflow.trace.succeeded().size(), + // pipeline versions.yml file for multiqc from which Nextflow version is removed because we test pipelines on multiple Nextflow versions + removeNextflowVersion("$outputDir/pipeline_info/nf_core_readmapping_software_mqc_versions.yml"), + // All stable path name, with a relative path + stable_name, + // All files with stable contents + stable_path + ).match() } + ) + } + } +} diff --git a/tests/nextflow.config b/tests/nextflow.config new file mode 100644 index 00000000..38ac7256 --- /dev/null +++ b/tests/nextflow.config @@ -0,0 +1,12 @@ +/* +======================================================================================== + Nextflow config file for running nf-test tests +======================================================================================== +*/ + +// TODO nf-core: Specify any additional parameters here +// Or any resources requirements +params.modules_testdata_base_path = 'https://raw.githubusercontent.com/nf-core/test-datasets/modules/data/' +params.pipelines_testdata_base_path = 'https://raw.githubusercontent.com/nf-core/test-datasets/refs/heads/readmapping' + +aws.client.anonymous = true // fixes S3 access issues on self-hosted runners From 9ea1b36e824cbdbb30175669bfe94162e19c4917 Mon Sep 17 00:00:00 2001 From: sainsachiko Date: Mon, 23 Jun 2025 10:56:15 +0100 Subject: [PATCH 2/9] update nf-test --- tests/default.nf.test | 12 ++++++++---- tests/nextflow.config | 2 +- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/tests/default.nf.test b/tests/default.nf.test index 1627725d..4d7a43e4 100644 --- a/tests/default.nf.test +++ b/tests/default.nf.test @@ -1,6 +1,6 @@ nextflow_pipeline { - name "Test pipeline" + name "Test readmapping pipeline with bwamem2 as short aligner" script "../main.nf" tag "pipeline" @@ -8,22 +8,26 @@ nextflow_pipeline { when { params { + input = "${projectDir}/assets/samplesheet_s3.csv" + short_aligner = "minimap2" + fasta = "https://tolit.cog.sanger.ac.uk/test-data/Meles_meles/assembly/release/mMelMel3.1_paternal_haplotype/GCA_922984935.2.subset.fasta.gz" + header = "https://tolit.cog.sanger.ac.uk/test-data/Meles_meles/assembly/release/mMelMel3.1_paternal_haplotype/GCA_922984935.2.subset.header.sam" outdir = "$outputDir" } } then { // stable_name: All files + folders in ${params.outdir}/ with a stable name - def stable_name = getAllFilesFromDir(params.outdir, relative: true, includeDir: true, ignore: ['pipeline_info/*.{html,json,txt}']) + def stable_name = getAllFilesFromDir("${params.outdir}", relative: true, includeDir: true, ignore: ['pipeline_info/*.{html,json,txt}']) // stable_path: All files in ${params.outdir}/ with stable content - def stable_path = getAllFilesFromDir(params.outdir, ignoreFile: 'tests/.nftignore') + def stable_path = getAllFilesFromDir("${params.outdir}", ignoreFile: 'tests/.nftignore') assertAll( { assert workflow.success}, { assert snapshot( // Number of successful tasks workflow.trace.succeeded().size(), // pipeline versions.yml file for multiqc from which Nextflow version is removed because we test pipelines on multiple Nextflow versions - removeNextflowVersion("$outputDir/pipeline_info/nf_core_readmapping_software_mqc_versions.yml"), + removeNextflowVersion("$outputDir/pipeline_info/readmapping_software_versions.yml"), // All stable path name, with a relative path stable_name, // All files with stable contents diff --git a/tests/nextflow.config b/tests/nextflow.config index 38ac7256..5010c7ae 100644 --- a/tests/nextflow.config +++ b/tests/nextflow.config @@ -7,6 +7,6 @@ // TODO nf-core: Specify any additional parameters here // Or any resources requirements params.modules_testdata_base_path = 'https://raw.githubusercontent.com/nf-core/test-datasets/modules/data/' -params.pipelines_testdata_base_path = 'https://raw.githubusercontent.com/nf-core/test-datasets/refs/heads/readmapping' +params.pipelines_testdata_base_path = 'https://tolit.cog.sanger.ac.uk/test-data/' aws.client.anonymous = true // fixes S3 access issues on self-hosted runners From 505188035f721f49bc5ad07040121bfc53cac9dc Mon Sep 17 00:00:00 2001 From: sainsachiko Date: Mon, 23 Jun 2025 11:05:35 +0100 Subject: [PATCH 3/9] Fix nf-core lint and prettier --- bin/pacbio_filter.sh | 1 - ro-crate-metadata.json | 2 +- subworkflows/local/prepare_genome.nf | 1 - 3 files changed, 1 insertion(+), 3 deletions(-) diff --git a/bin/pacbio_filter.sh b/bin/pacbio_filter.sh index 61d6682e..73d7caa7 100755 --- a/bin/pacbio_filter.sh +++ b/bin/pacbio_filter.sh @@ -4,4 +4,3 @@ input=$1 output=$2 grep -v 'MG551957' $input | awk -v OFS='\t' '{if (($2 ~ /NGB00972/ && $3 >= 97 && $4 >= 44) || ($2 ~ /NGB00973/ && $3 >= 97 && $4 >= 34) || ($2 ~ /^bc/ && $3 >= 99 && $4 >= 16)) print $1}' | sort -u > $output - diff --git a/ro-crate-metadata.json b/ro-crate-metadata.json index e3fb37e5..4bcf2af8 100644 --- a/ro-crate-metadata.json +++ b/ro-crate-metadata.json @@ -23,7 +23,7 @@ "@type": "Dataset", "creativeWorkStatus": "InProgress", "datePublished": "2025-05-19T08:55:17+00:00", - "description": "# sanger-tol/readmapping\n\n![sanger-tol/readmapping](docs/images/sanger-tol-readmapping_logo.png)\n\n[![GitHub Actions CI Status](https://github.com/sanger-tol/readmapping/actions/workflows/ci.yml/badge.svg)](https://github.com/sanger-tol/readmapping/actions/workflows/ci.yml)\n[![GitHub Actions Linting Status](https://github.com/sanger-tol/readmapping/actions/workflows/linting.yml/badge.svg)](https://github.com/sanger-tol/readmapping/actions/workflows/linting.yml)\n\n[![Cite with Zenodo](http://img.shields.io/badge/DOI-10.5281/zenodo.6563577-1073c8?labelColor=000000)](https://doi.org/10.5281/zenodo.6563577)\n\n[![Nextflow](https://img.shields.io/badge/nextflow%20DSL2-%E2%89%A524.04.2-23aa62.svg)](https://www.nextflow.io/)\n[![run with conda](http://img.shields.io/badge/run%20with-conda-3EB049?labelColor=000000&logo=anaconda)](https://docs.conda.io/en/latest/)\n[![run with docker](https://img.shields.io/badge/run%20with-docker-0db7ed?labelColor=000000&logo=docker)](https://www.docker.com/)\n[![run with singularity](https://img.shields.io/badge/run%20with-singularity-1d355c.svg?labelColor=000000)](https://sylabs.io/docs/)\n[![Launch on Seqera Platform](https://img.shields.io/badge/Launch%20%F0%9F%9A%80-Seqera%20Platform-%234256e7)](https://cloud.seqera.io/launch?pipeline=https://github.com/sanger-tol/readmapping)\n\n## Introduction\n\n**sanger-tol/readmapping** is a bioinformatics best-practice analysis pipeline for mapping reads generated using Illumina, HiC, PacBio and Nanopore technologies against a genome assembly.\n\nThe pipeline is built using [Nextflow](https://www.nextflow.io), a workflow tool to run tasks across multiple compute infrastructures in a very portable manner. It uses Docker/Singularity containers making installation trivial and results highly reproducible. The [Nextflow DSL2](https://www.nextflow.io/docs/latest/dsl2.html) implementation of this pipeline uses one container per process which makes it much easier to maintain and update software dependencies. Where possible, these processes have been submitted to and installed from [nf-core/modules](https://github.com/nf-core/modules) in order to make them available to all nf-core pipelines, and to everyone within the Nextflow community!\n\nOn merge to `dev` and `main` branch, automated continuous integration tests run the pipeline on a full-sized dataset on the Wellcome Sanger Institute HPC farm using the Nextflow Tower infrastructure. This ensures that the pipeline runs on full sized datasets, has sensible resource allocation defaults set to run on real-world datasets, and permits the persistent storage of results to benchmark between pipeline releases and other analysis sources.\n\n## Pipeline summary\n\n\n\n## Quick Start\n\n1. Install [`Nextflow`](https://www.nextflow.io/docs/latest/getstarted.html#installation) (`>=22.10.1`)\n\n2. Install any of [`Docker`](https://docs.docker.com/engine/installation/), [`Singularity`](https://www.sylabs.io/guides/3.0/user-guide/) (you can follow [this tutorial](https://singularity-tutorial.github.io/01-installation/)), [`Podman`](https://podman.io/), [`Shifter`](https://nersc.gitlab.io/development/shifter/how-to-use/) or [`Charliecloud`](https://hpc.github.io/charliecloud/) for full pipeline reproducibility _(you can use [`Conda`](https://conda.io/miniconda.html) both to install Nextflow itself and also to manage software within pipelines. Please only use it within pipelines as a last resort; see [docs](https://nf-co.re/usage/configuration#basic-configuration-profiles))_.\n\n3. Download the pipeline and test it on a minimal dataset with a single command:\n\n ```bash\n nextflow run sanger-tol/readmapping -profile test,YOURPROFILE --outdir \n ```\n\n Note that some form of configuration will be needed so that Nextflow knows how to fetch the required software. This is usually done in the form of a config profile (`YOURPROFILE` in the example command above). You can chain multiple config profiles in a comma-separated string.\n\n > - The pipeline comes with config profiles called `docker`, `singularity`, `podman`, `shifter`, `charliecloud` and `conda` which instruct the pipeline to use the named tool for software management. For example, `-profile test,docker`.\n > - Please check [nf-core/configs](https://github.com/nf-core/configs#documentation) to see if a custom config file to run nf-core pipelines already exists for your Institute. If so, you can simply use `-profile ` in your command. This will enable either `docker` or `singularity` and set the appropriate execution settings for your local compute environment.\n > - If you are using `singularity`, please use the [`nf-core download`](https://nf-co.re/tools/#downloading-pipelines-for-offline-use) command to download images first, before running the pipeline. Setting the [`NXF_SINGULARITY_CACHEDIR` or `singularity.cacheDir`](https://www.nextflow.io/docs/latest/singularity.html?#singularity-docker-hub) Nextflow options enables you to store and re-use the images from a central location for future pipeline runs.\n > - If you are using `conda`, it is highly recommended to use the [`NXF_CONDA_CACHEDIR` or `conda.cacheDir`](https://www.nextflow.io/docs/latest/conda.html) settings to store the environments in a central location for future pipeline runs.\n\n4. Start running your own analysis!\n\n ```bash\n nextflow run sanger-tol/readmapping --input samplesheet.csv --fasta genome.fa.gz --outdir -profile \n ```\n\n> [!WARNING]\n> Please provide pipeline parameters via the CLI or Nextflow `-params-file` option. Custom config files including those provided by the `-c` Nextflow option can be used to provide any configuration _**except for parameters**_; see [docs](https://nf-co.re/docs/usage/getting_started/configuration#custom-configuration-files).\n\n## Credits\n\nsanger-tol/readmapping was originally written by [Priyanka Surana](https://github.com/priyanka-surana).\n\nWe thank the following people for their extensive assistance in the development of this pipeline:\n\n- [Matthieu Muffato](https://github.com/muffato) for the text logo\n- [Guoying Qi](https://github.com/gq1) for being able to run tests using Nf-Tower and the Sanger HPC farm\n- [Tyler Chafin](https://github.com/tkchafin) for maintenance, updates, and code reviews\n- [Chau Duong](https://github.com/reichan1998) for updates and code reviews\n- [Sandra Babirye](https://github.com/SandraBabirye) for updates and code reviews\n\n## Contributions and Support\n\nIf you would like to contribute to this pipeline, please see the [contributing guidelines](.github/CONTRIBUTING.md).\n\nFor further information or help, please [create an issue](https://github.com/sanger-tol/readmapping/issues/new/choose) on GitHub if you are not on the Sanger slack channel.\n\n## Citations\n\nIf you use sanger-tol/readmapping for your analysis, please cite it using the following doi: [10.5281/zenodo.6563577](https://doi.org/10.5281/zenodo.6563577)\n\nAn extensive list of references for the tools used by the pipeline can be found in the [`CITATIONS.md`](CITATIONS.md) file.\n\nThis pipeline uses code and infrastructure developed and maintained by the [nf-core](https://nf-co.re) community, reused here under the [MIT license](https://github.com/nf-core/tools/blob/main/LICENSE).\n\n> **The nf-core framework for community-curated bioinformatics pipelines.**\n>\n> Philip Ewels, Alexander Peltzer, Sven Fillinger, Harshil Patel, Johannes Alneberg, Andreas Wilm, Maxime Ulysse Garcia, Paolo Di Tommaso & Sven Nahnsen.\n>\n> _Nat Biotechnol._ 2020 Feb 13. doi: [10.1038/s41587-020-0439-x](https://dx.doi.org/10.1038/s41587-020-0439-x).\n", + "description": "# sanger-tol/readmapping\n\n![sanger-tol/readmapping](docs/images/sanger-tol-readmapping_logo.png)\n\n[![GitHub Actions CI Status](https://github.com/sanger-tol/readmapping/actions/workflows/ci.yml/badge.svg)](https://github.com/sanger-tol/readmapping/actions/workflows/ci.yml)\n[![GitHub Actions Linting Status](https://github.com/sanger-tol/readmapping/actions/workflows/linting.yml/badge.svg)](https://github.com/sanger-tol/readmapping/actions/workflows/linting.yml)\n\n[![Cite with Zenodo](http://img.shields.io/badge/DOI-10.5281/zenodo.6563577-1073c8?labelColor=000000)](https://doi.org/10.5281/zenodo.6563577)\n\n[![Nextflow](https://img.shields.io/badge/version-%E2%89%A524.04.2-green?style=flat&logo=nextflow&logoColor=white&color=%230DC09D&link=https%3A%2F%2Fnextflow.io)](https://www.nextflow.io/)\n[![nf-core template version](https://img.shields.io/badge/nf--core_template-3.3.1-green?style=flat&logo=nfcore&logoColor=white&color=%2324B064&link=https%3A%2F%2Fnf-co.re)](https://github.com/nf-core/tools/releases/tag/3.3.1)\n[![run with conda](http://img.shields.io/badge/run%20with-conda-3EB049?labelColor=000000&logo=anaconda)](https://docs.conda.io/en/latest/)\n[![run with docker](https://img.shields.io/badge/run%20with-docker-0db7ed?labelColor=000000&logo=docker)](https://www.docker.com/)\n[![run with singularity](https://img.shields.io/badge/run%20with-singularity-1d355c.svg?labelColor=000000)](https://sylabs.io/docs/)\n[![Launch on Seqera Platform](https://img.shields.io/badge/Launch%20%F0%9F%9A%80-Seqera%20Platform-%234256e7)](https://cloud.seqera.io/launch?pipeline=https://github.com/sanger-tol/readmapping)\n\n## Introduction\n\n**sanger-tol/readmapping** is a bioinformatics best-practice analysis pipeline for mapping reads generated using Illumina, HiC, PacBio and Nanopore technologies against a genome assembly.\n\nThe pipeline is built using [Nextflow](https://www.nextflow.io), a workflow tool to run tasks across multiple compute infrastructures in a very portable manner. It uses Docker/Singularity containers making installation trivial and results highly reproducible. The [Nextflow DSL2](https://www.nextflow.io/docs/latest/dsl2.html) implementation of this pipeline uses one container per process which makes it much easier to maintain and update software dependencies. Where possible, these processes have been submitted to and installed from [nf-core/modules](https://github.com/nf-core/modules) in order to make them available to all nf-core pipelines, and to everyone within the Nextflow community!\n\nOn merge to `dev` and `main` branch, automated continuous integration tests run the pipeline on a full-sized dataset on the Wellcome Sanger Institute HPC farm using the Nextflow Tower infrastructure. This ensures that the pipeline runs on full sized datasets, has sensible resource allocation defaults set to run on real-world datasets, and permits the persistent storage of results to benchmark between pipeline releases and other analysis sources.\n\n## Pipeline summary\n\n\n\n## Quick Start\n\n1. Install [`Nextflow`](https://www.nextflow.io/docs/latest/getstarted.html#installation) (`>=22.10.1`)\n\n2. Install any of [`Docker`](https://docs.docker.com/engine/installation/), [`Singularity`](https://www.sylabs.io/guides/3.0/user-guide/) (you can follow [this tutorial](https://singularity-tutorial.github.io/01-installation/)), [`Podman`](https://podman.io/), [`Shifter`](https://nersc.gitlab.io/development/shifter/how-to-use/) or [`Charliecloud`](https://hpc.github.io/charliecloud/) for full pipeline reproducibility _(you can use [`Conda`](https://conda.io/miniconda.html) both to install Nextflow itself and also to manage software within pipelines. Please only use it within pipelines as a last resort; see [docs](https://nf-co.re/usage/configuration#basic-configuration-profiles))_.\n\n3. Download the pipeline and test it on a minimal dataset with a single command:\n\n ```bash\n nextflow run sanger-tol/readmapping -profile test,YOURPROFILE --outdir \n ```\n\n Note that some form of configuration will be needed so that Nextflow knows how to fetch the required software. This is usually done in the form of a config profile (`YOURPROFILE` in the example command above). You can chain multiple config profiles in a comma-separated string.\n\n > - The pipeline comes with config profiles called `docker`, `singularity`, `podman`, `shifter`, `charliecloud` and `conda` which instruct the pipeline to use the named tool for software management. For example, `-profile test,docker`.\n > - Please check [nf-core/configs](https://github.com/nf-core/configs#documentation) to see if a custom config file to run nf-core pipelines already exists for your Institute. If so, you can simply use `-profile ` in your command. This will enable either `docker` or `singularity` and set the appropriate execution settings for your local compute environment.\n > - If you are using `singularity`, please use the [`nf-core download`](https://nf-co.re/tools/#downloading-pipelines-for-offline-use) command to download images first, before running the pipeline. Setting the [`NXF_SINGULARITY_CACHEDIR` or `singularity.cacheDir`](https://www.nextflow.io/docs/latest/singularity.html?#singularity-docker-hub) Nextflow options enables you to store and re-use the images from a central location for future pipeline runs.\n > - If you are using `conda`, it is highly recommended to use the [`NXF_CONDA_CACHEDIR` or `conda.cacheDir`](https://www.nextflow.io/docs/latest/conda.html) settings to store the environments in a central location for future pipeline runs.\n\n4. Start running your own analysis!\n\n ```bash\n nextflow run sanger-tol/readmapping --input samplesheet.csv --fasta genome.fa.gz --outdir -profile \n ```\n\n> [!WARNING]\n> Please provide pipeline parameters via the CLI or Nextflow `-params-file` option. Custom config files including those provided by the `-c` Nextflow option can be used to provide any configuration _**except for parameters**_; see [docs](https://nf-co.re/docs/usage/getting_started/configuration#custom-configuration-files).\n\n## Credits\n\nsanger-tol/readmapping was originally written by [Priyanka Surana](https://github.com/priyanka-surana).\n\nWe thank the following people for their extensive assistance in the development of this pipeline:\n\n- [Matthieu Muffato](https://github.com/muffato) for the text logo\n- [Guoying Qi](https://github.com/gq1) for being able to run tests using Nf-Tower and the Sanger HPC farm\n- [Tyler Chafin](https://github.com/tkchafin) for maintenance, updates, and code reviews\n- [Chau Duong](https://github.com/reichan1998) for updates and code reviews\n- [Sandra Babirye](https://github.com/SandraBabirye) for updates and code reviews\n\n## Contributions and Support\n\nIf you would like to contribute to this pipeline, please see the [contributing guidelines](.github/CONTRIBUTING.md).\n\nFor further information or help, please [create an issue](https://github.com/sanger-tol/readmapping/issues/new/choose) on GitHub if you are not on the Sanger slack channel.\n\n## Citations\n\nIf you use sanger-tol/readmapping for your analysis, please cite it using the following doi: [10.5281/zenodo.6563577](https://doi.org/10.5281/zenodo.6563577)\n\nAn extensive list of references for the tools used by the pipeline can be found in the [`CITATIONS.md`](CITATIONS.md) file.\n\nThis pipeline uses code and infrastructure developed and maintained by the [nf-core](https://nf-co.re) community, reused here under the [MIT license](https://github.com/nf-core/tools/blob/main/LICENSE).\n\n> **The nf-core framework for community-curated bioinformatics pipelines.**\n>\n> Philip Ewels, Alexander Peltzer, Sven Fillinger, Harshil Patel, Johannes Alneberg, Andreas Wilm, Maxime Ulysse Garcia, Paolo Di Tommaso & Sven Nahnsen.\n>\n> _Nat Biotechnol._ 2020 Feb 13. doi: [10.1038/s41587-020-0439-x](https://dx.doi.org/10.1038/s41587-020-0439-x).\n", "hasPart": [ { "@id": "main.nf" diff --git a/subworkflows/local/prepare_genome.nf b/subworkflows/local/prepare_genome.nf index f7a4754d..a8103da1 100644 --- a/subworkflows/local/prepare_genome.nf +++ b/subworkflows/local/prepare_genome.nf @@ -89,4 +89,3 @@ def checkShortReads(filePath, columnToCheck="datatype") { return containsValues } - From 5caf467e47705c2680721504a09eea8274855cb5 Mon Sep 17 00:00:00 2001 From: sainsachiko Date: Mon, 23 Jun 2025 11:45:09 +0100 Subject: [PATCH 4/9] Add nf-test snapshot --- tests/default.nf.test.snap | 157 +++++++++++++++++++++++++++++++++++++ 1 file changed, 157 insertions(+) create mode 100644 tests/default.nf.test.snap diff --git a/tests/default.nf.test.snap b/tests/default.nf.test.snap new file mode 100644 index 00000000..9cba2989 --- /dev/null +++ b/tests/default.nf.test.snap @@ -0,0 +1,157 @@ +{ + "-profile test": { + "content": [ + 105, + { + "BLAST_BLASTN": { + "blast": "2.15.0+" + }, + "BWAMEM2_INDEX": { + "bwamem2": "2.2.1" + }, + "CONVERT_CRAM": { + "samtools": 1.21 + }, + "CRAM_FILTER": { + "samtools": 1.17, + "staden_io": 1.15 + }, + "CRAM_FILTER_MINIMAP2_FILTER5END_FIXMATE_SORT": { + "samtools": 1.17, + "minimap2": "2.24-r1122", + "staden_io": 1.15 + }, + "CRUMBLE": { + "crumble": "0.9.1" + }, + "GENERATE_CRAM_CSV": { + "samtools": 1.17 + }, + "GUNZIP": { + "gunzip": 1.1 + }, + "MINIMAP2_ALIGN": { + "minimap2": "2.28-r1209", + "samtools": 1.2 + }, + "MINIMAP2_INDEX": { + "minimap2": "2.28-r1209" + }, + "PACBIO_FILTER": { + "GNU Awk": "5.1.0" + }, + "SAMTOOLS_ADDREPLACERG": { + "samtools": 1.2 + }, + "SAMTOOLS_COLLATETOFASTA": { + "samtools": 1.21 + }, + "SAMTOOLS_CONVERT": { + "samtools": 1.21 + }, + "SAMTOOLS_CRAM": { + "samtools": 1.21 + }, + "SAMTOOLS_FILTERTOFASTQ": { + "samtools": 1.21 + }, + "SAMTOOLS_FLAGSTAT": { + "samtools": 1.21 + }, + "SAMTOOLS_IDXSTATS": { + "samtools": 1.21 + }, + "SAMTOOLS_INDEX": { + "samtools": 1.2 + }, + "SAMTOOLS_MERGE": { + "samtools": 1.21 + }, + "SAMTOOLS_REHEADER": { + "samtools": 1.21 + }, + "SAMTOOLS_SORMADUP": { + "samtools": 1.21 + }, + "SAMTOOLS_STATS": { + "samtools": 1.21 + }, + "UNMASK": { + "awk": null + }, + "UNTAR": { + "untar": 1.34 + }, + "Workflow": { + "sanger-tol/readmapping": "v1.4.0dev" + } + }, + [ + "pipeline_info", + "pipeline_info/readmapping_software_versions.yml", + "read_mapping", + "read_mapping/hic", + "read_mapping/hic/GCA_922984935.2.subset.unmasked.hic.mMelMel3.cram", + "read_mapping/hic/GCA_922984935.2.subset.unmasked.hic.mMelMel3.cram.crai", + "read_mapping/hic/GCA_922984935.2.subset.unmasked.hic.mMelMel3.flagstat", + "read_mapping/hic/GCA_922984935.2.subset.unmasked.hic.mMelMel3.idxstats", + "read_mapping/hic/GCA_922984935.2.subset.unmasked.hic.mMelMel3.stats", + "read_mapping/illumina", + "read_mapping/illumina/GCA_922984935.2.subset.unmasked.illumina.mMelMel1.cram", + "read_mapping/illumina/GCA_922984935.2.subset.unmasked.illumina.mMelMel1.cram.crai", + "read_mapping/illumina/GCA_922984935.2.subset.unmasked.illumina.mMelMel1.flagstat", + "read_mapping/illumina/GCA_922984935.2.subset.unmasked.illumina.mMelMel1.idxstats", + "read_mapping/illumina/GCA_922984935.2.subset.unmasked.illumina.mMelMel1.stats", + "read_mapping/illumina/GCA_922984935.2.subset.unmasked.illumina.mMelMel2.cram", + "read_mapping/illumina/GCA_922984935.2.subset.unmasked.illumina.mMelMel2.cram.crai", + "read_mapping/illumina/GCA_922984935.2.subset.unmasked.illumina.mMelMel2.flagstat", + "read_mapping/illumina/GCA_922984935.2.subset.unmasked.illumina.mMelMel2.idxstats", + "read_mapping/illumina/GCA_922984935.2.subset.unmasked.illumina.mMelMel2.stats", + "read_mapping/ont", + "read_mapping/ont/GCA_922984935.2.subset.unmasked.ont.mMelMel3.cram", + "read_mapping/ont/GCA_922984935.2.subset.unmasked.ont.mMelMel3.cram.crai", + "read_mapping/ont/GCA_922984935.2.subset.unmasked.ont.mMelMel3.flagstat", + "read_mapping/ont/GCA_922984935.2.subset.unmasked.ont.mMelMel3.idxstats", + "read_mapping/ont/GCA_922984935.2.subset.unmasked.ont.mMelMel3.stats", + "read_mapping/pacbio", + "read_mapping/pacbio/GCA_922984935.2.subset.unmasked.pacbio.mMelMel3.cram", + "read_mapping/pacbio/GCA_922984935.2.subset.unmasked.pacbio.mMelMel3.cram.crai", + "read_mapping/pacbio/GCA_922984935.2.subset.unmasked.pacbio.mMelMel3.flagstat", + "read_mapping/pacbio/GCA_922984935.2.subset.unmasked.pacbio.mMelMel3.idxstats", + "read_mapping/pacbio/GCA_922984935.2.subset.unmasked.pacbio.mMelMel3.stats" + ], + [ + "GCA_922984935.2.subset.unmasked.hic.mMelMel3.cram:md5,fd89cd70002543b605c6cff9cdfd51e8", + "GCA_922984935.2.subset.unmasked.hic.mMelMel3.cram.crai:md5,8f41d0db4a0a5c557bc5c54de878975f", + "GCA_922984935.2.subset.unmasked.hic.mMelMel3.flagstat:md5,80e3d99b5d0251703cccb9fc2887b78d", + "GCA_922984935.2.subset.unmasked.hic.mMelMel3.idxstats:md5,39b61149a3f039946a794fa3db8567fa", + "GCA_922984935.2.subset.unmasked.hic.mMelMel3.stats:md5,a10354b6cafad79f511731ff25c69572", + "GCA_922984935.2.subset.unmasked.illumina.mMelMel1.cram:md5,f4abc8e8dc265fa06068d50bbccb1a4c", + "GCA_922984935.2.subset.unmasked.illumina.mMelMel1.cram.crai:md5,223283a15112893e24b069de78368840", + "GCA_922984935.2.subset.unmasked.illumina.mMelMel1.flagstat:md5,4dcfe119e0f5040dc0ef52f26266ff31", + "GCA_922984935.2.subset.unmasked.illumina.mMelMel1.idxstats:md5,af38037069b58b5ae7993b9da5deaf4e", + "GCA_922984935.2.subset.unmasked.illumina.mMelMel1.stats:md5,cbf932f9ffb5edec0779b0e6d171599f", + "GCA_922984935.2.subset.unmasked.illumina.mMelMel2.cram:md5,c1321e4585efe193c6fb55d853635081", + "GCA_922984935.2.subset.unmasked.illumina.mMelMel2.cram.crai:md5,fede8f0814f7bfbffb0140a960f63516", + "GCA_922984935.2.subset.unmasked.illumina.mMelMel2.flagstat:md5,4dcfe119e0f5040dc0ef52f26266ff31", + "GCA_922984935.2.subset.unmasked.illumina.mMelMel2.idxstats:md5,af38037069b58b5ae7993b9da5deaf4e", + "GCA_922984935.2.subset.unmasked.illumina.mMelMel2.stats:md5,bd42fa8db910d0f01cbfd641a135fb4c", + "GCA_922984935.2.subset.unmasked.ont.mMelMel3.cram:md5,38cf83c6ede6b610c611de59070cfaa3", + "GCA_922984935.2.subset.unmasked.ont.mMelMel3.cram.crai:md5,2c38657b3e439c2145e8a97036f175f6", + "GCA_922984935.2.subset.unmasked.ont.mMelMel3.flagstat:md5,461ffc1ca2277141e9d965da073b5a0d", + "GCA_922984935.2.subset.unmasked.ont.mMelMel3.idxstats:md5,53e15a4c636c42b3366c8b0e72580528", + "GCA_922984935.2.subset.unmasked.ont.mMelMel3.stats:md5,cde926404a8905dbe7d3074a53459b3e", + "GCA_922984935.2.subset.unmasked.pacbio.mMelMel3.cram:md5,c3f5f0b45d6cb45edf29cfe065f59d2d", + "GCA_922984935.2.subset.unmasked.pacbio.mMelMel3.cram.crai:md5,75bafee2df2f59a5c638d48de11e3e2e", + "GCA_922984935.2.subset.unmasked.pacbio.mMelMel3.flagstat:md5,50df11e497989b79a1dc995a9cac2c85", + "GCA_922984935.2.subset.unmasked.pacbio.mMelMel3.idxstats:md5,4ca3897d0f28cdc6fbeab33d6c327e52", + "GCA_922984935.2.subset.unmasked.pacbio.mMelMel3.stats:md5,8d4b667d90bd61dc523dd2e4fcd88366" + ] + ], + "meta": { + "nf-test": "0.9.2", + "nextflow": "25.04.2" + }, + "timestamp": "2025-06-23T11:44:20.438943" + } +} \ No newline at end of file From 4f6e4fed90669e776afc0e6e91d2e1eb4862d7a9 Mon Sep 17 00:00:00 2001 From: sainsachiko Date: Mon, 23 Jun 2025 14:47:58 +0100 Subject: [PATCH 5/9] Temporarily disable assertion for ouput contents --- tests/default.nf.test | 4 ++-- tests/default.nf.test.snap | 29 +---------------------------- 2 files changed, 3 insertions(+), 30 deletions(-) diff --git a/tests/default.nf.test b/tests/default.nf.test index 4d7a43e4..19f7e3b4 100644 --- a/tests/default.nf.test +++ b/tests/default.nf.test @@ -29,9 +29,9 @@ nextflow_pipeline { // pipeline versions.yml file for multiqc from which Nextflow version is removed because we test pipelines on multiple Nextflow versions removeNextflowVersion("$outputDir/pipeline_info/readmapping_software_versions.yml"), // All stable path name, with a relative path - stable_name, + stable_name // All files with stable contents - stable_path + //stable_path ).match() } ) } diff --git a/tests/default.nf.test.snap b/tests/default.nf.test.snap index 9cba2989..30761b63 100644 --- a/tests/default.nf.test.snap +++ b/tests/default.nf.test.snap @@ -119,39 +119,12 @@ "read_mapping/pacbio/GCA_922984935.2.subset.unmasked.pacbio.mMelMel3.flagstat", "read_mapping/pacbio/GCA_922984935.2.subset.unmasked.pacbio.mMelMel3.idxstats", "read_mapping/pacbio/GCA_922984935.2.subset.unmasked.pacbio.mMelMel3.stats" - ], - [ - "GCA_922984935.2.subset.unmasked.hic.mMelMel3.cram:md5,fd89cd70002543b605c6cff9cdfd51e8", - "GCA_922984935.2.subset.unmasked.hic.mMelMel3.cram.crai:md5,8f41d0db4a0a5c557bc5c54de878975f", - "GCA_922984935.2.subset.unmasked.hic.mMelMel3.flagstat:md5,80e3d99b5d0251703cccb9fc2887b78d", - "GCA_922984935.2.subset.unmasked.hic.mMelMel3.idxstats:md5,39b61149a3f039946a794fa3db8567fa", - "GCA_922984935.2.subset.unmasked.hic.mMelMel3.stats:md5,a10354b6cafad79f511731ff25c69572", - "GCA_922984935.2.subset.unmasked.illumina.mMelMel1.cram:md5,f4abc8e8dc265fa06068d50bbccb1a4c", - "GCA_922984935.2.subset.unmasked.illumina.mMelMel1.cram.crai:md5,223283a15112893e24b069de78368840", - "GCA_922984935.2.subset.unmasked.illumina.mMelMel1.flagstat:md5,4dcfe119e0f5040dc0ef52f26266ff31", - "GCA_922984935.2.subset.unmasked.illumina.mMelMel1.idxstats:md5,af38037069b58b5ae7993b9da5deaf4e", - "GCA_922984935.2.subset.unmasked.illumina.mMelMel1.stats:md5,cbf932f9ffb5edec0779b0e6d171599f", - "GCA_922984935.2.subset.unmasked.illumina.mMelMel2.cram:md5,c1321e4585efe193c6fb55d853635081", - "GCA_922984935.2.subset.unmasked.illumina.mMelMel2.cram.crai:md5,fede8f0814f7bfbffb0140a960f63516", - "GCA_922984935.2.subset.unmasked.illumina.mMelMel2.flagstat:md5,4dcfe119e0f5040dc0ef52f26266ff31", - "GCA_922984935.2.subset.unmasked.illumina.mMelMel2.idxstats:md5,af38037069b58b5ae7993b9da5deaf4e", - "GCA_922984935.2.subset.unmasked.illumina.mMelMel2.stats:md5,bd42fa8db910d0f01cbfd641a135fb4c", - "GCA_922984935.2.subset.unmasked.ont.mMelMel3.cram:md5,38cf83c6ede6b610c611de59070cfaa3", - "GCA_922984935.2.subset.unmasked.ont.mMelMel3.cram.crai:md5,2c38657b3e439c2145e8a97036f175f6", - "GCA_922984935.2.subset.unmasked.ont.mMelMel3.flagstat:md5,461ffc1ca2277141e9d965da073b5a0d", - "GCA_922984935.2.subset.unmasked.ont.mMelMel3.idxstats:md5,53e15a4c636c42b3366c8b0e72580528", - "GCA_922984935.2.subset.unmasked.ont.mMelMel3.stats:md5,cde926404a8905dbe7d3074a53459b3e", - "GCA_922984935.2.subset.unmasked.pacbio.mMelMel3.cram:md5,c3f5f0b45d6cb45edf29cfe065f59d2d", - "GCA_922984935.2.subset.unmasked.pacbio.mMelMel3.cram.crai:md5,75bafee2df2f59a5c638d48de11e3e2e", - "GCA_922984935.2.subset.unmasked.pacbio.mMelMel3.flagstat:md5,50df11e497989b79a1dc995a9cac2c85", - "GCA_922984935.2.subset.unmasked.pacbio.mMelMel3.idxstats:md5,4ca3897d0f28cdc6fbeab33d6c327e52", - "GCA_922984935.2.subset.unmasked.pacbio.mMelMel3.stats:md5,8d4b667d90bd61dc523dd2e4fcd88366" ] ], "meta": { "nf-test": "0.9.2", "nextflow": "25.04.2" }, - "timestamp": "2025-06-23T11:44:20.438943" + "timestamp": "2025-06-23T14:22:16.950642" } } \ No newline at end of file From 23b36967b6bd2b85f4be88c43e7d00d30ffe78a2 Mon Sep 17 00:00:00 2001 From: Matthieu Muffato Date: Wed, 25 Jun 2025 14:48:55 +0100 Subject: [PATCH 6/9] Now that we have nf-test we can add this back --- .github/CONTRIBUTING.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md index 96c45ea7..05572888 100644 --- a/.github/CONTRIBUTING.md +++ b/.github/CONTRIBUTING.md @@ -23,6 +23,12 @@ If you're not used to this workflow with git, you can start with some [docs from ## Tests +You have the option to test your changes locally by running the pipeline. For receiving warnings about process selectors and other `debug` information, it is recommended to use the debug profile. Execute all the tests with the following command: + +```bash +nf-test test --profile debug,test,docker --verbose +``` + When you create a pull request with changes, [GitHub Actions](https://github.com/features/actions) will run automatic tests. Typically, pull-requests are only fully reviewed when these tests are passing, though of course we can help out before then. From a7dce1a1d0b6312da42613b34c53c16b66874524 Mon Sep 17 00:00:00 2001 From: Matthieu Muffato Date: Wed, 25 Jun 2025 16:35:15 +0100 Subject: [PATCH 7/9] Updated the author list --- CITATION.cff | 5 +++++ nextflow.config | 7 +++++++ ro-crate-metadata.json | 12 ++++++++++++ 3 files changed, 24 insertions(+) diff --git a/CITATION.cff b/CITATION.cff index 53ca37c0..7b18e41b 100644 --- a/CITATION.cff +++ b/CITATION.cff @@ -16,6 +16,11 @@ authors: given-names: Chau orcid: https://orcid.org/0009-0001-0649-2291 website: https://github.com/reichan1998 + - affiliation: Wellcome Sanger Institute + family-names: "Ho\xE0ng" + given-names: "H\u1EA1nh" + orcid: https://orcid.org/0009-0008-6911-9903 + website: https://github.com/sainsachiko - affiliation: Wellcome Sanger Institute email: mm49@sanger.ac.uk family-names: Muffato diff --git a/nextflow.config b/nextflow.config index da866c68..c4554b07 100644 --- a/nextflow.config +++ b/nextflow.config @@ -292,6 +292,13 @@ manifest { contribution: ['author'], orcid: 'https://orcid.org/0000-0002-7167-0875' ], + [ + name: 'Hoàng, Hạnh', + affiliation: 'Wellcome Sanger Institute', + github: 'https://github.com/sainsachiko', + contribution: ['maintainer'], + orcid: 'https://orcid.org/0009-0008-6911-9903' + ], [ name: 'Yates, Bethan', affiliation: 'Wellcome Sanger Institute', diff --git a/ro-crate-metadata.json b/ro-crate-metadata.json index 4bcf2af8..1a30f77e 100644 --- a/ro-crate-metadata.json +++ b/ro-crate-metadata.json @@ -169,6 +169,9 @@ "maintainer": [ { "@id": "https://orcid.org/0000-0002-7860-3560" + }, + { + "@id": "https://orcid.org/0009-0008-6911-9903" } ], "name": [ @@ -360,6 +363,7 @@ "@id": "https://orcid.org/0009-0001-0649-2291", "@type": "Person", "affiliation": "Wellcome Sanger Institute", + "email": "120034608+reichan1998@users.noreply.github.com", "name": "Duong, Chau", "url": "https://github.com/reichan1998" }, @@ -383,6 +387,7 @@ "@id": "https://orcid.org/0000-0003-1262-8973", "@type": "Person", "affiliation": "Wellcome Sanger Institute", + "email": "729395+gq1@users.noreply.github.com", "name": "Qi, Guoying", "url": "https://github.com/gq1" }, @@ -394,6 +399,13 @@ "name": "Surana, Priyanka", "url": "https://github.com/priyanka-surana" }, + { + "@id": "https://orcid.org/0009-0008-6911-9903", + "@type": "Person", + "affiliation": "Wellcome Sanger Institute", + "name": "Ho\u00e0ng, H\u1ea1nh", + "url": "https://github.com/sainsachiko" + }, { "@id": "https://orcid.org/0000-0003-1658-1762", "@type": "Person", From 2c67625319f06433ea2b8a374ac51f4a888b632a Mon Sep 17 00:00:00 2001 From: Hanh Hoang <134130358+sainsachiko@users.noreply.github.com> Date: Thu, 3 Jul 2025 11:54:36 +0100 Subject: [PATCH 8/9] Delete abundant params in tests/default.nf.test Co-authored-by: Matthieu Muffato --- tests/default.nf.test | 4 ---- 1 file changed, 4 deletions(-) diff --git a/tests/default.nf.test b/tests/default.nf.test index 19f7e3b4..25459e57 100644 --- a/tests/default.nf.test +++ b/tests/default.nf.test @@ -8,10 +8,6 @@ nextflow_pipeline { when { params { - input = "${projectDir}/assets/samplesheet_s3.csv" - short_aligner = "minimap2" - fasta = "https://tolit.cog.sanger.ac.uk/test-data/Meles_meles/assembly/release/mMelMel3.1_paternal_haplotype/GCA_922984935.2.subset.fasta.gz" - header = "https://tolit.cog.sanger.ac.uk/test-data/Meles_meles/assembly/release/mMelMel3.1_paternal_haplotype/GCA_922984935.2.subset.header.sam" outdir = "$outputDir" } } From 2d5edbd7bab5eecf6c322e10c2d1ed203ec8bec5 Mon Sep 17 00:00:00 2001 From: Matthieu Muffato Date: Fri, 4 Jul 2025 08:45:27 +0100 Subject: [PATCH 9/9] Names should be in the alphabetical order --- nextflow.config | 14 +++++++------- ro-crate-metadata.json | 18 +++++++++--------- 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/nextflow.config b/nextflow.config index c4554b07..caa58c27 100644 --- a/nextflow.config +++ b/nextflow.config @@ -263,6 +263,13 @@ manifest { contribution: ['contributor'], orcid: 'https://orcid.org/0009-0001-0649-2291' ], + [ + name: 'Hoàng, Hạnh', + affiliation: 'Wellcome Sanger Institute', + github: 'https://github.com/sainsachiko', + contribution: ['maintainer'], + orcid: 'https://orcid.org/0009-0008-6911-9903' + ], [ name: 'Muffato, Matthieu', affiliation: 'Wellcome Sanger Institute', @@ -292,13 +299,6 @@ manifest { contribution: ['author'], orcid: 'https://orcid.org/0000-0002-7167-0875' ], - [ - name: 'Hoàng, Hạnh', - affiliation: 'Wellcome Sanger Institute', - github: 'https://github.com/sainsachiko', - contribution: ['maintainer'], - orcid: 'https://orcid.org/0009-0008-6911-9903' - ], [ name: 'Yates, Bethan', affiliation: 'Wellcome Sanger Institute', diff --git a/ro-crate-metadata.json b/ro-crate-metadata.json index 1a30f77e..129cb430 100644 --- a/ro-crate-metadata.json +++ b/ro-crate-metadata.json @@ -168,10 +168,10 @@ ], "maintainer": [ { - "@id": "https://orcid.org/0000-0002-7860-3560" + "@id": "https://orcid.org/0009-0008-6911-9903" }, { - "@id": "https://orcid.org/0009-0008-6911-9903" + "@id": "https://orcid.org/0000-0002-7860-3560" } ], "name": [ @@ -367,6 +367,13 @@ "name": "Duong, Chau", "url": "https://github.com/reichan1998" }, + { + "@id": "https://orcid.org/0009-0008-6911-9903", + "@type": "Person", + "affiliation": "Wellcome Sanger Institute", + "name": "Ho\u00e0ng, H\u1ea1nh", + "url": "https://github.com/sainsachiko" + }, { "@id": "https://orcid.org/0000-0002-7860-3560", "@type": "Person", @@ -399,13 +406,6 @@ "name": "Surana, Priyanka", "url": "https://github.com/priyanka-surana" }, - { - "@id": "https://orcid.org/0009-0008-6911-9903", - "@type": "Person", - "affiliation": "Wellcome Sanger Institute", - "name": "Ho\u00e0ng, H\u1ea1nh", - "url": "https://github.com/sainsachiko" - }, { "@id": "https://orcid.org/0000-0003-1658-1762", "@type": "Person",