Workflow output definition (second preview)

.github/workflows/build.yml

@@ -25,7 +25,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        java_version: [11, 17, 22]
+        java_version: [11, 17, 23]
 
     steps:
       - name: Environment
@@ -37,6 +37,23 @@ jobs:
           fetch-depth: 1
           submodules: true
 
+      - name: Get the commit message
+        id: get_commit_message
+        run: |
+            if [ "${{ github.event_name }}" = "pull_request" ]; then
+                echo "GitHub event=pull_request"
+                COMMIT_SHA=${{ github.event.pull_request.head.sha }}
+                COMMIT_MESSAGE=$(curl -s \
+                  -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \
+                  https://api.github.com/repos/${{ github.repository }}/commits/$COMMIT_SHA | jq -r '.commit.message')
+                echo "Commit message=$COMMIT_MESSAGE" | head -n 1
+                echo "commit_message=$COMMIT_MESSAGE" | head -n 1 >> $GITHUB_OUTPUT
+            else
+               echo "GitHub event=${{ github.event_name }}"
+               echo "Commit message=${{ github.event.head_commit.message }}" | head -n 1
+               echo "commit_message=${{ github.event.head_commit.message }}" | head -n 1 >> $GITHUB_OUTPUT
+            fi 
+
       - name: Get changed files
         id: changed-files
         uses: tj-actions/changed-files@v43
@@ -106,16 +123,17 @@ jobs:
 
     outputs:
       any_changed: ${{ steps.changed-files.outputs.any_changed }}
+      commit_message: ${{ steps.get_commit_message.outputs.commit_message }}
 
   test:
-    if: ${{ !contains(github.event.head_commit.message, '[ci fast]') && needs.build.outputs.any_changed == 'true' }}
+    if: ${{ !contains(needs.build.outputs.commit_message, '[ci fast]') && needs.build.outputs.any_changed == 'true' }}
     needs: build
     runs-on: ubuntu-latest
     timeout-minutes: 90
     strategy:
       fail-fast: false
       matrix:
-        java_version: [11, 21]
+        java_version: [11, 23]
         test_mode: ["test_integration", "test_docs", "test_aws", "test_azure", "test_google", "test_wave"]
     steps:
       - name: Checkout
@@ -143,7 +161,7 @@ jobs:
       - name: Run tests
         run: |
           cat $HOME/.nextflow/scm
-          make assemble install
+          make clean assemble install
           bash test-ci.sh
         env:
           TEST_JDK: ${{ matrix.java_version }}
@@ -175,3 +193,48 @@ jobs:
           path: |
             validation/**
             integration-tests.tar
+
+  test-e2e:
+      if: ${{ contains(needs.build.outputs.commit_message,'[e2e stage]') || contains(needs.build.outputs.commit_message,'[e2e prod]') }}
+      needs: build
+      runs-on: ubuntu-latest
+      timeout-minutes: 10
+      permissions:
+          actions: write  # Allow writing to actions
+          contents: write # Allow writing to repository contents
+      steps:
+          - name: Checkout
+            uses: actions/checkout@v4
+            with:
+                fetch-depth: 1
+                submodules: true
+
+          - name: Setup Java 17
+            uses: actions/setup-java@v4
+            with:
+                java-version: 17
+                distribution: 'temurin'
+                architecture: x64
+                cache: gradle
+
+          - name: Setup env
+            run: |
+                wget -q -O wave https://github.com/seqeralabs/wave-cli/releases/download/v1.4.1/wave-1.4.1-linux-x86_64
+                chmod +x wave
+                mv wave /usr/local/bin/
+                echo "COMMIT_MESSAGE=\"${{ needs.build.outputs.commit_message }}\"" >> $GITHUB_ENV
+
+          - name : Docker Login to Seqera public CR
+            uses : docker/login-action@v3
+            with :
+                registry : "public.cr.seqera.io"
+                username : "public-cr-admin"
+                password : ${{ secrets.SEQERA_PUBLIC_CR_PASSWORD }}
+
+          - name: Launch tests
+            run: |
+                cd test-e2e
+                bash run.sh
+            env:
+                GITHUB_TOKEN: ${{ secrets.AUTOMATION_GITHUB_TOKEN }}
+                GRADLE_OPTS: '-Dorg.gradle.daemon=false'

Makefile

@@ -95,10 +95,7 @@ upload:
 # Create self-contained distribution package
 #
 pack:
-	BUILD_PACK=1 ./gradlew packAll
-
-packCore:
-	BUILD_PACK=1 ./gradlew packCore
+	BUILD_PACK=1 ./gradlew pack
 
 #
 # Upload NF launcher to nextflow.io web site

VERSION

@@ -1 +1 @@
-24.06.0-edge
+24.08.0-edge

build.gradle

@@ -109,8 +109,8 @@ allprojects {
 
         // Documentation required libraries
         groovyDoc 'org.fusesource.jansi:jansi:2.4.0'
-        groovyDoc "org.apache.groovy:groovy-groovydoc:4.0.22"
-        groovyDoc "org.apache.groovy:groovy-ant:4.0.22"
+        groovyDoc "org.apache.groovy:groovy-groovydoc:4.0.23"
+        groovyDoc "org.apache.groovy:groovy-ant:4.0.23"
     }
 
     test {

changelog.txt

@@ -1,7 +1,87 @@
 NEXTFLOW CHANGE-LOG
 ===================
+24.08.0-edge - 4 Sep 2024
+- Add Google Batch warning when for conflicting disk image config (#5279) [96cb57cb]
+- Add support for Google Batch used specified boot images  (#5268) [0aaa6482]
+- Disable AWS spot retry (#5215) [f28fcb25]
+- Disable Google Batch automatic spot retries (#5223) [aad21533]
+- Disable automatic detection of virtual threads (#5270) [b3ba2c2d]
+- Fix  missing .command.env when eval is used and task runs on a cloud env [4a6b54aa]
+- Fix job array syntax for PBS/Torque executor (#5281) [d59f5fae]
+- Fix k8s client status cond is possible null in podState (#5264) [46672415]
+- Fix non-determinist behaviour of join operator with bag array as key (#5189) [e7dc0d69]
+- Fix stage retry on corrupted HTTP downloads  (#5275) [bf0cd326]
+- Support Azure Managed Identities in Fusion configuration logic (#5278) [a0bf8b40]
+- Use public.cr.seqera.io in place of AWS ECR [5a01f277]
+- Wave client logs improvement [5a37e617]
+- Bump amazoncorretto:21-al2023 [59aed581]
+- Bump [email protected] [97c4e08f]
+- Bump [email protected] [24133f2a]
+- Bump [email protected] [29f49ba7]
+- Bump [email protected] [bbc3adca]
+
+24.07.0-edge - 8 Aug 2024
+- Add runtime error for missing channel factory (#5170) [1f9210ab]
+- Apply k8s.cpuLimits to kuberun driver pod (#5160) [4300adf1]
+- Await build completion for all Wave containers [2b8117e9]
+- Deprecate module addParams() and params() (#5200) [82c97f8c]
+- Remove capsule launcher dependencies (#3395) [f15e4246]
+- Fix AWS Cloudwatch access when using custom log group name [30195838]
+- Fix Invalid AWS Fargate CPUs usage error reporting [d9c50e59]
+- Fix Prevent AwS Batch retry the job execution when the container does not exist [4e218f22]
+- Fix Prevent negative cpus values [af2e4ef7]
+- Fix ProcessUnrecoverableException detection [17ec6c96]
+- Fix aws logs group name (#5146) [643b16b0]
+- Fix readAllBytes with http files (#5202) [7e90ce62]
+- Improve Google Batch 5000x error class handling (#5141) [61b2205f]
+- Improve docs [7703f0d0]
+- Improve error message for topic channel feature flag (#5161) [c4d407fb]
+- Improve fusion docs (#5166) [abdd78f1]
+- Improve queue docs (#5154) [bc2fb409]
+- Improve version checking [e7a879de]
+- Link to other docs on the failOnIgnore option (#5122) [572f2116]
+- Make Google Batch auto retry codes configurable (#5148) [e562ce06]
+- More robust parsing of shm-size containerOptions (#5177) [b56802a3]
+- Prevent default output/error logs for job arrays (#5152) [71f6ed0b]
+- Bump amazon sdk to version 1.12.766 [cc6ec314]
+- Bump gradle 8.9 [a437e97f]
+- Bump jgit 6.10.0 [177dc474]
+- Bump [email protected] [46c69d77]
+- Bump [email protected] [26dcb604]
+- Bump [email protected] [c171b601]
+- Bump [email protected] [d7dd4611]
+- Bump [email protected] [c1175157]
+- Bump [email protected] [1eda221a]
+- Bump [email protected] [257bebfd]
+- Bump [email protected] [e8d643c2]
+- Bump pf4j to version 3.12.0 [96117b9a]
+- Bump wave-api to 0.11.1 [96ec4ded]
+
+24.04.4 - 1 Aug 2024
+- Fix parsing of shm-size containerOptions (#5177) [98cf0068]
+- Fix aws logs group name (#5146) [b2ab651c]
+- Update Azure batch deps [72576648]
+- Bump [email protected] [19a7272f]
+- Bump [email protected] [f2bcce99]
+- Bump [email protected] [0ea1f102]
+- Bump [email protected] [b13b5548]
+- Bump wave-api to 0.11.1 [87828afd]
+- Bump amazon sdk to version 1.12.766 [5ce42b79]
+- Bump pf4j to version 3.12.0 [1a8f086a]
+- Bump jgit 6.10.0 [cc2ccd81]
+
+23.10.4 - 31 Jul 2024
+- Add docker launcher [474427a5]
+- Update Azure batch deps [e0c6d77d]
+- Bump [email protected] [54683615]
+- Bump [email protected] [084092f1]
+- Bump [email protected] [de43b17e]
+- Bump amazon sdk to version 1.12.766 [189f58ed]
+- Bump pf4j to version 3.12.0 [8dfa4076]
+- Bump jgit 6.10.0 [4cf6b9f7]
+
 24.04.3 - 9 Jul 2024
-- Add ability to override failOnError setting default via env variable (#5117) [ci fast] [6852429c]
+- Add ability to override failOnError setting default via env variable (#5117) [6852429c]
 - Fix normalization of consecutive slashes in uri path (#5114) [3f366b7e]
 - Fix executions hangs on finalisation exception (#5070) [4c207c23]
 - Bump [email protected] [55ec5ec5]

docker/Dockerfile

@@ -1,4 +1,4 @@
-FROM amazoncorretto:17-al2023
+FROM amazoncorretto:21-al2023
 RUN yum install -y procps-ng shadow-utils which
 
 ENV NXF_HOME=/.nextflow

docker/Makefile

@@ -24,14 +24,10 @@ build-arm: dist/docker/arm64
 	docker buildx build --platform linux/arm64 --output=type=docker --progress=plain --tag nextflow/nextflow:${version} --build-arg TARGETPLATFORM=linux/arm64 .
 
 release: build
-	docker tag nextflow/nextflow:${version} nextflow/nextflow:latest
 	docker push nextflow/nextflow:${version}
-	docker push nextflow/nextflow:latest
 	#
-	docker tag nextflow/nextflow:${version} public.ecr.aws/seqera-labs/nextflow:${version}
-	docker tag nextflow/nextflow:${version} public.ecr.aws/seqera-labs/nextflow:latest
-	docker push public.ecr.aws/seqera-labs/nextflow:${version}
-	docker push public.ecr.aws/seqera-labs/nextflow:latest
+	docker tag nextflow/nextflow:${version} public.cr.seqera.io/nextflow/nextflow:${version}
+	docker push public.cr.seqera.io/nextflow/nextflow:${version}
 
 #Static builds can now be found at:
 #

docs/Dockerfile

@@ -1,6 +1,4 @@
-FROM mambaorg/micromamba:1.3.1
-
-MAINTAINER Ben Sherman <[email protected]>
+FROM mambaorg/micromamba:1.5.8
 
 RUN micromamba install --yes --name base --channel conda-forge \
       make=4.3 \

docs/_templates/layout.html

@@ -21,7 +21,7 @@
     <div class="nav-footer-logo">
         <a href="https://seqera.io/" target="_blank" title="Developed by Seqera Labs">
             Nextflow is developed by:<br>
-            <img src="_static/seqera-logo.svg" alt="Seqera Labs">
+            <img src="{{ pathto('_static/seqera-logo.svg', 1) }}" alt="Seqera Labs">
         </a>
     </div>
 {% endblock %}

docs/amazons3.md

@@ -1,6 +1,6 @@
 (amazons3-page)=
 
-# AWS S3 storage
+# Amazon S3
 
 Nextflow includes support for AWS S3 storage. Files stored in an S3 bucket can be accessed transparently in your pipeline script like any other file in the local file system.
 
@@ -20,7 +20,7 @@ The usual file operations can be applied to a path handle with the above notatio
 println file('s3://my-bucket/data/sequences.fa').text
 ```
 
-See the {ref}`script-file-io` section to learn more about available file operations.
+See {ref}`working-with-files` and the {ref}`stdlib-types-path` reference to learn more about available file operations.
 
 ## Security credentials
 

docs/aws.md

@@ -1,6 +1,6 @@
 (aws-page)=
 
-# AWS Cloud
+# Amazon Web Services
 
 :::{tip}
 This page describes how to manually set up and use Nextflow with AWS Cloud.

docs/azure.md

@@ -1,6 +1,6 @@
 (azure-page)=
 
-# Azure Cloud
+# Azure
 
 :::{versionadded} 21.04.0
 :::

docs/cache-and-resume.md

@@ -16,7 +16,7 @@ The task cache is used in conjunction with the [work directory](#work-directory)
 
 The task hash is computed from the following metadata:
 
-- Session ID (see `workflow.sessionId` in {ref}`metadata-workflow`)
+- Session ID (see `workflow.sessionId` in {ref}`stdlib-constants`)
 - Task name (see `name` in {ref}`trace-report`)
 - Task container image (if applicable)
 - Task {ref}`environment modules <process-module>` (if applicable)