Add new gene unification approach and testing

.github/workflows/ci.yml

@@ -11,6 +11,10 @@ on:
 
 env:
   NXF_ANSI_LOG: false
+  NFT_VER: "0.9.0"
+  NFT_WORKDIR: "~"
+  NFT_DIFF: "pdiff"
+  NFT_DIFF_ARGS: "--line-numbers --expand-tabs=2"
   NXF_SINGULARITY_CACHEDIR: ${{ github.workspace }}/.singularity
   NXF_SINGULARITY_LIBRARYDIR: ${{ github.workspace }}/.singularity
 
@@ -85,3 +89,81 @@ jobs:
       - name: "Run pipeline with test data ${{ matrix.NXF_VER }} | ${{ matrix.test_name }} | ${{ matrix.profile }}"
         run: |
           nextflow run ${GITHUB_WORKSPACE} -profile ${{ matrix.test_name }},${{ matrix.profile }} --outdir ./results
+
+  nf-test:
+    name: "Run nf-test pipeline tests (${{ matrix.test_name }} | ${{ matrix.NXF_VER }} | ${{ matrix.profile }})"
+    if: "${{ github.event_name != 'push' || (github.event_name == 'push' && github.repository == 'nf-core/scdownstream') }}"
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        NXF_VER:
+          - "24.04.2"
+          - "latest-everything"
+        profile:
+          - "conda"
+          - "docker"
+          - "singularity"
+        test_name:
+          - "build"
+          - "extend"
+        isMaster:
+          - ${{ github.base_ref == 'master' }}
+        # Exclude conda and singularity on dev
+        exclude:
+          - isMaster: false
+            profile: "conda"
+          - isMaster: false
+            profile: "singularity"
+
+    steps:
+      - name: Disk space cleanup
+        uses: jlumbroso/free-disk-space@54081f138730dfa15788a46383842cd2f914a1be # v1.3.1
+
+      - name: Check out pipeline code
+        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4
+        with:
+          fetch-depth: 0
+
+      - uses: actions/setup-python@v4
+        with:
+          python-version: "3.11"
+          architecture: "x64"
+
+      - uses: actions/setup-java@8df1039502a15bceb9433410b1a100fbe190c53b # v4
+        with:
+          distribution: "temurin"
+          java-version: "17"
+
+      - name: Install pdiff to see diff between nf-test snapshots
+        run: |
+          python -m pip install --upgrade pip
+          pip install pdiff
+
+      - uses: nf-core/setup-nextflow@v2
+        with:
+          version: "${{ matrix.NXF_VER }}"
+
+      - uses: nf-core/setup-nf-test@v1
+        with:
+          version: ${{ env.NFT_VER }}
+
+      - name: Run nf-test
+        run: |
+          nf-test test \
+            --ci \
+            --junitxml=test.xml \
+            --profile ${{ matrix.profile }} \
+            tests/main_pipeline_${{ matrix.test_name }}.nf.test
+
+      - name: Output log on failure
+        if: failure()
+        run: |
+          sudo apt install bat > /dev/null
+          batcat --decorations=always --color=always ${{ github.workspace }}/.nf-test/tests/*/meta/nextflow.log
+
+      - name: Publish Test Report
+        uses: mikepenz/action-junit-report@v3
+        if: always() # always run even if the previous step fails
+        with:
+          report_paths: test.xml

.gitignore

@@ -7,3 +7,4 @@ testing/
 testing*
 *.pyc
 null/
+.nf-test*

assets/schema_input.json

@@ -36,7 +36,6 @@
             "batch_col": {
                 "type": "string",
                 "pattern": "^[a-zA-Z][a-zA-Z0-9_-]*$",
-                "default": "batch",
                 "errorMessage": "Batch column needs to start with a letter and can contain letters, numbers, underlines and dashes.",
                 "meta": ["batch_col"]
             },
@@ -47,6 +46,13 @@
                 "errorMessage": "Symbol column needs to start with a letter and can contain letters, numbers, underlines and dashes.",
                 "meta": ["symbol_col"]
             },
+            "geneid_col": {
+                "type": "string",
+                "pattern": "^[a-zA-Z][a-zA-Z0-9_-]*$",
+                "default": "index",
+                "errorMessage": "Gene ID column needs to start with a letter and can contain letters, numbers, underlines and dashes.",
+                "meta": ["geneid_col"]
+            },
             "label_col": {
                 "type": "string",
                 "pattern": "^[a-zA-Z][a-zA-Z0-9_-]*$",
@@ -60,31 +66,38 @@
                 "errorMessage": "Unknown label needs to start with a letter and can contain letters, numbers, underlines and dashes.",
                 "meta": ["unknown_label"]
             },
+            "counts_layer": {
+                "type": "string",
+                "pattern": "^[a-zA-Z][a-zA-Z0-9_-]*$",
+                "default": "X",
+                "errorMessage": "Counts layer needs to start with a letter and can contain letters, numbers, underlines and dashes.",
+                "meta": ["counts_layer"]
+            },
             "min_genes": {
                 "type": "integer",
-                "minimum": 1,
-                "default": 1,
+                "minimum": 0,
+                "default": 0,
                 "errorMessage": "Minimum number of genes must be an integer greater than 0.",
                 "meta": ["min_genes"]
             },
             "min_cells": {
                 "type": "integer",
-                "minimum": 1,
-                "default": 1,
+                "minimum": 0,
+                "default": 0,
                 "errorMessage": "Minimum number of cells must be an integer greater than 0.",
                 "meta": ["min_cells"]
             },
             "min_counts_cell": {
                 "type": "integer",
-                "minimum": 1,
-                "default": 1,
+                "minimum": 0,
+                "default": 0,
                 "errorMessage": "Minimum number of counts per cell must be an integer greater than 0.",
                 "meta": ["min_counts_cell"]
             },
             "min_counts_gene": {
                 "type": "integer",
-                "minimum": 1,
-                "default": 1,
+                "minimum": 0,
+                "default": 0,
                 "errorMessage": "Minimum number of counts per gene must be an integer greater than 0.",
                 "meta": ["min_counts_gene"]
             },