.gitlab-ci.yml

# ############################################################################################################################################################
# ######################################################### Gitlab CI/CD pipeline ###########################################################################
# ############################################################################################################################################################

# This is the Gitlab CI/CD pipeline for the SC FRAMEWORK repository.
# The pipeline is divided into several stages:
# 1. .pre: This stage is used to export variables and to get the sctoolbox version as global variable.
# 2. build: This stage checks if something in the enviroment (setup.py or sctoolbox_env.yml) has changed.
#           If yes then a new docker image will be build and pushed to the container registry.
#           The newly build image is then used for testing later.
#           If no changes were made to the enviroment nothing is build and the latest image is used later on.
# 3. setup: This stage is used to get the container tag for the testing stage.
# 4. test: This stage is used to lint the code and docstrings, to test if the minimal sctoolbox package can still be imported without additional dependencies,
#          to run tests and to test if the notebooks contain output.
# 5. deploy: This stage is used to build and deploy the documentation to the public pages if it was the main branch.

stages:
  - .pre
  - build
  - setup
  - test
  - deploy
  - .post

# ############################################################################################################################################################
# ######################################################### pre ##############################################################################################
# ############################################################################################################################################################

# Shows all CI-variables with values. Only for debugging.
ci-variables:
  stage: .pre
  image: python:3.10-alpine3.19
  script:
    - export
  rules:
    # To enable the job again switch "when: never" to "if: $CI".
    - when: never
    # - if: $CI

# get the sctoolbox version as global variable
extract-version:
  stage: .pre
  image: python:3.10-alpine3.19
  script:
    - VERSION=$(python -c "import sys; sys.path.append('./sctoolbox'); from _version import __version__; print(__version__)")
    - echo "BUILD_VERSION=$VERSION"
    - echo "BUILD_VERSION=$VERSION" >> build.env
  artifacts:
    reports:
      dotenv: build.env
  rules:
    - if: $CI


# ############################################################################################################################################################
# ######################################################### build docker image ###############################################################################
# ############################################################################################################################################################

# The build stage below checks if something in the enviroment (setup.py or sctoolbox_env.yml) has changed.
# If yes then a new docker image will be build and pushed to the container registry.
# The newly build image is then used for testing later. 
# If no changes were made to the enviroment nothing is build and the latest image is used later on.

# build job
build:
  stage: build
  image: docker:latest # image as building environment
  needs: [extract-version]
  tags:
    - sctoolbox
  services:
      - name: "docker:dind" # docker in docker to build
        alias: docker
  before_script:
      - echo $BUILD_VERSION
      - apk update && apk add git # install git for authentification and pushing to registry
      - apk update -qq && apk add git
      - echo $CI_REGISTRY_PASSWORD | docker login -u $CI_REGISTRY_USER $CI_REGISTRY --password-stdin # login to registry
  script:
      - >
        if [ "$CI_COMMIT_BRANCH" == "main" ]; then
          docker build --pull -t "$CI_REGISTRY_IMAGE:$BUILD_VERSION" -t "$CI_REGISTRY_IMAGE:latest" . # build the image from Dockerfile in the repository
          docker push --all-tags "$CI_REGISTRY_IMAGE" # push image to registry
        elif [ "$CI_COMMIT_BRANCH" == "dev" ]; then
          docker build --pull -t "$CI_REGISTRY_IMAGE:dev" . # build the image from Dockerfile in the repository
          docker push --all-tags "$CI_REGISTRY_IMAGE" # push image to registry
        elif [ "$CI_COMMIT_BRANCH" != "main" ] && [ "$CI_COMMIT_BRANCH" != "dev" ]; then
          docker build --pull -t "$CI_REGISTRY_IMAGE:$CI_COMMIT_BRANCH" . # build the image from Dockerfile in the repository
          docker push --all-tags "$CI_REGISTRY_IMAGE" # push image to registry
        else
          echo "No image will be build."
        fi

  rules:
    - if: $CI_COMMIT_BRANCH
      changes:  # if any of the files were changed
        - setup.py
        - sctoolbox_env.yml
        - Dockerfile
        - MANIFEST.in
        - pyproject.toml
    - if: $CI_COMMIT_BRANCH == "main" && $CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_REF_SLUG == 'main'  # all MR to main


# ############################################################################################################################################################
# ######################################################### setup #############################################################################################
# ############################################################################################################################################################

# The setup stage is used to get the container tag for the testing stage.
# The container tag is determined by the branch name or the merge request id.
# The tag is then written to a file and used as an artifact for the testing stage.
get-container-tag:
  stage: setup
  image:
    name: gcr.io/go-containerregistry/crane:debug
    entrypoint: [""]
  before_script:
    - crane auth login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
    - ALL_TAGS=$(crane ls $CI_REGISTRY_IMAGE)
  script:
    - echo $ALL_TAGS
    - > 
      if [ "$CI_COMMIT_BRANCH" == "main" ]; then 
          echo "DYNAMIC_IMAGE_TAG=$BUILD_VERSION (commit on: $CI_COMMIT_BRANCH)"
          echo "DYNAMIC_IMAGE_TAG=$BUILD_VERSION" >> build.env
      elif [ "$CI_COMMIT_BRANCH" == "dev" ]; then 
          echo "DYNAMIC_IMAGE_TAG=dev (commit on: $CI_COMMIT_BRANCH)"
          echo "DYNAMIC_IMAGE_TAG=dev" >> build.env
      elif [ "$CI_COMMIT_BRANCH" != "main" ] && [ "$CI_COMMIT_BRANCH" != "dev" ] && [ -n "$CI_COMMIT_BRANCH" ]; then 
          if echo "$ALL_TAGS" | grep -qF "$CI_COMMIT_BRANCH"; then
              echo "DYNAMIC_IMAGE_TAG=$CI_COMMIT_BRANCH (commit on: $CI_COMMIT_BRANCH)"
              echo "DYNAMIC_IMAGE_TAG=$CI_COMMIT_BRANCH" >> build.env
          else 
              echo "DYNAMIC_IMAGE_TAG=dev (commit on: $CI_COMMIT_BRANCH, has no image. Falling back to dev tag)"
              echo "DYNAMIC_IMAGE_TAG=dev" >> build.env
          fi
      elif [ "$CI_PIPELINE_SOURCE" == "merge_request_event" ]; then 
          if [ "$CI_COMMIT_REF_NAME" == "dev" ]; then 
              echo "DYNAMIC_IMAGE_TAG=dev (merge request: $CI_COMMIT_REF_NAME)"
              echo "DYNAMIC_IMAGE_TAG=dev" >> build.env
          elif [ "$CI_COMMIT_REF_NAME" != "dev" ]; then
              echo "all tags: $ALL_TAGS"
              echo "Reference branch: $CI_COMMIT_REF_NAME" 
              if echo "$ALL_TAGS" | grep -qF "$CI_COMMIT_REF_NAME"; then
                  echo "DYNAMIC_IMAGE_TAG=$CI_COMMIT_REF_NAME (merge request: $CI_COMMIT_REF_NAME)"
                  echo "DYNAMIC_IMAGE_TAG=$CI_COMMIT_REF_NAME" >> build.env
              else 
                  echo "DYNAMIC_IMAGE_TAG=dev (merge request: $CI_COMMIT_REF_NAME, has no image. Falling back to dev tag)"
                  echo "DYNAMIC_IMAGE_TAG=dev" >> build.env
              fi
          fi
      else 
          echo "DYNAMIC_IMAGE_TAG=latest (unknown action)"
          echo "DYNAMIC_IMAGE_TAG=latest" >> build.env
      fi

  artifacts:
    reports:
      dotenv: build.env
      
  rules:
    - if: $CI

# ############################################################################################################################################################
# ######################################################### test #############################################################################################
# ############################################################################################################################################################

# Change pip's cache directory to be inside the project directory since we can
# only cache local items.
variables:
  PIP_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pip"
# Pip's cache doesn't store the python packages
# https://pip.pypa.io/en/stable/topics/caching/
#
# If you want to also cache the installed packages, you have to install
# them in a virtualenv and cache it as well.
cache:
  key: ${CI_COMMIT_REF_SLUG}
  paths:
    - .cache/pip
    - venv/

# lint code and docstrings
# flake8 ignore rules:
# E501 = Line too long
# W503 = Line break occurred before a binary operator
# D202 = No blank lines allowed after function docstring
# DOC301 = __init__() should not have a docstring; <- conflicts with "D107 Missing docstring in __init__"
lint:
  image: 
    name: python:3.10 # pybedtools is not compatible with python 3.11
    entrypoint: [ '/bin/bash', '-c', 'ln -snf /bin/bash /bin/sh && /bin/bash -c $0' ]  # Fixes shell not found error (See Issue #129)
  inherit:
    default: false # do not inherit before_script
  stage: test
  tags:
    - sctoolbox
  script:
    - python --version
    - pip install flake8
    - pip install pydoclint>=0.1.7
    - pip install flake8-docstrings  # adds pydocstyle to flake
    - flake8 sctoolbox setup.py tests --ignore=E501,W503,D202,DOC301,DOC105 -sdpca True --docstring-convention numpy --show-source --extend-exclude=sctoolbox/__init__.py,sctoolbox/data,tests/data
  allow_failure: false
  rules:
    - if: $CI_PIPELINE_SOURCE == "merge_request_event" # all MR
      changes:
        - setup.py
        - tests/*.py
        - tests/**/*.py
        - sctoolbox/**/*.py

# Test that the minimal sctoolbox package can still be imported without additional dependencies
test import:
  image: python:3.10  # pybedtools is not compatible with python 3.11
  inherit:
      default: false # do not inherit before_script
  stage: test
  script:
    - pip install .
    - python -c "import sctoolbox"
  rules:
    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
      changes:
        - setup.py
        - sctoolbox/**/*.py

# Run tests
test coverage:
    stage: test
    image: "$CI_REGISTRY_IMAGE:$DYNAMIC_IMAGE_TAG"
    needs: [get-container-tag]
    tags:
      - sctoolbox
    coverage: '/TOTAL.*\s(\d*.\d*\%)/'
    before_script:
      - pip install .[all]
    script:
      - echo $DYNAMIC_IMAGE_TAG
      - chmod +x scripts/bedGraphToBigWig  # make script executable
      - pytest --cov-report=term --cov=./sctoolbox . --junitxml=pytest.xml --cov-report=html:htmlcov --html=pytest.html --durations=0  # durations=0 shows the ranked duration of all tests for improving compute time
    artifacts:
      when: always
      paths:
          - pytest.xml
          - pytest.html
          - htmlcov
      reports:
        junit: pytest.xml
    rules:
      - if: $CI_PIPELINE_SOURCE == "merge_request_event"  #all merge requests with changes
        changes:
          - setup.py
          - sctoolbox_env.yml
          - Dockerfile
          - sctoolbox/_version.py
          - sctoolbox/**/*.py
          - tests/*.py
          - tests/**/*.py
      - if: $CI_COMMIT_BRANCH == "main" # commits to dev (for coverage)
      - if: $CI_COMMIT_BRANCH == "dev" # commits to dev (for coverage)


#Changes.rst file must be changed in merge requests
check-changes:
  image: python:3.10
  inherit:
    default: false
  stage: test
  script:
    - python scripts/check_changes.py $CI_MERGE_REQUEST_TARGET_BRANCH_NAME
  rules:
    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
      changes:
        - tests/*.py
        - tests/**/*.py
        - setup.py
        - sctoolbox_env.yml
        - Dockerfile
        - sctoolbox/_version.py
        - sctoolbox/**/*.py
        - rna_analysis/notebooks/*.ipynb
        - atac_analysis/notebooks/*.ipynb
        - general_notebooks/*.ipynb

#In addition to CHANGES.rst, the _version.py file must be changed in merge requests to main in order to make a new version
check-version:
  image:
    name: cicirello/alpine-plus-plus
    entrypoint: [""]  #hack from https://gitlab.com/gitlab-org/gitlab-runner/-/issues/4027
  inherit:
    default: false
  script:
    - git fetch origin $CI_MERGE_REQUEST_TARGET_BRANCH_NAME
    - git diff-tree --name-only -r "HEAD..origin/$CI_MERGE_REQUEST_TARGET_BRANCH_NAME" > changed_files.txt
    - cat changed_files.txt
    - if grep -q "sctoolbox/_version.py" changed_files.txt; then echo "The 'sctoolbox/_version.py' file was changed."; else echo "The 'sctoolbox/_version.py' file must be changed for merge requests to main!"; exit 1; fi
    - version=$(sed -e 's/.*\"\(.*\)\".*/\1/' <<< $(head -n 1 sctoolbox/_version.py)) # Get version from _version.py file
    - echo ${version} # for debugging
    - if [[ ! ${version} =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then exit 1; fi
  rules:
    - if: $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "main" # only on MR to main

# ############################################################################################################################################################
# ######################################################### Test Notebooks ###################################################################################
# ############################################################################################################################################################

# test if notebooks contain output; these should be removed using the .gitconfig addition given in the README
notebook-output-check:
  image: python:3.10-alpine3.19
  inherit:
    default: false
  stage: test
  script:
    - python --version
    - python scripts/check_notebooks.py */*.ipynb */*/*.ipynb
  rules:
    - if: $CI_PIPELINE_SOURCE == "merge_request_event" # all MR

# Test notebooks
notebooks:
  image: "$CI_REGISTRY_IMAGE:$DYNAMIC_IMAGE_TAG"
  needs: [get-container-tag]
  inherit:
    default: false
  stage: test
  tags:
      - sctoolbox
  artifacts:
    paths:
      - rna_analysis/logs/*.txt
      - rna_analysis/*.ipynb
      - atac_analysis/*.ipynb
      - atac_analysis/logs/*.txt
  before_script:
    - pip install .[all]
  script:
    - pip install papermill
    #  - mv /opt/conda/lib/python3.10/site-packages/urllib3-2.1.0.dist-info/ .  # Fixes metadata bug
    - apt-get update
    - apt-get -y install libsasl2-dev python-dev-is-python3 libldap2-dev libssl-dev
    - pip install python-ldap pyOpenSSL
    - pip install git+https://gitlab-ci-token:${CI_JOB_TOKEN}@gitlab.gwdg.de/loosolab/software/mampok.git
    - python -m ipykernel install --user --name sctoolbox
    - cp -t ./rna_analysis/notebooks/ ./general_notebooks/prepare_for_cellxgene.ipynb ./general_notebooks/annotation.ipynb ./general_notebooks/GSEA.ipynb ./general_notebooks/group_markers.ipynb # ./general_notebooks/pseudotime_analysis.ipynb
    - python scripts/run_notebooks.py
  after_script:
    - pwd
    - echo -------------------------------------------------- RNA logs --------------------------------------------------
    - ls -l rna_analysis/logs/ 2>/dev/null || true && echo "Log dir not found! Please check the artifacts to identify potential errors that occured within the notebooks."
    - if [[ -d rna_analysis/logs/ && "$(ls -A rna_analysis/logs/)" ]]; then for f in rna_analysis/logs/*; do echo "==> $f <=="; cat $f; done; fi
    - echo -------------------------------------------------- ATAC logs --------------------------------------------------
    - ls -l atac_analysis/logs/ 2>/dev/null || true && echo "Log dir not found! Please check the artifacts to identify potential errors that occured within the notebooks."
    - if [[ -d atac_analysis/logs/ && "$(ls -A atac_analysis/logs/)" ]]; then for f in atac_analysis/logs/*; do echo "==> $f <=="; cat $f; done; fi
  rules:
    - if: $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "main" # only on MR to main
    - if: $CI_MERGE_REQUEST_TARGET_BRANCH_NAME != "main" && $CI_PIPELINE_SOURCE == "merge_request_event"
      changes:  # if any of the files were changed
        - setup.py
        - sctoolbox_env.yml
        - Dockerfile
        - sctoolbox/_version.py
        - rna_analysis/notebooks/*.ipynb
        - atac_analysis/notebooks/*.ipynb
        - general_notebooks/*.ipynb

# In addition to CHANGES.rst and the _version.py file the notebook version for every notebook
# must be changed in merge requests to main in order to make a new version
check-notebook-version:
  image: python:3.10-alpine3.19
  needs: [check-version]
  inherit:
    default: false
  stage: test
  before_script:
    pip install nbformat packaging
  script:
    - python --version
    - python scripts/check_notebook_version.py -n rna_analysis/notebooks/ atac_analysis/notebooks/ general_notebooks/ -v sctoolbox
  rules:
    - if: $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "main" # only on MR to main


# ############################################################################################################################################################
# ######################################################### Build and deploy documentation ###################################################################
# ############################################################################################################################################################

build-pages:
  image: "$CI_REGISTRY_IMAGE:$DYNAMIC_IMAGE_TAG"
  needs: 
    - job: get-container-tag
      artifacts: true
  stage: deploy
  before_script:
    - pip install .[all]
  script:
    - apt-get autoclean  # fix "You don't have enough free space in /var/cache/apt/archives/."
    - apt-get update -qq && apt-get install -qq -y pandoc  # system install of pandoc is needed
    - pip install sphinx sphinx-rtd-theme
    - pip install sphinx-exec-code
    - pip install nbsphinx
    - pip install nbsphinx_link
    - cd docs
    - make html
  artifacts:
    paths:
      - docs/build/html/
  rules:
    - if: $CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH != "dev" && $CI_COMMIT_BRANCH != "main" # dev/main are built automatically
      when: manual
      allow_failure: true
    - if: $CI_COMMIT_BRANCH == "dev"  # always for commits to dev
      changes:
        - sctoolbox/**/*.py
        - rna_analysis/notebooks/*.ipynb
        - atac_analysis/notebooks/*.ipynb
        - general_notebooks/*.ipynb  # changes in general notebooks
      allow_failure: false
    - if: $CI_COMMIT_BRANCH == "main"  # always for commits to dev
      allow_failure: false
    - if: $CI_PIPELINE_SOURCE == "merge_request_event"  # all MR
      when: manual
      allow_failure: true

deploy-pages:
  stage: deploy
  needs:
    - job: build-pages
      artifacts: True
  inherit:
    default: false
  environment:
    name: pages/$CI_COMMIT_REF_NAME
    url: "https://loosolab.pages.gwdg.de/-/software/sc_framework/-/jobs/$CI_JOB_ID/artifacts/docs/build/html/index.html"
  artifacts:
    paths:
      - docs/build/html/
  script:
    - ls -l docs/
    - echo "deploy"
  rules:
    - if: $CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH != "dev" && $CI_COMMIT_BRANCH != "main" # dev/main are built automatically
      when: manual
      allow_failure: true
    - if: $CI_COMMIT_BRANCH == "dev"  # always for commits to dev
      changes:
        - sctoolbox/**/*.py
        - rna_analysis/notebooks/*.ipynb
        - atac_analysis/notebooks/*.ipynb
        - general_notebooks/*.ipynb  # changes in general notebooks
      allow_failure: false
    - if: $CI_COMMIT_BRANCH == "main"  # always for commits to dev
      allow_failure: false
    - if: $CI_PIPELINE_SOURCE == "merge_request_event"  # all MR
      when: manual
      allow_failure: true

# Deploy documentation to public pages if it was the main branch
pages:
  stage: deploy
  needs:
    - job: build-pages
      artifacts: True
  inherit:
    default: false
  script:
    - mv docs/build/html/ public/
  artifacts:
    paths:
      - public
  rules:
    - if: $CI_COMMIT_BRANCH == "main" # after accepted MR to main
      allow_failure: false  # main not allowed to fail

# ############################################################################################################################################################
# ######################################################### .post #############################################################################################
# ############################################################################################################################################################

# clean up
# This job is used to delete orphan tags from the registry.
# It is used to delete tags that are not associated with a branch or a semantic versioning.
# The job is only executed if the commit is on the main branch.
# For the cleanupo the branches and tags are fetched from the repository via the Gitlab API.
# Requests are made by using curl and the jq package is used to parse the json responses.
# The tags are then compared against the branches and deleted if they are not associated with a branch or a semantic versioning.
# The job uses crane (https://github.com/google/go-containerregistry/tree/main/cmd/crane) to delete the tags from the registry.
cleanup-orphan-tags:
  stage: .post
  image: alpine:latest
  before_script:
    # Install curl and jq
    - apk add --no-cache curl jq
    # Install crane
    - curl -L "https://github.com/google/go-containerregistry/releases/download/v0.7.0/go-containerregistry_Linux_x86_64.tar.gz" -o /tmp/crane.tar.gz
    - tar -zxvf /tmp/crane.tar.gz -C /usr/local/bin/ crane
    - rm /tmp/crane.tar.gz
    - crane version
    # Login to the registry
    - crane auth login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
  script:
    - |
      # Fetch all branch names
      BRANCHES=$(curl --silent --header "PRIVATE-TOKEN: $API_TOKEN" "https://gitlab.gwdg.de/api/v4/projects/$CI_PROJECT_ID/repository/branches" | jq -r '.[].name')
      echo "Branches:"
      echo "$BRANCHES"
      
      # Fetch repository ID for Docker Registry
      repo_id=$(curl --silent --header "PRIVATE-TOKEN: $API_TOKEN" "https://gitlab.gwdg.de/api/v4/projects/$CI_PROJECT_ID/registry/repositories" | jq -r ".[0].id")
      
      # Fetch all image tags
      TAGS=$(curl --silent --header "PRIVATE-TOKEN: $API_TOKEN" "https://gitlab.gwdg.de/api/v4/projects/$CI_PROJECT_ID/registry/repositories/${repo_id}/tags" | jq -r '.[].name')
      echo "Tags:"
      echo "$TAGS"
      
      # Regular expression for semantic versioning
      SEMVER_REGEX="^([0-9]+)\.([0-9]+)\.([0-9]+)$"

      # Compare tags against branches and skip semantic versioning tags
      for tag in $TAGS; do
        if [[ $tag =~ $SEMVER_REGEX ]] || [[ $tag == "latest" ]] || [[ $tag == "dev" ]]; then
          echo "Tag '$tag' follows semantic versioning or is 'latest' and will be kept."
        elif ! echo "$BRANCHES" | grep -qx "$tag"; then
          echo "Tag '$tag' does not match any branch and does not follow semantic versioning, will be deleted."
          # Use crane to delete the tag
          crane delete "$CI_REGISTRY_IMAGE:$tag"
        else
          echo "Tag '$tag' matches a branch and will not be deleted."
        fi
      done
  rules:
    - if: $CI_COMMIT_BRANCH == "main"  # only for commits to main