Run examples using pytest + optimize CI setup (conda#675)

jaimergp · web-flow · commit 3bb9c98e1448 · 2023-06-19T15:59:40.000+02:00
* run examples with pytest (draft)

* rethink testing CI

* typo in path

* do not use python explicitly

* import or skip

* just include pillow in core deps

* add debug, pre-commit

* set level in the root constructor logger

* mkdir

* ensure string?

* move to 3.11

* enable keep_artifacts via env var

* deprecate script

* parametrize sh miniforge tests

* format

* refactor a bit

* ensure debug

* add news

* constructor debug as an env var too

* implement windows signing in pytest too

* update docstring

* make str

* fix missing arg

* skip on unix

* fix space check

* membership is enough (nested configs...)

* str env var

* fix input for interactive path

* iterate

* print directory contents?

* pwd too

* str here too

* uninstall after sentinel checks

* fix path for scripts

* fix sentinel checks on macos PKGs

* export certificate path

* do not initialize or register conda on examples

* sort shell installers first

* just move

* remove $HOME stuff after PKG installs

* add extension as suffix

* pre-commit

* add warning

* use conda-canary

* Update tests/test_examples.py

* update readme
diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
@@ -28,201 +28,143 @@ concurrency:
 
 defaults:
   run:
-    shell: bash
+    shell: bash -el {0}
 jobs:
-  package:
-    name: ${{ matrix.os }}, Python ${{ matrix.pyver }}, ${{ matrix.micromamba && 'micromamba' || 'conda-standalone' }}
+  tests:
+    name: ${{ matrix.os }}, Python ${{ matrix.python-version }}, ${{ matrix.micromamba && 'micromamba' || 'conda-standalone' }}
     runs-on: ${{ matrix.os }}-latest
     strategy:
       fail-fast: false
       matrix:
         os: [macos, ubuntu, windows]
-        pyver: ["3.7", "3.8", "3.9", "3.10"]
+        python-version: ["3.8", "3.9", "3.10", "3.11"]
+        check-docs: [false]
         include:
           - os: ubuntu
-            pyver: "3.9"
+            python-version: "3.9"
             micromamba: true
+            check-docs: true
           - os: macos
-            pyver: "3.10"
+            python-version: "3.10"
             micromamba: true
+            check-docs: false
           # Re-enable once micromamba supports menu creation
           # - os: windows
-          #   pyver: "3.8"
+          #   python-version: "3.8"
           #   micromamba: true
+          #   check-docs: false
     env:
-      PYTHONUNBUFFERED: True
+      PYTHONUNBUFFERED: "1"
     steps:
-      - name: Print github context
-        run: |
-          echo "EVENT_NAME:" "$GITHUB_EVENT_NAME"
-          echo "       REF:" "$GITHUB_REF"
-          echo "  HEAD_REF:" "$GITHUB_HEAD_REF"
-          echo "  BASE_REF:" "$GITHUB_BASE_REF"
-          echo "       SHA:" "$GITHUB_SHA"
-      - name: Set temp dirs correctly
-        if: startsWith(matrix.os, 'windows')
-        # https://github.com/actions/virtual-environments/issues/712
-        shell: powershell
-        run: |
-          echo "TMPDIR=$env:USERPROFILE\AppData\Local\Temp" >> $env:GITHUB_ENV
-          echo "TEMP=$env:USERPROFILE\AppData\Local\Temp" >> $env:GITHUB_ENV
-          echo "TMP=$env:USERPROFILE\AppData\Local\Temp" >> $env:GITHUB_ENV
       - name: Retrieve the source code
         uses: actions/checkout@v3
         with:
           fetch-depth: 0
-          ref: ${{ github.event.pull_request.head.sha }}
-      - name: Build the build environment
-        run: |
-          source $CONDA/etc/profile.d/conda.sh
-          [ $RUNNER_OS == macOS ] && export CONDA_PKGS_DIRS=~/.pkgs
-          conda create -p ../conda conda-build conda-verify
-      - name: Build the package
-        env:
-          PYTHONIOENCODING: utf-8
-          # Uncomment to run within conda build
-          # RUN_EXAMPLES: "1"
-        run: |
-          source $CONDA/etc/profile.d/conda.sh
-          conda activate ../conda
-          export CODECOV_COMMIT=$(git rev-parse --verify HEAD)
-          CODECOV_FOLDER=${PWD} \
-          CONDA_BLD_PATH="${{ runner.temp }}/conda-bld" \
-            conda build conda.recipe --python=${{ matrix.pyver }}
-      - uses: codecov/codecov-action@v3
+      - uses: conda-incubator/setup-miniconda@v2
         with:
-          token: ${{ secrets.CODECOV_TOKEN }}
-          flags: unit
-      - name: Upload the packages as artifact
-        if: github.event_name == 'push'
-        uses: actions/upload-artifact@v3
-        with:
-          # By uploading to the same artifact we can download all of the packages
-          # and upload them all to anaconda.org in a single job
-          name: package-${{ github.sha }}
-          path: ${{ runner.temp }}/conda-bld/*/*.tar.bz2
-      - name: Install local constructor
-        run: |
-          source $CONDA/etc/profile.d/conda.sh
-          CONDA_BLD_PATH="${{ runner.temp }}/conda-bld" \
-            conda create -n constructor -c local --strict-channel-priority constructor coverage
-          conda activate constructor
-          set -x
-          installed_channel=$(conda list constructor --json | jq -r '.[].channel')
-          if [[ "$installed_channel" != "conda-bld" ]]; then
-            echo $(conda list constructor --json)
-            echo "Installed constructor is not local!"
-            exit 1
-          fi
-          constructor --version
-          constructor --help-construct
-      - name: Update to NSIS logging builds on Windows
-        if: startsWith(matrix.os, 'windows')
+          activate-environment: constructor-dev
+          environment-file: dev/environment.yml
+          python-version: ${{ matrix.python-version }}
+      - name: Supply extra dependencies and install constructor
         run: |
-          source $CONDA/etc/profile.d/conda.sh
-          conda activate constructor
-          conda install -y "conda-forge::nsis=*=*_log_*"
+          files=(--file "tests/requirements.txt")
+          test -f dev/extra-requirements-${{ matrix.os }}.txt \
+            && files+=(--file "dev/extra-requirements-${{ matrix.os }}.txt")
+          conda install ${files[@]} -y
           echo "NSIS_USING_LOG_BUILD=1" >> $GITHUB_ENV
-      - name: Generate self-signed certificate (Windows)
-        if: startsWith(matrix.os, 'windows')
-        shell: cmd
-        run: |
-          set "CONSTRUCTOR_SIGNING_CERTIFICATE=${{ runner.temp }}\certificate.pfx"
-          set "CONSTRUCTOR_PFX_CERTIFICATE_PASSWORD=1234"
-          powershell scripts\create_self_signed_certificate.ps1
-          copy /Y "%CONSTRUCTOR_SIGNING_CERTIFICATE%" examples\signing\certificate.pfx
-
-          :: Careful with the trailing spaces before the >> redirect!
-          echo CONSTRUCTOR_PFX_CERTIFICATE_PASSWORD=1234>> %GITHUB_ENV%
-          echo CONSTRUCTOR_SIGNTOOL_PATH=C:\Program Files (x86)\Windows Kits\10\bin\10.0.17763.0\x86\signtool.exe>> %GITHUB_ENV%
+          pip install -e . --no-deps --no-build-isolation
       - name: Set up conda executable
         run: |
-          source $CONDA/etc/profile.d/conda.sh
           if [[ "${{ matrix.micromamba }}" != "" ]]; then
-            conda create -yqp ./micromamba -c conda-forge micromamba
+            conda create -yqp "${{ runner.temp }}/micromamba" -c conda-forge micromamba
             if [[ ${{ matrix.os }} == "windows" ]]; then
-              echo "CONSTRUCTOR_CONDA_EXE=./micromamba/Library/bin/micromamba.exe" >> $GITHUB_ENV
+              echo "CONSTRUCTOR_CONDA_EXE=${{ runner.temp }}/micromamba/Library/bin/micromamba.exe" >> $GITHUB_ENV
             else
-              echo "CONSTRUCTOR_CONDA_EXE=./micromamba/bin/micromamba" >> $GITHUB_ENV
+              echo "CONSTRUCTOR_CONDA_EXE=${{ runner.temp }}/micromamba/bin/micromamba" >> $GITHUB_ENV
             fi
           else
-            conda activate constructor
+            conda activate constructor-dev
             echo "CONSTRUCTOR_CONDA_EXE=$CONDA_PREFIX/standalone_conda/conda.exe" >> $GITHUB_ENV
           fi
-      - name: Run examples and prepare artifacts
+      - name: Run unit tests
         run: |
-          rm -rf coverage.json
-          source $CONDA/etc/profile.d/conda.sh
-          conda activate constructor
-          mkdir -p examples_artifacts/
-          python scripts/run_examples.py \
-            --keep-artifacts=examples_artifacts/ \
-            --conda-exe="${CONSTRUCTOR_CONDA_EXE}"
+          pytest -vv --cov=constructor --cov-branch tests/ -m "not examples"
+          coverage run --branch --append -m constructor -V
           coverage json
-      - name: Test with conda-libmamba-solver
+      - uses: codecov/codecov-action@v3
+        with:
+          token: ${{ secrets.CODECOV_TOKEN }}
+          flags: unit
+      - name: Run examples
+        env:
+          CONSTRUCTOR_EXAMPLES_KEEP_ARTIFACTS: "${{ runner.temp }}/examples_artifacts"
+          # signtool only exists on Windows, but doesn't cause errors on unix when absent
+          CONSTRUCTOR_SIGNTOOL_PATH: "C:/Program Files (x86)/Windows Kits/10/bin/10.0.17763.0/x86/signtool.exe"
         run: |
-          source $CONDA/etc/profile.d/conda.sh
-          conda activate constructor
-          conda install -yq conda-libmamba-solver
-          conda list
-          CONDA_SOLVER=libmamba CONDA_VERBOSITY=1 constructor examples/noconda/ --output-dir=examples_artifacts/
+          rm -rf coverage.json
+          pytest -vv --cov=constructor --cov-branch tests/test_examples.py
+          coverage run --branch --append -m constructor -V
+          coverage json
       - uses: codecov/codecov-action@v3
         with:
           token: ${{ secrets.CODECOV_TOKEN }}
           flags: integration
+      - name: Test with conda-libmamba-solver
+        run: |
+          conda install -yq conda-libmamba-solver
+          CONDA_SOLVER=libmamba CONDA_VERBOSITY=1 pytest -vv tests/test_examples.py -k noconda
+      - name: Check docs are up-to-date
+        if: matrix.check-docs
+        run: |
+          python scripts/make_docs.py
+          git diff --exit-code
       - name: Upload the example installers as artifacts
-        if: github.event_name == 'pull_request' && matrix.pyver == '3.9'
+        if: github.event_name == 'pull_request' && matrix.python-version == '3.9'
         uses: actions/upload-artifact@v3
         with:
           name: installers-${{ runner.os }}-${{ github.sha }}-${{ github.run_id }}-${{ github.run_number }}-${{ github.run_attempt }}
-          path: examples_artifacts/
+          path: "${{ runner.temp }}/examples_artifacts"
           retention-days: 7
 
-  upload:
-    needs: package
-    runs-on: ubuntu-latest
-    if: github.event_name == 'push'
+  build:
+    name: Canary Build
+    needs: [tests]
+    # only build canary build if
+    # only build canary build if
+    # - prior steps succeeded,
+    # - this is the main repo, and
+    # - we are on the main (or feature) branch
+    if: >-
+      success()
+      && !github.event.repository.fork
+      && (
+        github.ref_name == 'main'
+        || startsWith(github.ref_name, 'feature/')
+      )
+    strategy:
+      matrix:
+        include:
+          - runner: ubuntu-latest
+            subdir: linux-64
+          - runner: macos-latest
+            subdir: osx-64
+          - runner: windows-latest
+            subdir: win-64
+    runs-on: ${{ matrix.runner }}
     steps:
-      - name: Retrieve the source code
-        uses: actions/checkout@v3
+      # Clean checkout of specific git ref needed for package metadata version
+      # which needs env vars GIT_DESCRIBE_TAG and GIT_BUILD_STR:
+      - uses: actions/checkout@v3
         with:
+          ref: ${{ github.ref }}
+          clean: true
           fetch-depth: 0
-      - name: Download the build artifacts
-        uses: actions/download-artifact@v3
-        with:
-          name: package-${{ github.sha }}
-          path: conda-bld
-      - name: Install conda packages
-        run: |
-          source $CONDA/bin/activate
-          conda install -y sphinx anaconda-client
-      - name: Upload to anaconda.org
-        env:
-          ANACONDA_TOKEN: ${{ secrets.ANACONDA_TOKEN }}
-          GITHUB_REF: ${{ github.ref }}
-        run: |
-          source $CONDA/bin/activate
-          [[ "$GITHUB_REF" =~ ^refs/tags/ ]] || export LABEL="--label dev"
-          anaconda --verbose --token $ANACONDA_TOKEN upload --user ctools $LABEL conda-bld/*/*.tar.bz2 --force
-
-  docs:
-    name: Check docs are up-to-date
-    runs-on: ubuntu-latest
-    steps:
-      - name: Retrieve the source code
-        uses: actions/checkout@v3
+      - name: Create and upload canary build
+        uses: conda/actions/canary-release@v23.3.0
         with:
-          fetch-depth: 0
-      - name: Install local constructor
-        run: |
-          source $CONDA/bin/activate
-          conda create -n constructor constructor jinja2
-          conda activate constructor
-          pip install -U . --no-deps
-      - name: Build docs
-        run: |
-          source $CONDA/bin/activate
-          conda activate constructor
-          set -ex
-          python scripts/make_docs.py
-          git diff --exit-code
+          package-name: ${{ github.event.repository.name }}
+          subdir: ${{ matrix.subdir }}
+          anaconda-org-channel: conda-canary
+          anaconda-org-label: ${{ github.ref_name == 'main' && 'dev' || format('{0}-{1}', github.event.repository.name, github.ref_name) }}
+          anaconda-org-token: ${{ secrets.CONDA_CANARY_ANACONDA_ORG_TOKEN }}
+          conda-build-arguments: '--override-channels -c conda-forge -c defaults'
diff --git a/README.md b/README.md
@@ -42,8 +42,8 @@ the script directly with ``python scripts/make_docs.py``.
 
 ## Build status
 
-| [![Build status](https://github.com/conda/constructor/actions/workflows/main.yml/badge.svg)](https://github.com/conda/constructor/actions/workflows/main.yml) [![Docs status](https://github.com/conda/constructor/actions/workflows/docs.yml/badge.svg)](https://github.com/conda/constructor/actions/workflows/docs.yml) [![codecov](https://codecov.io/gh/conda/constructor/branch/main/graph/badge.svg)](https://codecov.io/gh/conda/constructor) [![pre-commit.ci status](https://results.pre-commit.ci/badge/github/conda/constructor/main.svg)](https://results.pre-commit.ci/latest/github/conda/constructor/main)  | [![Anaconda-Server Badge](https://anaconda.org/ctools/constructor/badges/latest_release_date.svg)](https://anaconda.org/ctools/constructor) |
+| [![Build status](https://github.com/conda/constructor/actions/workflows/main.yml/badge.svg)](https://github.com/conda/constructor/actions/workflows/main.yml) [![Docs status](https://github.com/conda/constructor/actions/workflows/docs.yml/badge.svg)](https://github.com/conda/constructor/actions/workflows/docs.yml) [![codecov](https://codecov.io/gh/conda/constructor/branch/main/graph/badge.svg)](https://codecov.io/gh/conda/constructor) [![pre-commit.ci status](https://results.pre-commit.ci/badge/github/conda/constructor/main.svg)](https://results.pre-commit.ci/latest/github/conda/constructor/main)  | [![Anaconda-Server Badge](https://anaconda.org/conda-canary/constructor/badges/latest_release_date.svg)](https://anaconda.org/conda-canary/constructor) |
 | --- | :-: |
 | [`conda install defaults::constructor`](https://anaconda.org/anaconda/constructor) | [![Anaconda-Server Badge](https://anaconda.org/anaconda/constructor/badges/version.svg)](https://anaconda.org/anaconda/constructor) |
 | [`conda install conda-forge::constructor`](https://anaconda.org/conda-forge/constructor) | [![Anaconda-Server Badge](https://anaconda.org/conda-forge/constructor/badges/version.svg)](https://anaconda.org/conda-forge/constructor) |
-| [`conda install ctools/label/dev::constructor`](https://anaconda.org/ctools/constructor) | [![Anaconda-Server Badge](https://anaconda.org/ctools/constructor/badges/version.svg)](https://anaconda.org/ctools/constructor) |
+| [`conda install conda-canary/label/dev::constructor`](https://anaconda.org/conda-canary/constructor) | [![Anaconda-Server Badge](https://anaconda.org/conda-canary/constructor/badges/version.svg)](https://anaconda.org/conda-canary/constructor) |
diff --git a/constructor/main.py b/constructor/main.py
@@ -314,7 +314,7 @@ def main():
     logger.info("Got the following cli arguments: '%s'", args)
 
     if args.verbose or args.debug:
-        logger.setLevel(logging.DEBUG)
+        logging.getLogger("constructor").setLevel(logging.DEBUG)
 
     if args.clean:
         import shutil
diff --git a/constructor/osxpkg.py b/constructor/osxpkg.py
@@ -10,7 +10,8 @@
 from tempfile import NamedTemporaryFile
 
 from . import preconda
-from .construct import ns_platform
+from .conda_interface import conda_context
+from .construct import ns_platform, parse
 from .imaging import write_images
 from .utils import add_condarc, approx_size_kb, fill_template, get_final_channels, preprocess, rm_rf
 
@@ -20,6 +21,17 @@
 logger = logging.getLogger(__name__)
 
 
+def calculate_install_dir(yaml_file, subdir=None):
+    contents = parse(yaml_file, subdir or conda_context.subdir)
+    if contents.get("installer_type") == "sh":
+        return contents["name"]
+    name = contents.get("pkg_name") or contents["name"]
+    location = contents.get("default_location_pkg")
+    if location:
+        return f"{location}/{name}"
+    return name
+
+
 def write_readme(dst, info):
 
     src = join(OSX_DIR, 'readme_header.rtf')
diff --git a/dev/environment.yml b/dev/environment.yml
@@ -0,0 +1,9 @@
+name: constructor-dev
+channels:
+  - conda-forge
+dependencies:
+  - python
+  - conda >=4.6
+  - ruamel.yaml >=0.11.14,<0.18
+  - conda-standalone <23.1.0  # https://github.com/conda/constructor/issues/628
+  - pillow >=3.1  # [osx or win]
diff --git a/dev/extra-requirements-windows.txt b/dev/extra-requirements-windows.txt
@@ -0,0 +1 @@
+conda-forge::nsis>=3.08=*_log_*
diff --git a/examples/extra_envs/construct.yaml b/examples/extra_envs/construct.yaml
@@ -31,3 +31,5 @@ build_outputs:
   - licenses:
       include_text: True
       text_errors: replace
+initialize_by_default: false
+register_python: False
diff --git a/examples/miniforge/construct.yaml b/examples/miniforge/construct.yaml
@@ -20,3 +20,5 @@ specs:
 installer_type: all
 post_install: test_install.sh  # [unix]
 post_install: test_install.bat  # [win]
+initialize_by_default: false
+register_python: False
diff --git a/examples/miniforge/test_install.bat b/examples/miniforge/test_install.bat
@@ -5,4 +5,4 @@ SetLocal EnableDelayedExpansion
 call "%PREFIX%\Scripts\activate.bat
 conda info || exit 1
 conda config --show-sources || exit 1
-conda config --show --json | python -c "import sys, json; info = json.loads(sys.stdin.read()); assert info['channels'] == ['conda-forge'], info"
+conda config --show --json | python -c "import sys, json; info = json.loads(sys.stdin.read()); assert 'conda-forge' in info['channels'], info"
diff --git a/examples/miniforge/test_install.sh b/examples/miniforge/test_install.sh
@@ -16,5 +16,5 @@ echo "+ conda config"
 conda config --show-sources
 
 echo "+ Testing channels"
-conda config --show --json | python -c "import sys, json; info = json.loads(sys.stdin.read()); assert info['channels'] == ['conda-forge'], info"
+conda config --show --json | python -c "import sys, json; info = json.loads(sys.stdin.read()); assert 'conda-forge' in info['channels'], info"
 echo "  OK"
diff --git a/examples/shortcuts/construct.yaml b/examples/shortcuts/construct.yaml
@@ -9,3 +9,6 @@ specs:
   - python
   - conda
   - console_shortcut # [win]
+
+initialize_by_default: false
+register_python: False
diff --git a/examples/signing/construct.yaml b/examples/signing/construct.yaml
@@ -5,7 +5,7 @@ channels:
   - http://repo.anaconda.com/pkgs/main/
 specs:
   - python
-# This certificate is generated an copied over on the spot during CI
+# This certificate is generated and copied over on the spot during CI
 # See scripts/create_self_signed_certificate.ps1 for more info
-signing_certificate: certificate.pfx # [win]
+signing_certificate: certificate.pfx  # [win]
 check_path_spaces: False
diff --git a/news/675-pytest-examples b/news/675-pytest-examples
diff --git a/pyproject.toml b/pyproject.toml
diff --git a/recipe/meta.yaml b/recipe/meta.yaml
diff --git a/scripts/run_examples.py b/scripts/run_examples.py
diff --git a/tests/requirements.txt b/tests/requirements.txt
diff --git a/tests/test_examples.py b/tests/test_examples.py
