diff --git a/.github/actions/setup-pyenv-venv/action.yml b/.github/actions/setup-pyenv-venv/action.yml new file mode 100644 index 0000000000..9d79eb3302 --- /dev/null +++ b/.github/actions/setup-pyenv-venv/action.yml @@ -0,0 +1,59 @@ +name: Setup pyenv virtualenv +description: Ensure a specific Python version and virtualenv via pyenv, optionally installing dependencies. +inputs: + python-version: + description: Python version to install via pyenv. + default: "3.12.6" + venv-name: + description: Name of the pyenv virtualenv to create/use. + default: "pytest-embedded" + requirements: + description: | + New-line separated argument lists passed to `pip install`. Each non-empty line + is appended to `python -m pip install `. + default: "" + requirements-file: + description: | + New-line separated list of requirement file paths to pass with `-r`. + default: "" + pip-extra-args: + description: Additional arguments passed to every `pip install` invocation. + default: "" + pip-extra-index-url: + description: Value exported as PIP_EXTRA_INDEX_URL and appended as --extra-index-url. + default: "" + pyenv-root: + description: Override path to pyenv root (defaults to $HOME/.pyenv). + default: "" +runs: + using: composite + steps: + - name: Prepare pyenv virtualenv + shell: bash + env: + PYENV_PYTHON_VERSION: ${{ inputs.python-version }} + PYENV_VENV_NAME: ${{ inputs.venv-name }} + PYENV_REQUIREMENTS: ${{ inputs.requirements }} + PYENV_REQUIREMENTS_FILE: ${{ inputs.requirements-file }} + PYENV_PIP_EXTRA_ARGS: ${{ inputs.pip-extra-args }} + PYENV_PIP_EXTRA_INDEX_URL: ${{ inputs.pip-extra-index-url }} + PYENV_ROOT_OVERRIDE: ${{ inputs.pyenv-root }} + run: | + bash "${{ github.workspace }}/ci/scripts/setup_pyenv.sh" + - name: Export pyenv environment + shell: bash + env: + PYENV_VENV_NAME: ${{ inputs.venv-name }} + PYENV_ROOT_INPUT: ${{ inputs.pyenv-root }} + PIP_EXTRA_INDEX_URL_INPUT: ${{ inputs.pip-extra-index-url }} + run: | + PYENV_ROOT_VALUE="${PYENV_ROOT_INPUT:-$HOME/.pyenv}" + { + echo "PYENV_ROOT=${PYENV_ROOT_VALUE}" + echo "PYENV_VERSION=${PYENV_VENV_NAME}" + } >> "$GITHUB_ENV" + if [[ -n "${PIP_EXTRA_INDEX_URL_INPUT}" ]]; then + echo "PIP_EXTRA_INDEX_URL=${PIP_EXTRA_INDEX_URL_INPUT}" >> "$GITHUB_ENV" + fi + echo "${PYENV_ROOT_VALUE}/bin" >> "$GITHUB_PATH" + echo "${PYENV_ROOT_VALUE}/shims" >> "$GITHUB_PATH" diff --git a/.github/workflows/asio__build-target-test.yml b/.github/workflows/asio__build-target-test.yml index b7d0e1556a..c7331ca735 100644 --- a/.github/workflows/asio__build-target-test.yml +++ b/.github/workflows/asio__build-target-test.yml @@ -81,18 +81,22 @@ jobs: with: name: examples_app_bin_${{ matrix.idf_target }}_${{ matrix.idf_ver }}_${{ matrix.example }} path: ${{ env.TEST_DIR }}/${{ matrix.example }}/build - - name: Install Python packages - env: - PIP_EXTRA_INDEX_URL: "https://www.piwheels.org/simple" - run: | - sudo apt-get install -y dnsutils - - name: Download Example Test to target ${{ matrix.config }} - run: | - python -m esptool --chip ${{ matrix.idf_target }} write_flash 0x0 ${{ env.TEST_DIR }}/${{ matrix.example }}/build/flash_image.bin + - name: Prepare Python environment + uses: ./.github/actions/setup-pyenv-venv + with: + python-version: "3.12.6" + venv-name: "target-pytest" + pip-extra-index-url: "https://www.piwheels.org/simple" + pip-extra-args: "--prefer-binary --extra-index-url https://dl.espressif.com/pypi/" + requirements: | + cryptography pytest-embedded pytest-embedded-serial-esp pytest-embedded-idf pytest-custom_exit_code esptool + requirements-file: | + $GITHUB_WORKSPACE/ci/requirements.txt - name: Run Example Test ${{ matrix.example }} on target working-directory: ${{ env.TEST_DIR }}/${{ matrix.example }} run: | - python -m pytest --log-cli-level DEBUG --junit-xml=./examples_results_${{ matrix.idf_target }}_${{ matrix.idf_ver }}_${{ matrix.config }}.xml --target=${{ matrix.idf_target }} + python --version + python -m pytest --log-cli-level DEBUG --junit-xml=./examples_results_${{ matrix.idf_target }}_${{ matrix.idf_ver }}_${{ matrix.config }}.xml --target=${{ matrix.idf_target }} - uses: actions/upload-artifact@v4 if: always() with: diff --git a/.github/workflows/lws_build.yml b/.github/workflows/lws_build.yml index a0aac2815f..c3b615e1e0 100644 --- a/.github/workflows/lws_build.yml +++ b/.github/workflows/lws_build.yml @@ -65,24 +65,19 @@ jobs: with: name: lws_target_esp32_${{ matrix.idf_ver }}_${{ matrix.test.app }} path: ${{ env.TEST_DIR }}/ci/ + - name: Prepare Python environment + uses: ./.github/actions/setup-pyenv-venv + with: + python-version: "3.12.6" + venv-name: "target-pytest" + pip-extra-index-url: "https://www.piwheels.org/simple" + - name: Install Python packages + run: | + python -m pip install --only-binary cryptography --extra-index-url https://dl.espressif.com/pypi/ -r $GITHUB_WORKSPACE/ci/requirements.txt - name: Run Example Test on target working-directory: ${{ env.TEST_DIR }} run: | - export PYENV_ROOT="$HOME/.pyenv" - export PATH="$PYENV_ROOT/bin:$PATH" - eval "$(pyenv init --path)" - eval "$(pyenv init -)" - if ! pyenv versions --bare | grep -q '^3\.12\.6$'; then - echo "Installing Python 3.12.6..." - pyenv install -s 3.12.6 - fi - if ! pyenv virtualenvs --bare | grep -q '^myenv$'; then - echo "Creating pyenv virtualenv 'myenv'..." - pyenv virtualenv 3.12.6 myenv - fi - pyenv activate myenv python --version - pip install --only-binary cryptography --extra-index-url https://dl.espressif.com/pypi/ -r $GITHUB_WORKSPACE/ci/requirements.txt unzip ci/artifacts.zip -d ci for dir in `ls -d ci/build_*`; do rm -rf build sdkconfig.defaults diff --git a/.github/workflows/mdns__build-target-test.yml b/.github/workflows/mdns__build-target-test.yml index e6da77ec9b..717f51850b 100644 --- a/.github/workflows/mdns__build-target-test.yml +++ b/.github/workflows/mdns__build-target-test.yml @@ -73,25 +73,21 @@ jobs: PIP_EXTRA_INDEX_URL: "https://www.piwheels.org/simple" run: | sudo apt-get install -y dnsutils + - name: Prepare Python environment + uses: ./.github/actions/setup-pyenv-venv + with: + python-version: "3.12.6" + venv-name: "target-pytest" + pip-extra-index-url: "https://www.piwheels.org/simple" + pip-extra-args: "--prefer-binary --extra-index-url https://dl.espressif.com/pypi/" + requirements: | + cryptography pytest-embedded pytest-embedded-serial-esp pytest-embedded-idf pytest-custom_exit_code esptool + requirements-file: | + $GITHUB_WORKSPACE/ci/requirements.txt - name: Run ${{ matrix.test.app }} application on ${{ matrix.idf_target }} working-directory: components/mdns/${{ matrix.test.path }} run: | - export PYENV_ROOT="$HOME/.pyenv" - export PATH="$PYENV_ROOT/bin:$PATH" - eval "$(pyenv init --path)" - eval "$(pyenv init -)" - if ! pyenv versions --bare | grep -q '^3\.12\.6$'; then - echo "Installing Python 3.12.6..." - pyenv install -s 3.12.6 - fi - if ! pyenv virtualenvs --bare | grep -q '^myenv$'; then - echo "Creating pyenv virtualenv 'myenv'..." - pyenv virtualenv 3.12.6 myenv - fi - pyenv activate myenv python --version - pip install --prefer-binary cryptography pytest-embedded pytest-embedded-serial-esp pytest-embedded-idf pytest-custom_exit_code esptool - pip install --extra-index-url https://dl.espressif.com/pypi/ -r $GITHUB_WORKSPACE/ci/requirements.txt unzip ci/artifacts.zip -d ci for dir in `ls -d ci/build_*`; do rm -rf build sdkconfig.defaults diff --git a/.github/workflows/modem__target-test.yml b/.github/workflows/modem__target-test.yml index e2f8783a6c..30d1284a72 100644 --- a/.github/workflows/modem__target-test.yml +++ b/.github/workflows/modem__target-test.yml @@ -77,13 +77,19 @@ jobs: with: name: modem_target_bin_${{ matrix.idf_target }}_${{ matrix.idf_ver }}_${{ matrix.test.app }} path: ${{ env.TEST_DIR }}/build + - name: Prepare Python environment + uses: ./.github/actions/setup-pyenv-venv + with: + python-version: "3.12.6" + venv-name: "target-pytest" + pip-extra-index-url: "https://dl.espressif.com/pypi/" + pip-extra-args: "--prefer-binary" + requirements: | + cryptography pytest-embedded pytest-embedded-serial-esp pytest-embedded-idf pytest-custom_exit_code esptool + requirements-file: | + $GITHUB_WORKSPACE/ci/requirements.txt - name: Run Example Test on target - env: - PIP_EXTRA_INDEX_URL: "https://dl.espressif.com/pypi/" run: | - python -m venv .venv - source .venv/bin/activate - pip install --prefer-binary cryptography pytest-embedded pytest-embedded-serial-esp pytest-embedded-idf pytest-custom_exit_code esptool - pip install -r $GITHUB_WORKSPACE/ci/requirements.txt + python --version cd ${{ env.TEST_DIR }} python -m pytest --log-cli-level DEBUG --target=${{ matrix.idf_target }} diff --git a/.github/workflows/websocket__build-target-test.yml b/.github/workflows/websocket__build-target-test.yml index 292597d3bc..3a963180dd 100644 --- a/.github/workflows/websocket__build-target-test.yml +++ b/.github/workflows/websocket__build-target-test.yml @@ -65,27 +65,23 @@ jobs: with: name: websocket_bin_esp32_${{ matrix.idf_ver }}_${{ matrix.test.app }} path: ${{ env.TEST_DIR }}/ci/ + - name: Prepare Python environment + uses: ./.github/actions/setup-pyenv-venv + with: + python-version: "3.12.6" + venv-name: "target-pytest" + pip-extra-index-url: "https://www.piwheels.org/simple" + pip-extra-args: "--prefer-binary --extra-index-url https://dl.espressif.com/pypi/" + requirements: | + cryptography pytest-embedded pytest-embedded-serial-esp pytest-embedded-idf pytest-custom_exit_code esptool + requirements-file: | + $GITHUB_WORKSPACE/ci/requirements.txt - name: Run Example Test on target working-directory: ${{ env.TEST_DIR }} env: PIP_EXTRA_INDEX_URL: "https://www.piwheels.org/simple" run: | - export PYENV_ROOT="$HOME/.pyenv" - export PATH="$PYENV_ROOT/bin:$PATH" - eval "$(pyenv init --path)" - eval "$(pyenv init -)" - if ! pyenv versions --bare | grep -q '^3\.12\.6$'; then - echo "Installing Python 3.12.6..." - pyenv install -s 3.12.6 - fi - if ! pyenv virtualenvs --bare | grep -q '^myenv$'; then - echo "Creating pyenv virtualenv 'myenv'..." - pyenv virtualenv 3.12.6 myenv - fi - pyenv activate myenv python --version - pip install --prefer-binary cryptography pytest-embedded pytest-embedded-serial-esp pytest-embedded-idf pytest-custom_exit_code esptool - pip install --extra-index-url https://dl.espressif.com/pypi/ -r $GITHUB_WORKSPACE/ci/requirements.txt unzip ci/artifacts.zip -d ci for dir in `ls -d ci/build_*`; do rm -rf build sdkconfig.defaults diff --git a/ci/scripts/setup_pyenv.sh b/ci/scripts/setup_pyenv.sh new file mode 100755 index 0000000000..a0ad3dbed4 --- /dev/null +++ b/ci/scripts/setup_pyenv.sh @@ -0,0 +1,107 @@ +#!/usr/bin/env bash +# shellcheck shell=bash + +set -euo pipefail + +log() { + echo "[setup-pyenv] $*" +} + +abort() { + log "ERROR: $*" + if [[ "${BASH_SOURCE[0]}" != "${0}" ]]; then + return 1 + else + exit 1 + fi +} + +PYENV_PYTHON_VERSION="${PYENV_PYTHON_VERSION:-3.12.6}" +PYENV_VENV_NAME="${PYENV_VENV_NAME:-pyenv-venv}" +PYENV_ROOT_OVERRIDE="${PYENV_ROOT_OVERRIDE:-}" +PYENV_REQUIREMENTS="${PYENV_REQUIREMENTS:-}" +PYENV_REQUIREMENTS_FILE="${PYENV_REQUIREMENTS_FILE:-}" +PYENV_PIP_EXTRA_ARGS="${PYENV_PIP_EXTRA_ARGS:-}" +PYENV_PIP_EXTRA_INDEX_URL="${PYENV_PIP_EXTRA_INDEX_URL:-}" + +PYENV_ROOT="${PYENV_ROOT_OVERRIDE:-${PYENV_ROOT:-$HOME/.pyenv}}" +export PYENV_ROOT +export PATH="${PYENV_ROOT}/bin:${PATH}" + +if ! command -v pyenv >/dev/null 2>&1; then + abort "pyenv command not found. Ensure pyenv is installed on the runner." +fi + +# Initialize pyenv environments +eval "$(pyenv init --path)" +eval "$(pyenv init -)" +if pyenv commands | grep -q "virtualenv"; then + eval "$(pyenv virtualenv-init -)" +else + abort "pyenv-virtualenv plugin is required but not available." +fi + +if ! pyenv versions --bare | grep -Fxq "${PYENV_PYTHON_VERSION}"; then + log "Installing Python ${PYENV_PYTHON_VERSION} via pyenv..." + pyenv install -s "${PYENV_PYTHON_VERSION}" +else + log "Python ${PYENV_PYTHON_VERSION} already installed." +fi + +if ! pyenv virtualenvs --bare | grep -Fxq "${PYENV_VENV_NAME}"; then + log "Creating virtualenv ${PYENV_VENV_NAME}..." + pyenv virtualenv "${PYENV_PYTHON_VERSION}" "${PYENV_VENV_NAME}" +else + log "Virtualenv ${PYENV_VENV_NAME} already exists." +fi + +export PYENV_VERSION="${PYENV_VENV_NAME}" +pyenv shell "${PYENV_VENV_NAME}" >/dev/null + +python --version + +pip_args=() +if [[ -n "${PYENV_PIP_EXTRA_ARGS}" ]]; then + # shellcheck disable=SC2206 + pip_args+=(${PYENV_PIP_EXTRA_ARGS}) +fi +if [[ -n "${PYENV_PIP_EXTRA_INDEX_URL}" ]]; then + pip_args+=(--extra-index-url "${PYENV_PIP_EXTRA_INDEX_URL}") +fi + +run_pip_install() { + local line="$1" + line="${line%%\#*}" # strip inline comments + line="$(echo -E "${line}" | xargs || true)" + if [[ -z "${line}" ]]; then + return + fi + # shellcheck disable=SC2086 + python -m pip install "${pip_args[@]}" ${line} +} + +if [[ -n "${PYENV_REQUIREMENTS}" ]]; then + while IFS=$'\n' read -r req_line || [[ -n "${req_line}" ]]; do + run_pip_install "${req_line}" + done <<< "${PYENV_REQUIREMENTS}" +fi + +if [[ -n "${PYENV_REQUIREMENTS_FILE}" ]]; then + while IFS=$'\n' read -r req_file || [[ -n "${req_file}" ]]; do + req_file="$(echo -E "${req_file}" | xargs || true)" + [[ -z "${req_file}" ]] && continue + if [[ ! -f "${req_file}" ]]; then + abort "requirements file '${req_file}' not found." + fi + # shellcheck disable=SC2086 + python -m pip install "${pip_args[@]}" -r "${req_file}" + done <<< "${PYENV_REQUIREMENTS_FILE}" +fi + +log "pyenv environment '${PYENV_VENV_NAME}' is ready." + +if [[ "${BASH_SOURCE[0]}" == "${0}" ]]; then + exit 0 +else + return 0 +fi