From d143caa9f0560f946e452a28be9716bf11e72a79 Mon Sep 17 00:00:00 2001 From: Jirka Date: Tue, 12 Dec 2023 21:42:31 +0100 Subject: [PATCH] ci: freeze requirements used in workflows --- .actions/requirements.txt | 2 +- .github/actions/pip-wheels/action.yml | 2 +- .github/actions/pkg-check/action.yml | 2 +- .github/workflows/_legacy-checkpoints.yml | 2 +- .github/workflows/ci-checkpoints.yml | 1 + .github/workflows/ci-pkg-install.yml | 1 + .github/workflows/ci-tests-app.yml | 2 +- .github/workflows/ci-tests-data.yml | 4 ++-- .github/workflows/ci-tests-fabric.yml | 4 ++-- .github/workflows/ci-tests-pytorch.yml | 4 ++-- .github/workflows/ci-tests-store.yml | 4 ++-- .github/workflows/docs-build.yml | 3 ++- .github/workflows/release-pkg.yml | 1 + requirements/ci.txt | 6 ++++++ 14 files changed, 24 insertions(+), 14 deletions(-) create mode 100644 requirements/ci.txt diff --git a/.actions/requirements.txt b/.actions/requirements.txt index fdd60050f2896..a19661dbae299 100644 --- a/.actions/requirements.txt +++ b/.actions/requirements.txt @@ -1,2 +1,2 @@ -jsonargparse>=4.16.0 +jsonargparse >=4.16.0, <4.28.0 requests diff --git a/.github/actions/pip-wheels/action.yml b/.github/actions/pip-wheels/action.yml index 3defc336436b4..28d6e346b7aa2 100644 --- a/.github/actions/pip-wheels/action.yml +++ b/.github/actions/pip-wheels/action.yml @@ -19,7 +19,7 @@ runs: using: "composite" steps: - name: install dev. env - run: pip install setuptools wheel + run: pip install -r requirements/ci.txt shell: bash - name: Freeze local emv. diff --git a/.github/actions/pkg-check/action.yml b/.github/actions/pkg-check/action.yml index ce8c32796917d..83bdcd308938b 100644 --- a/.github/actions/pkg-check/action.yml +++ b/.github/actions/pkg-check/action.yml @@ -14,7 +14,7 @@ runs: using: "composite" steps: - name: install dev. env - run: pip install "twine==4.0.1" setuptools wheel flake8 + run: pip install -r requirements/ci.txt shell: bash - name: Set PACKAGE_NAME envvar diff --git a/.github/workflows/_legacy-checkpoints.yml b/.github/workflows/_legacy-checkpoints.yml index b0aaf78d06ab1..767d62f9268c3 100644 --- a/.github/workflows/_legacy-checkpoints.yml +++ b/.github/workflows/_legacy-checkpoints.yml @@ -117,7 +117,7 @@ jobs: AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_PUB_SECRET_KEY }} AWS_DEFAULT_REGION: ${{ secrets.AWS_REGION }} run: | - pip install -q awscli + pip install -q -r requirements/ci.txt aws s3 sync $AWS_RUN checkpoints/ s3://pl-public-data/legacy/checkpoints/ zip -r checkpoints.zip checkpoints aws s3 cp $AWS_RUN checkpoints.zip s3://pl-public-data/legacy/ --acl public-read diff --git a/.github/workflows/ci-checkpoints.yml b/.github/workflows/ci-checkpoints.yml index 1f0cde9b6fd45..020d02dba9f3b 100644 --- a/.github/workflows/ci-checkpoints.yml +++ b/.github/workflows/ci-checkpoints.yml @@ -9,6 +9,7 @@ on: paths: - ".github/workflows/ci-checkpoints.yml" - ".github/workflows/_legacy-checkpoints.yml.yml" + - "requirements/ci.txt" jobs: try-legacy-checkpoints: diff --git a/.github/workflows/ci-pkg-install.yml b/.github/workflows/ci-pkg-install.yml index 261c07b0d78cc..993e2643321f5 100644 --- a/.github/workflows/ci-pkg-install.yml +++ b/.github/workflows/ci-pkg-install.yml @@ -9,6 +9,7 @@ on: types: [opened, reopened, ready_for_review, synchronize] # added `ready_for_review` since draft is skipped paths: - ".actions/*" + - "requirements/ci.txt" - ".github/actions/pkg-check/*" - ".github/actions/pkg-install/*" - ".github/workflows/_build-packages.yml" diff --git a/.github/workflows/ci-tests-app.yml b/.github/workflows/ci-tests-app.yml index 79bd396005f46..2b1740a04af89 100644 --- a/.github/workflows/ci-tests-app.yml +++ b/.github/workflows/ci-tests-app.yml @@ -10,6 +10,7 @@ on: paths: - ".actions/*" - ".github/workflows/ci-tests-app.yml" + - "requirements/ci.txt" - "src/lightning/app/**" - "src/lightning_app/*" - "tests/tests_app/**" @@ -91,7 +92,6 @@ jobs: - name: Install package & dependencies timeout-minutes: 20 run: | - python -m pip install -q pip -U pip install -e ".[${EXTRA_PREFIX}dev]" -U --prefer-binary \ --find-links="${TORCH_URL}" --find-links="${PYPI_CACHE_DIR}" pip list diff --git a/.github/workflows/ci-tests-data.yml b/.github/workflows/ci-tests-data.yml index baf37c27944a7..0ad603ca17b78 100644 --- a/.github/workflows/ci-tests-data.yml +++ b/.github/workflows/ci-tests-data.yml @@ -9,6 +9,7 @@ on: types: [opened, reopened, ready_for_review, synchronize] # added `ready_for_review` since draft is skipped paths: - ".actions/*" + - "requirements/ci.txt" - "requirements/data/**" - "src/lightning/data/**" - "src/lightning/__init__.py" @@ -69,7 +70,7 @@ jobs: - name: Adjust PyTorch versions in requirements files if: ${{ matrix.requires != 'oldest' && matrix.release != 'pre' }} run: | - pip install -q wget packaging + pip install -q -r requirements/ci.txt python -m wget https://raw.githubusercontent.com/Lightning-AI/utilities/main/scripts/adjust-torch-versions.py for fpath in `ls requirements/data/*.txt`; do \ python ./adjust-torch-versions.py $fpath ${{ matrix.pytorch-version }}; \ @@ -88,7 +89,6 @@ jobs: - name: Install package & dependencies timeout-minutes: 20 run: | - python -m pip install -q pip -U pip install -e ".[data-dev]" -U --prefer-binary -f ${TORCH_URL} pip list diff --git a/.github/workflows/ci-tests-fabric.yml b/.github/workflows/ci-tests-fabric.yml index 0a142bdea83db..e32ebcf3ba13f 100644 --- a/.github/workflows/ci-tests-fabric.yml +++ b/.github/workflows/ci-tests-fabric.yml @@ -9,6 +9,7 @@ on: types: [opened, reopened, ready_for_review, synchronize] # added `ready_for_review` since draft is skipped paths: - ".actions/*" + - "requirements/ci.txt" - "requirements/fabric/**" - "src/lightning/fabric/**" - "src/lightning_fabric/*" @@ -109,7 +110,7 @@ jobs: - name: Adjust PyTorch versions in requirements files if: ${{ matrix.requires != 'oldest' }} run: | - pip install -q wget packaging + pip install -q -r requirements/ci.txt python -m wget https://raw.githubusercontent.com/Lightning-AI/utilities/main/scripts/adjust-torch-versions.py for fpath in `ls requirements/**/*.txt`; do \ python ./adjust-torch-versions.py $fpath ${{ matrix.pytorch-version }}; \ @@ -136,7 +137,6 @@ jobs: - name: Install package & dependencies timeout-minutes: 20 run: | - #python -m pip install -q pip -U pip install -e ".[${EXTRA_PREFIX}test,${EXTRA_PREFIX}strategies]" -U --prefer-binary \ --find-links="${TORCH_URL}" --find-links="${PYPI_CACHE_DIR}" pip list diff --git a/.github/workflows/ci-tests-pytorch.yml b/.github/workflows/ci-tests-pytorch.yml index 57ec482ee12e1..c3d2aacb8cc37 100644 --- a/.github/workflows/ci-tests-pytorch.yml +++ b/.github/workflows/ci-tests-pytorch.yml @@ -9,6 +9,7 @@ on: types: [opened, reopened, ready_for_review, synchronize] # added `ready_for_review` since draft is skipped paths: - ".actions/*" + - "requirements/ci.txt" - "requirements/pytorch/**" - "src/lightning/pytorch/**" - "src/pytorch_lightning/*" @@ -114,7 +115,7 @@ jobs: - name: Adjust PyTorch versions in requirements files if: ${{ matrix.requires != 'oldest' }} run: | - pip install -q wget packaging + pip install -q -r requirements/ci.txt python -m wget https://raw.githubusercontent.com/Lightning-AI/utilities/main/scripts/adjust-torch-versions.py for fpath in `ls requirements/**/*.txt`; do \ python ./adjust-torch-versions.py $fpath ${{ matrix.pytorch-version }}; \ @@ -142,7 +143,6 @@ jobs: - name: Install package & dependencies timeout-minutes: 20 run: | - #python -m pip install -q pip -U pip install ".[${EXTRA_PREFIX}extra,${EXTRA_PREFIX}test,${EXTRA_PREFIX}strategies]" -U --prefer-binary \ -r requirements/_integrations/accelerators.txt \ --find-links="${TORCH_URL}" --find-links="${PYPI_CACHE_DIR}" diff --git a/.github/workflows/ci-tests-store.yml b/.github/workflows/ci-tests-store.yml index c9bb6d8ebc7fc..5a34a1243b455 100644 --- a/.github/workflows/ci-tests-store.yml +++ b/.github/workflows/ci-tests-store.yml @@ -9,6 +9,7 @@ on: types: [opened, reopened, ready_for_review, synchronize] # added `ready_for_review` since draft is skipped paths: - ".actions/*" + - "requirements/ci.txt" - "requirements/store/**" - "src/lightning/__init__.py" - "src/lightning/__setup__.py" @@ -57,7 +58,7 @@ jobs: - name: Adjust PyTorch versions in requirements files if: ${{ matrix.requires != 'oldest' && matrix.release != 'pre' }} run: | - pip install -q wget packaging + pip install -q -r requirements/ci.txt python -m wget https://raw.githubusercontent.com/Lightning-AI/utilities/main/scripts/adjust-torch-versions.py for fpath in `ls requirements/store/*.txt`; do \ python ./adjust-torch-versions.py $fpath ${{ matrix.pytorch-version }}; \ @@ -66,7 +67,6 @@ jobs: - name: Install package & dependencies timeout-minutes: 20 run: | - python -m pip install -q pip -U pip install -e ".[store,store-test]" -U -f ${TORCH_URL} --prefer-binary pip list diff --git a/.github/workflows/docs-build.yml b/.github/workflows/docs-build.yml index 9d8b0e4d498b5..c407f11e9f69f 100644 --- a/.github/workflows/docs-build.yml +++ b/.github/workflows/docs-build.yml @@ -11,6 +11,7 @@ on: paths: - ".actions/*" - ".github/workflows/docs-build.yml" + - "requirements/ci.txt" - "docs/**" - "_notebooks" - "requirements/**" @@ -74,7 +75,7 @@ jobs: - name: Pull sphinx template run: | - pip install awscli + pip install -q -r requirements/ci.txt aws s3 sync --no-sign-request s3://sphinx-packages/ ${PYPI_LOCAL_DIR} pip install lai-sphinx-theme -U -f ${PYPI_LOCAL_DIR} diff --git a/.github/workflows/release-pkg.yml b/.github/workflows/release-pkg.yml index 5abda9d7e7977..d2d60bbc29f66 100644 --- a/.github/workflows/release-pkg.yml +++ b/.github/workflows/release-pkg.yml @@ -10,6 +10,7 @@ on: branches: [master, "release/*"] types: [opened, reopened, ready_for_review, synchronize] paths: + - "requirements/ci.txt" - ".github/actions/pkg-publish/*" - ".github/workflows/_legacy-checkpoints.yml.yml" - ".github/workflows/_build-packages.yml" diff --git a/requirements/ci.txt b/requirements/ci.txt new file mode 100644 index 0000000000000..f96160c3cb289 --- /dev/null +++ b/requirements/ci.txt @@ -0,0 +1,6 @@ +setuptools +wheel +awscli >=1.30.0, <1.31.0 +twine ==4.0.1 +wget +packaging \ No newline at end of file