From 63cc4b2b80eecc8c8e3825f0a968f2ed5d55bfba Mon Sep 17 00:00:00 2001 From: James Lamb Date: Wed, 17 Jul 2024 14:49:19 -0700 Subject: [PATCH 1/5] manage dependencies with rapids-dependency-file-generator --- .github/workflows/github-actions.yml | 32 +++++++++++------ .gitignore | 1 + .pre-commit-config.yaml | 5 +++ conda/environments/all_cuda-118.yaml | 28 +++++++++++++++ dependencies.yaml | 53 ++++++++++++++++++++++++++++ 5 files changed, 109 insertions(+), 10 deletions(-) create mode 100644 conda/environments/all_cuda-118.yaml create mode 100644 dependencies.yaml diff --git a/.github/workflows/github-actions.yml b/.github/workflows/github-actions.yml index bf9f62f1..3746a5a2 100644 --- a/.github/workflows/github-actions.yml +++ b/.github/workflows/github-actions.yml @@ -11,7 +11,15 @@ on: - "branch-*" - "main" jobs: + pre-commit: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + - uses: pre-commit/action@v3.0.1 + build-test: + needs: [pre-commit] defaults: run: shell: bash -el {0} @@ -25,8 +33,20 @@ jobs: steps: - name: Install legate/cunumeric run: | - mamba install -y -c legate -c conda-forge legate-core=24.06 cunumeric=24.06 build cmake scikit-build scikit-learn hypothesis 'pytest<8' notebook 'numpy<2' mypy openblas - pip install matplotlib seaborn xgboost + conda install --yes -c conda-forge \ + 'rapids-dependency-file-generator>=1.14.0' + + rapids-dependency-file-generator \ + --output conda \ + --file-key all \ + --matrix "cuda=${CUDA_VERSION};arch=$(arch)" | tee /tmp/env.yaml + + conda env create \ + --yes \ + --name legate-boost-dev \ + --file /tmp/env.yaml + + source activate legate-boost-dev - name: Checkout legateboost uses: actions/checkout@v4 with: @@ -52,7 +72,6 @@ jobs: - name: Build legateboost docs working-directory: docs run: | - pip install sphinx pydata-sphinx-theme myst-parser # invoke sphinx build using legate legate --run-mode=python --module sphinx.cmd.build source build -b html - uses: actions/upload-pages-artifact@v1 @@ -81,10 +100,3 @@ jobs: - name: Deploy to GitHub Pages id: deployment uses: actions/deploy-pages@v2 - - pre-commit: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 - - uses: pre-commit/action@v3.0.1 diff --git a/.gitignore b/.gitignore index 2f071c1d..dad547e4 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ build +dist/ _skbuild *.pyc *.egg-info diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 5ec0d198..7fe2d855 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -28,5 +28,10 @@ repos: - id: clang-format files: \.(cu|cuh|h|cc|inl)$ types_or: [] + - repo: https://github.com/rapidsai/dependency-file-generator + rev: v1.14.0 + hooks: + - id: rapids-dependency-file-generator + args: ["--clean"] default_language_version: python: python3 diff --git a/conda/environments/all_cuda-118.yaml b/conda/environments/all_cuda-118.yaml new file mode 100644 index 00000000..c5f07951 --- /dev/null +++ b/conda/environments/all_cuda-118.yaml @@ -0,0 +1,28 @@ +# This file is generated by `rapids-dependency-file-generator`. +# To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. +channels: +- legate +- conda-forge +dependencies: +- cmake>=3.24.0,!=3.30.0 +- cunumeric=24.06.* +- hypothesis +- legate-core=24.06.* +- matplotlib +- mypy +- myst-parser +- nbconvert +- notebook +- numpy +- openblas +- pydata-sphinx-theme +- pytest<8 +- python-build +- scikit-build>=0.18.0 +- scikit-learn +- seaborn +- setuptools>=70.0 +- sphinx +- typing-extensions>=4.0 +- xgboost +name: all_cuda-118 diff --git a/dependencies.yaml b/dependencies.yaml new file mode 100644 index 00000000..36469d87 --- /dev/null +++ b/dependencies.yaml @@ -0,0 +1,53 @@ +# Dependency list for https://github.com/rapidsai/dependency-file-generator +files: + all: + output: conda + matrix: + cuda: ["11.8"] + includes: + - build + - docs + - run + - test +channels: + - legate + - conda-forge +dependencies: + build: + common: + - output_types: [conda] + packages: + - cmake>=3.24.0,!=3.30.0 + - &legate_core legate-core=24.06.* + - openblas + - python-build + - scikit-build>=0.18.0 + - setuptools>=70.0 + docs: + common: + - output_types: [conda] + packages: + - sphinx + - pydata-sphinx-theme + - myst-parser + run: + common: + - output_types: [conda] + packages: + - cunumeric=24.06.* + - *legate_core + - numpy + - scikit-learn + - typing-extensions>=4.0 + test: + common: + - output_types: [conda] + packages: + - hypothesis + - matplotlib + - mypy + - nbconvert + - notebook + - pytest<8 + - seaborn + - xgboost From 2cf8a5955b85e16dfa106d1cdf43845987ee00f6 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Thu, 18 Jul 2024 07:34:57 -0700 Subject: [PATCH 2/5] get dfg from rapidsai channel --- .github/workflows/github-actions.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/github-actions.yml b/.github/workflows/github-actions.yml index 3746a5a2..09875d0e 100644 --- a/.github/workflows/github-actions.yml +++ b/.github/workflows/github-actions.yml @@ -33,7 +33,7 @@ jobs: steps: - name: Install legate/cunumeric run: | - conda install --yes -c conda-forge \ + conda install --yes -c rapidsai \ 'rapids-dependency-file-generator>=1.14.0' rapids-dependency-file-generator \ From adc19da41675a6009ff7058b9f9a051526d5733d Mon Sep 17 00:00:00 2001 From: James Lamb Date: Thu, 18 Jul 2024 07:48:01 -0700 Subject: [PATCH 3/5] need to check out source before running dfg --- .github/workflows/github-actions.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/github-actions.yml b/.github/workflows/github-actions.yml index 09875d0e..3b254961 100644 --- a/.github/workflows/github-actions.yml +++ b/.github/workflows/github-actions.yml @@ -31,6 +31,10 @@ jobs: env: NVIDIA_VISIBLE_DEVICES: ${{ env.NVIDIA_VISIBLE_DEVICES }} # GPU jobs must set this container env variable steps: + - name: Checkout legateboost + uses: actions/checkout@v4 + with: + lfs: true - name: Install legate/cunumeric run: | conda install --yes -c rapidsai \ @@ -47,10 +51,6 @@ jobs: --file /tmp/env.yaml source activate legate-boost-dev - - name: Checkout legateboost - uses: actions/checkout@v4 - with: - lfs: true - name: Type check legateboost run: | mypy ./legateboost --config-file ./setup.cfg --exclude=legateboost/test --exclude=install_info From d4a7d6e934d1bc22d483812581809cc7cde691ab Mon Sep 17 00:00:00 2001 From: James Lamb Date: Thu, 18 Jul 2024 08:03:43 -0700 Subject: [PATCH 4/5] env create does not take --yes, and this project uses 'mamba' --- .github/workflows/github-actions.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/github-actions.yml b/.github/workflows/github-actions.yml index 3b254961..f0900f11 100644 --- a/.github/workflows/github-actions.yml +++ b/.github/workflows/github-actions.yml @@ -37,7 +37,7 @@ jobs: lfs: true - name: Install legate/cunumeric run: | - conda install --yes -c rapidsai \ + mamba install --yes -c rapidsai \ 'rapids-dependency-file-generator>=1.14.0' rapids-dependency-file-generator \ @@ -45,8 +45,7 @@ jobs: --file-key all \ --matrix "cuda=${CUDA_VERSION};arch=$(arch)" | tee /tmp/env.yaml - conda env create \ - --yes \ + mamba env create \ --name legate-boost-dev \ --file /tmp/env.yaml From b0ecfc773b9bc72a491e0c5b890b6591cb67724d Mon Sep 17 00:00:00 2001 From: James Lamb Date: Thu, 18 Jul 2024 08:49:38 -0700 Subject: [PATCH 5/5] use 'mamba env update' instead --- .github/workflows/github-actions.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/github-actions.yml b/.github/workflows/github-actions.yml index f0900f11..aa490a62 100644 --- a/.github/workflows/github-actions.yml +++ b/.github/workflows/github-actions.yml @@ -45,11 +45,11 @@ jobs: --file-key all \ --matrix "cuda=${CUDA_VERSION};arch=$(arch)" | tee /tmp/env.yaml - mamba env create \ - --name legate-boost-dev \ + # update the current environment (instead of creating a new one), as that + # persists across all steps + mamba env update \ + --name base \ --file /tmp/env.yaml - - source activate legate-boost-dev - name: Type check legateboost run: | mypy ./legateboost --config-file ./setup.cfg --exclude=legateboost/test --exclude=install_info