Switch presubmit CI workflows to use pinned IREE versions. (#774)

Progress on #760.

The idea here is that we will test with only pinned versions in all
workflows that run on `pull_request` and `push` triggers, then we will
create pull requests (ideally via automation like dependabot) that
attempt to update the pinned versions. This will give us confidence that
test regressions are _only_ due to the code changes in the pull request
and not due to a dependency changing. Workflows will also be more
reproducible as the versions they fetch will come from source code and
not an external, time-dependent source.

Author: ScottTodd

.github/workflows/ci-llama-quick-tests.yaml

@@ -55,18 +55,13 @@ jobs:
           python -m pip install --no-compile --upgrade pip
 
           # Note: We install in three steps in order to satisfy requirements
-          # from non default locations first. Installing the PyTorch CPU
-          # wheels saves multiple minutes and a lot of bandwidth on runner setup.
+          # from non default locations first.
           pip install --no-compile -r pytorch-cpu-requirements.txt
+          pip install -r requirements-iree-pinned.txt
+          pip install --no-compile \
+            -r sharktank/requirements-tests.txt \
+            -e sharktank/
 
-          # Install nightly IREE packages.
-          # We could also pin to a known working or stable version.
-          pip install -f https://iree.dev/pip-release-links.html --pre --upgrade \
-            iree-base-compiler \
-            iree-base-runtime \
-            iree-turbine
-
-          pip install --no-compile -r requirements.txt -r sharktank/requirements-tests.txt -e sharktank/
           pip freeze
 
       - name: Run llama 8b f16 decomposed test

.github/workflows/ci-shark-ai.yml

@@ -51,18 +51,12 @@ jobs:
           python -m pip install --no-compile --upgrade pip
 
           # Note: We install in three steps in order to satisfy requirements
-          # from non default locations first. Installing the PyTorch CPU
-          # wheels saves multiple minutes and a lot of bandwidth on runner setup.
+          # from non default locations first.
           pip install --no-compile -r pytorch-cpu-requirements.txt
-
-          # Install nightly IREE packages.
-          # We could also pin to a known working or stable version.
-          pip install -f https://iree.dev/pip-release-links.html --pre --upgrade \
-            iree-base-compiler \
-            iree-base-runtime \
-            iree-turbine
-
-          pip install --no-compile -r requirements.txt -e sharktank/ shortfin/
+          pip install -r requirements-iree-pinned.txt
+          pip install --no-compile \
+            -r requirements.txt \
+            -e sharktank/ shortfin/
 
           pip freeze
 

.github/workflows/ci-sharktank.yml

@@ -72,18 +72,15 @@ jobs:
           python -m pip install --no-compile --upgrade pip
 
           # Note: We install in three steps in order to satisfy requirements
-          # from non default locations first. Installing the PyTorch CPU
-          # wheels saves multiple minutes and a lot of bandwidth on runner setup.
-          pip install --no-compile --index-url https://download.pytorch.org/whl/cpu torch==${{matrix.torch-version}}+cpu
+          # from non default locations first.
+          pip install --no-compile \
+            --index-url https://download.pytorch.org/whl/cpu torch==${{matrix.torch-version}}+cpu
+          pip install -r requirements-iree-pinned.txt
+          pip install --no-compile \
+            -r sharktank/requirements-tests.txt \
+            -e sharktank/
 
-          # Install nightly IREE packages.
-          # We could also pin to a known working or stable version.
-          pip install -f https://iree.dev/pip-release-links.html --pre \
-            iree-base-compiler \
-            iree-base-runtime \
-            iree-turbine
-
-          pip install --no-compile -r requirements.txt -r sharktank/requirements-tests.txt -e sharktank/
+          pip freeze
 
       - name: Run sharktank tests
         if: ${{ !cancelled() }}
@@ -121,19 +118,16 @@ jobs:
         run: |
           source ${VENV_DIR}/bin/activate
           python -m pip install --no-compile --upgrade pip
+
           # Note: We install in three steps in order to satisfy requirements
-          # from non default locations first. Installing the PyTorch CPU
-          # wheels saves multiple minutes and a lot of bandwidth on runner setup.
+          # from non default locations first.
           pip install --no-compile -r pytorch-cpu-requirements.txt
+          pip install -r requirements-iree-pinned.txt
+          pip install --no-compile \
+            -r sharktank/requirements-tests.txt \
+            -e sharktank/
 
-          # Install nightly IREE packages.
-          # We could also pin to a known working or stable version.
-          pip install -f https://iree.dev/pip-release-links.html --pre --upgrade \
-            iree-base-compiler \
-            iree-base-runtime \
-            iree-turbine
-
-          pip install --no-compile -r requirements.txt -r sharktank/requirements-tests.txt -e sharktank/
+          pip freeze
 
       - name: Run tests
       # TODO: unify with-*-data flags into a single flag and make it possible to run
@@ -180,18 +174,14 @@ jobs:
           python -m pip install --no-compile --upgrade pip
 
           # Note: We install in three steps in order to satisfy requirements
-          # from non default locations first. Installing the PyTorch CPU
-          # wheels saves multiple minutes and a lot of bandwidth on runner setup.
+          # from non default locations first.
           pip install --no-compile -r pytorch-cpu-requirements.txt
+          pip install -r requirements-iree-pinned.txt
+          pip install --no-compile \
+            -r sharktank/requirements-tests.txt \
+            -e sharktank/
 
-          # Install nightly IREE packages.
-          # We could also pin to a known working or stable version.
-          pip install -f https://iree.dev/pip-release-links.html --pre \
-            iree-base-compiler \
-            iree-base-runtime \
-            iree-turbine
-
-          pip install --no-compile -r requirements.txt -r sharktank/requirements-tests.txt -e sharktank/
+          pip freeze
 
       - name: Run punet tests
         run: |

.github/workflows/ci_eval_short.yaml

@@ -52,18 +52,13 @@ jobs:
           python -m pip install --no-compile --upgrade pip
 
           # Note: We install in three steps in order to satisfy requirements
-          # from non default locations first. Installing the PyTorch CPU
-          # wheels saves multiple minutes and a lot of bandwidth on runner setup.
+          # from non default locations first.
           pip install --no-compile -r pytorch-cpu-requirements.txt
+          pip install -r requirements-iree-pinned.txt
+          pip install --no-compile \
+            -r sharktank/requirements-tests.txt \
+            -e sharktank/
 
-          # Install nightly IREE packages.
-          # We could also pin to a known working or stable version.
-          pip install -f https://iree.dev/pip-release-links.html --pre --upgrade \
-            iree-base-compiler \
-            iree-base-runtime \
-            iree-turbine
-
-          pip install --no-compile -r requirements.txt -r sharktank/requirements-tests.txt -e sharktank/
           pip freeze
 
       - name: Run perplexity test with vmfb

requirements-iree-pinned.txt

@@ -3,8 +3,6 @@
 # Keep these versions synced with SHORTFIN_IREE_GIT_TAG in shortfin/CMakeLists.txt
 --pre
 --find-links https://iree.dev/pip-release-links.html
-iree-base-compiler==3.1.0rc20241206
-iree-base-runtime==3.1.0rc20241206
-
-# TODO(#760): include iree-turbine in this requirements file too?
-# iree-turbine==3.1.0rc20241206
+iree-base-compiler==3.1.0rc20250107
+iree-base-runtime==3.1.0rc20250107
+iree-turbine==3.1.0rc20250107

requirements-iree-unpinned.txt

@@ -4,6 +4,4 @@
 --find-links https://iree.dev/pip-release-links.html
 iree-base-compiler
 iree-base-runtime
-
-# TODO(#760): include iree-turbine in this requirements file too?
-# iree-turbine
+iree-turbine

sharktank/requirements-tests.txt

@@ -1,5 +1,6 @@
 datasets==3.0.0
+diffusers
 parameterized
 pytest==8.0.0
 pytest-html
-diffusers
+pytest-xdist==3.5.0