Migrate back to mypy and enable typechecking in CI (#2595)

Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>

Author: VeckoTheGecko

.github/workflows/ci.yml

@@ -30,7 +30,7 @@ jobs:
         with:
           fetch-depth: 2
           persist-credentials: false
-      - uses: xarray-contrib/ci-trigger@74ddc46fc6ca7509549ac7b660d7a185948c1e74 # v1
+      - uses: xarray-contrib/ci-trigger@74ddc46fc6ca7509549ac7b660d7a185948c1e74 # v1.2
         id: check-skip
         with:
           keyword: "[skip-ci]"
@@ -54,7 +54,7 @@ jobs:
           persist-credentials: false
       - uses: Parcels-code/pixi-lock/create-and-cache@38495788b79a5ff26009aecc15daa9a8310b8832 # v0.1.0
         id: pixi-lock
-      - uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7
+      - uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0
         with:
           name: pixi-lock
           path: pixi.lock
@@ -122,7 +122,7 @@ jobs:
           flags: unit-tests
       - name: Upload test results
         if: ${{ always() }} # Always run this step, even if tests fail
-        uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7
+        uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0
         with:
           name: Unittest report ${{ matrix.os }}-${{ matrix.pixi-environment }}
           path: ${{ env.COVERAGE_REPORT }}
@@ -169,7 +169,7 @@ jobs:
           flags: integration-tests
       - name: Upload test results
         if: ${{ always() }} # Always run this step, even if tests fail
-        uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7
+        uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0
         with:
           name: Integration test report ${{ matrix.os }}-${{ matrix.pixi-environment }}
           path: ${{ env.COVERAGE_REPORT }}
@@ -182,7 +182,7 @@ jobs:
       - typechecking
     steps:
       - name: Merge Artifacts
-        uses: actions/upload-artifact/merge@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7
+        uses: actions/upload-artifact/merge@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0
         with:
           name: Testing reports
           pattern: "* report *"
@@ -208,8 +208,14 @@ jobs:
           cache: true
           cache-write: ${{ github.event_name == 'push' && github.ref_name == 'main' }}
       - name: Typechecking
-        run: | # TODO: Remove `|| true` once typechecking is stable
-          pixi run typing --output-format github || true
+        run: |
+          pixi run typing --non-interactive --html-report mypy-report
+      - name: Upload test results
+        if: ${{ always() }} # Upload even on mypy error
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
+        with:
+          name: Mypy report
+          path: mypy-report
   build-and-upload-nightly-parcels: # for alpha testing
     needs: [cache-pixi-lock]
     permissions:

.github/workflows/pypi-release.yml

@@ -47,7 +47,7 @@ jobs:
           else
             echo "✅ Looks good"
           fi
-      - uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7
+      - uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0
         with:
           name: releases
           path: dist

docs/development/index.md

@@ -132,8 +132,8 @@ See below for more Pixi commands relevant to development.
 
 **Code quality**
 
-- `pixi run lint` - Run [pre-commit](https://pre-commit.com/) hooks on all files (includes formatting, linting, and other code quality checks)
-- `pixi run typing` - Run [ty](https://docs.astral.sh/ty/) type checking on the codebase
+- `pixi run lint` - Run pre-commit hooks on all files (includes formatting, linting, and other code quality checks)
+- `pixi run typing` - Run mypy type checking on the codebase
 
 **Different environments**
 

pixi.toml

@@ -141,10 +141,12 @@ numpydoc = "*"
 numpydoc-lint = { cmd = "python tools/numpydoc-public-api.py", description = "Lint public API docstrings with numpydoc." }
 
 [feature.typing.dependencies]
-ty = "*"
+mypy = "*"
+lxml = "*" # in CI
+types-tqdm = "*"
 
 [feature.typing.tasks]
-typing = { cmd = "ty check", description = "Run static type checking with ty." }
+typing = { cmd = "mypy src/parcels --install-types", description = "Run static type checking with mypy." }
 
 
 [environments]

pyproject.toml

@@ -159,9 +159,53 @@ convention = "numpy"
 [tool.ruff.lint.isort]
 known-first-party = ["parcels"]
 
-[tool.ty.src]
-include = ["./src/"]
+[tool.mypy]
+files = ['src']
+check_untyped_defs = true
+show_error_context = true
+warn_redundant_casts = true
+warn_unused_configs = true
+warn_unused_ignores = true
+
 exclude = [
-  "./src/parcels/interpolators/", # ignore for now
-  "./src/parcels/kernels/_advection.py",
+  # Temporarily exclude files. As we improve typechecking across the codebase, remove these
+  "src/parcels/_core/utils/time.py",
+  "src/parcels/_core/utils/interpolation.py",
+  "src/parcels/_core/utils/unstructured.py",
+  "src/parcels/_core/particle.py",
+  "src/parcels/_core/field.py",
+  "src/parcels/_core/kernel.py",
+  "src/parcels/_core/particleset.py",
+  "src/parcels/_core/xgrid.py",
+  "src/parcels/_core/warnings.py",
+  "src/parcels/_core/constants.py",
+  "src/parcels/_core/utils/sgrid.py",
+  "src/parcels/_core/utils/string.py",
+  "src/parcels/_core/particlefile.py",
+  "src/parcels/_core/uxgrid.py",
+  "src/parcels/_core/spatialhash.py",
+  "src/parcels/_core/statuscodes.py",
+  "src/parcels/_core/particlesetview.py",
+  "src/parcels/_core/fieldset.py",
+  "src/parcels/_core/index_search.py",
+  "src/parcels/interpolators/_xinterpolators.py",
+  "src/parcels/interpolators/__init__.py",
+  "src/parcels/interpolators/_uxinterpolators.py",
+  "src/parcels/kernels/_sigmagrids.py",
+  "src/parcels/kernels/_advectiondiffusion.py",
+  "src/parcels/kernels/__init__.py",
+  "src/parcels/kernels/_advection.py",
+]
+
+[[tool.mypy.overrides]]
+module = [
+  "scipy.spatial",
+  "zarr",
+  "zarr.storage",
+  "cftime",
+  "netCDF4",
+  "pooch",
+  "xgcm",
+  "uxarray",
 ]
+ignore_missing_imports = true

src/parcels/_core/basegrid.py

@@ -24,6 +24,8 @@ class GridType(IntEnum):
 class BaseGrid(ABC):
     """Base class for parcels.XGrid and parcels.UxGrid defining common methods and properties"""
 
+    _spatialhash: SpatialHash | None
+
     @abstractmethod
     def search(self, z: float, y: float, x: float, ei=None) -> dict[str, tuple[int, float | np.ndarray]]:
         """

src/parcels/_core/utils/sgrid.py

@@ -694,7 +694,7 @@ def _grid3d_to_ascii(grid: Grid3DMetadata) -> str:
     return "\n".join(lines)
 
 
-def _attach_sgrid_metadata(ds, grid: Grid2DMetadata | Grid3DMetadata):
+def _attach_sgrid_metadata(ds: xr.Dataset, grid: Grid2DMetadata | Grid3DMetadata):
     """Copies the dataset and attaches the SGRID metadata in 'grid' variable. Modifies 'conventions' attribute."""
     ds = ds.copy()
     ds["grid"] = (

src/parcels/_datasets/remote.py

@@ -18,10 +18,15 @@
 
 _DATA_URL = f"https://github.com/Parcels-code/parcels-data/raw/{_DATA_REPO_TAG}"
 
-_DATA_HOME = os.environ.get("PARCELS_EXAMPLE_DATA")
-if _DATA_HOME is None:
-    _DATA_HOME = pooch.os_cache("parcels")
-_DATA_HOME = Path(_DATA_HOME)
+
+def _get_data_home() -> Path:
+    data_home = os.environ.get("PARCELS_EXAMPLE_DATA")
+    if data_home is None:
+        data_home = pooch.os_cache("parcels")
+    return Path(data_home)
+
+
+_DATA_HOME = _get_data_home()
 
 # See instructions at https://github.com/Parcels-code/parcels-data for adding new datasets
 _ODIE_REGISTRY_FILES: list[str] = (
@@ -193,7 +198,7 @@ class _Purpose(enum.Enum):
 
 # The first here is a human readable key used to open datasets, with an object to open the datasets
 # fmt: off
-_DATASET_KEYS_AND_CONFIGS: dict[str, tuple[_V3Dataset, _Purpose]] = dict([
+_DATASET_KEYS_AND_CONFIGS: dict[str, tuple[_ParcelsDataset, _Purpose]] = dict([
     ("MovingEddies_data/P", (_V3Dataset(_ODIE,"data/MovingEddies_data/moving_eddiesP.nc"), _Purpose.TUTORIAL)),
     ("MovingEddies_data/U", (_V3Dataset(_ODIE,"data/MovingEddies_data/moving_eddiesU.nc"), _Purpose.TUTORIAL)),
     ("MovingEddies_data/V", (_V3Dataset(_ODIE,"data/MovingEddies_data/moving_eddiesV.nc"), _Purpose.TUTORIAL)),

src/parcels/_datasets/utils.py

@@ -231,10 +231,10 @@ def replace_arrays_with_zeros(
     return ds
 
 
-def _fill_with_dummy_data(d: dict[str, dict]):
+def _fill_with_dummy_data(d: dict[str, Any]):
     assert isinstance(d, dict)
     if "dtype" in d:
-        d["data"] = np.zeros(d["shape"], dtype=d["dtype"])  # type:ignore[no-matching-overload] # loading from unsanitized data
+        d["data"] = np.zeros(d["shape"], dtype=d["dtype"])
         del d["dtype"]
         del d["shape"]