diff --git a/.github/workflows/ci-night.yml b/.github/workflows/ci-night.yml new file mode 100644 index 000000000..1586f1825 --- /dev/null +++ b/.github/workflows/ci-night.yml @@ -0,0 +1,311 @@ +name: CI Night - Daily Quality Gate + +on: + schedule: + - cron: "0 3 * * *" + workflow_dispatch: + +# Use bash explicitly so micromamba environment is active for all run steps. +defaults: + run: + shell: bash -el {0} + +permissions: + actions: read + checks: write + contents: read + +jobs: + quality: + name: Daily Test, Coverage and Lint + runs-on: self-hosted + timeout-minutes: 20 + + steps: + - name: Checkout develop branch + uses: actions/checkout@v4 + with: + ref: ${{ github.event_name == 'workflow_dispatch' && github.ref_name || 'develop' }} + + - name: Get current date + id: date + run: echo "date=$(date +%Y-%m-%d)" >> "${GITHUB_OUTPUT}" + + - name: Create environment with micromamba + uses: mamba-org/setup-micromamba@v1 + with: + environment-name: petals_env + environment-file: requirements/env_climada.yml + create-args: >- + python=3.12 + make + cache-environment-key: env-${{ github.ref_name }}-${{ steps.date.outputs.date }} + + - name: Update local conda environment + run: micromamba update -n climada_env -f requirements/env_climada.yml + + - name: Install package and test tools + run: python -m pip install -e . pytest pytest-cov pylint + + - name: Run pytest on climada_petals + id: pytest + run: | + set +e + mkdir -p tests_xml + python -m pytest \ + --junitxml=tests_xml/tests.xml \ + --cov \ + --cov-config=.coveragerc \ + --cov-report xml:coverage.xml \ + --cov-report term:skip-covered \ + climada_petals/ + echo "exit_code=$?" >> "$GITHUB_OUTPUT" + exit 0 + + - name: Run pylint + id: pylint + run: | + set +e + python -m pylint --output-format=json climada_petals > pylint.json + echo "exit_code=$?" >> "$GITHUB_OUTPUT" + exit 0 + + - name: Build current quality metrics + id: current_metrics + env: + PYTEST_EXIT_CODE: ${{ steps.pytest.outputs.exit_code }} + run: | + python - <<'PY' + import json + import os + import xml.etree.ElementTree as ET + from pathlib import Path + + coverage_pct = 0.0 + coverage_file = Path("coverage.xml") + if coverage_file.exists(): + root = ET.parse(coverage_file).getroot() + line_rate = root.attrib.get("line-rate") + if line_rate is not None: + coverage_pct = round(float(line_rate) * 100, 4) + + lint_errors = 0 + pylint_file = Path("pylint.json") + if pylint_file.exists() and pylint_file.stat().st_size > 0: + try: + issues = json.loads(pylint_file.read_text(encoding="utf-8")) + lint_errors = sum(1 for issue in issues if issue.get("type") in {"error", "fatal"}) + except json.JSONDecodeError: + lint_errors = 0 + + tests_total = 0 + tests_failures = 0 + tests_errors = 0 + junit_file = Path("tests_xml/tests.xml") + if junit_file.exists(): + root = ET.parse(junit_file).getroot() + if root.tag == "testsuite": + tests_total = int(root.attrib.get("tests", 0)) + tests_failures = int(root.attrib.get("failures", 0)) + tests_errors = int(root.attrib.get("errors", 0)) + elif root.tag == "testsuites": + tests_total = int(root.attrib.get("tests", 0)) + tests_failures = int(root.attrib.get("failures", 0)) + tests_errors = int(root.attrib.get("errors", 0)) + + pytest_exit = int(os.environ.get("PYTEST_EXIT_CODE", "1")) + + metrics = { + "coverage_pct": coverage_pct, + "lint_errors": lint_errors, + "tests_total": tests_total, + "tests_failures": tests_failures, + "tests_errors": tests_errors, + "pytest_exit_code": pytest_exit, + } + Path("metrics.json").write_text(json.dumps(metrics, indent=2), encoding="utf-8") + + with open(os.environ["GITHUB_STEP_SUMMARY"], "a", encoding="utf-8") as fh: + fh.write("## Current run metrics\n") + fh.write("| Metric | Value |\n") + fh.write("|---|---:|\n") + fh.write(f"| Coverage (%) | {coverage_pct:.4f} |\n") + fh.write(f"| Lint errors (pylint error+fatal) | {lint_errors} |\n") + fh.write(f"| Tests total | {tests_total} |\n") + fh.write(f"| Test failures | {tests_failures} |\n") + fh.write(f"| Test errors | {tests_errors} |\n") + fh.write(f"| Pytest exit code | {pytest_exit} |\n") + PY + + - name: Download previous successful daily metrics + id: previous_metrics + env: + GH_TOKEN: ${{ github.token }} + GH_REPO: ${{ github.repository }} + WORKFLOW_FILE: daily-quality.yml + CURRENT_RUN_ID: ${{ github.run_id }} + TARGET_BRANCH: ${{ github.event_name == 'workflow_dispatch' && github.ref_name || 'develop' }} + run: | + python - <<'PY' + import io + import json + import os + import urllib.request + import zipfile + from pathlib import Path + + token = os.environ["GH_TOKEN"] + repo = os.environ["GH_REPO"] + workflow_file = os.environ["WORKFLOW_FILE"] + current_run_id = int(os.environ["CURRENT_RUN_ID"]) + target_branch = os.environ.get("TARGET_BRANCH", "develop") + + def gh_get(url): + req = urllib.request.Request( + url, + headers={ + "Authorization": f"Bearer {token}", + "Accept": "application/vnd.github+json", + "X-GitHub-Api-Version": "2022-11-28", + }, + ) + with urllib.request.urlopen(req) as resp: + return json.loads(resp.read().decode("utf-8")) + + runs_url = ( + f"https://api.github.com/repos/{repo}/actions/workflows/{workflow_file}/runs" + f"?branch={target_branch}&status=success&per_page=20" + ) + runs_payload = gh_get(runs_url) + runs = runs_payload.get("workflow_runs", []) + + previous_run = None + for run in runs: + run_id = run.get("id") + if run_id is not None and int(run_id) < current_run_id: + previous_run = run + break + + out_path = Path(os.environ["GITHUB_OUTPUT"]) + if previous_run is None: + with out_path.open("a", encoding="utf-8") as fh: + fh.write("found=false\n") + with open(os.environ["GITHUB_STEP_SUMMARY"], "a", encoding="utf-8") as fh: + fh.write( + f"\nNo previous successful run found on branch '{target_branch}'. Regression checks are skipped for this run.\n" + ) + raise SystemExit(0) + + run_id = previous_run["id"] + artifacts_url = f"https://api.github.com/repos/{repo}/actions/runs/{run_id}/artifacts?per_page=100" + artifacts_payload = gh_get(artifacts_url) + artifacts = artifacts_payload.get("artifacts", []) + match = next((a for a in artifacts if a.get("name") == "daily-quality-metrics"), None) + + if match is None: + with out_path.open("a", encoding="utf-8") as fh: + fh.write("found=false\n") + with open(os.environ["GITHUB_STEP_SUMMARY"], "a", encoding="utf-8") as fh: + fh.write( + f"\nPrevious run #{run_id} exists but has no daily-quality-metrics artifact. Regression checks are skipped.\n" + ) + raise SystemExit(0) + + archive_url = match["archive_download_url"] + req = urllib.request.Request( + archive_url, + headers={ + "Authorization": f"Bearer {token}", + "Accept": "application/vnd.github+json", + "X-GitHub-Api-Version": "2022-11-28", + }, + ) + with urllib.request.urlopen(req) as resp: + zip_bytes = resp.read() + + with zipfile.ZipFile(io.BytesIO(zip_bytes)) as zf: + with zf.open("metrics.json") as metrics_file: + previous_metrics = json.loads(metrics_file.read().decode("utf-8")) + + Path("previous_metrics.json").write_text(json.dumps(previous_metrics, indent=2), encoding="utf-8") + with out_path.open("a", encoding="utf-8") as fh: + fh.write("found=true\n") + fh.write(f"run_id={run_id}\n") + + with open(os.environ["GITHUB_STEP_SUMMARY"], "a", encoding="utf-8") as fh: + fh.write( + f"\nComparing against previous successful run on branch '{target_branch}': #{run_id}.\n" + ) + PY + + - name: Compare metrics and enforce quality gates + env: + PREVIOUS_FOUND: ${{ steps.previous_metrics.outputs.found }} + run: | + python - <<'PY' + import json + import os + import sys + from pathlib import Path + + current = json.loads(Path("metrics.json").read_text(encoding="utf-8")) + previous_found = os.environ.get("PREVIOUS_FOUND", "false").lower() == "true" + previous = ( + json.loads(Path("previous_metrics.json").read_text(encoding="utf-8")) + if previous_found and Path("previous_metrics.json").exists() + else None + ) + + test_failed = current.get("pytest_exit_code", 1) != 0 + coverage_decreased = False + lint_errors_increased = False + + if previous is not None: + coverage_decreased = current.get("coverage_pct", 0.0) < previous.get("coverage_pct", 0.0) + lint_errors_increased = current.get("lint_errors", 0) > previous.get("lint_errors", 0) + + with open(os.environ["GITHUB_STEP_SUMMARY"], "a", encoding="utf-8") as fh: + fh.write("\n## Quality gate result\n") + fh.write("| Gate | Status |\n") + fh.write("|---|---|\n") + fh.write(f"| Test failures | {'FAIL' if test_failed else 'PASS'} |\n") + fh.write(f"| Coverage decrease vs previous daily run | {'FAIL' if coverage_decreased else 'PASS'} |\n") + fh.write(f"| Lint error increase vs previous daily run | {'FAIL' if lint_errors_increased else 'PASS'} |\n") + + if test_failed or coverage_decreased or lint_errors_increased: + print("Quality gate failed.") + if test_failed: + print("- Tests failed (pytest returned non-zero).") + if coverage_decreased: + print("- Coverage decreased compared to previous successful daily run.") + if lint_errors_increased: + print("- Lint error count increased compared to previous successful daily run.") + sys.exit(1) + + print("Quality gate passed.") + PY + + - name: Publish test results + if: always() + uses: EnricoMi/publish-unit-test-result-action@v2 + with: + files: tests_xml/tests.xml + check_name: Daily Pytest Results + comment_mode: off + + - name: Upload quality reports + if: always() + uses: actions/upload-artifact@v4 + with: + name: daily-quality-reports + path: | + tests_xml/tests.xml + coverage.xml + pylint.json + + - name: Upload daily quality metrics + if: always() + uses: actions/upload-artifact@v4 + with: + name: daily-quality-metrics + path: metrics.json diff --git a/.gitignore b/.gitignore index b7b671f9d..c73b75432 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,9 @@ # Hidden Files .* +!.github/ +!.github/workflows/ +!.github/workflows/* +!.gitignore # Byte-compiled / optimized / DLL files __pycache__/ @@ -10,12 +14,10 @@ __pycache__/ *.so # Distribution / packaging -.Python build/ develop-eggs/ downloads/ eggs/ -.eggs/ lib/ lib64/ parts/ @@ -23,7 +25,6 @@ sdist/ var/ wheels/ *.egg-info/ -.installed.cfg *.egg MANIFEST @@ -41,14 +42,9 @@ pip-delete-this-directory.txt coverage/ htmlcov/ tests_xml/ -.tox/ -.coverage -.coverage.* -.cache nosetests.xml coverage.xml *.cover -.hypothesis/ # Temporary data generated by unit tests climada_petals/engine/test/data/supplychain/* @@ -81,10 +77,6 @@ local_settings.py # Flask stuff: instance/ -.webassets-cache - -# Scrapy stuff: -.scrapy # Sphinx documentation doc/_build/ @@ -92,43 +84,19 @@ doc/_build/ # PyBuilder target/ -# Jupyter Notebook -.ipynb_checkpoints - -# pyenv -.python-version - -# celery beat schedule file -celerybeat-schedule - # SageMath parsed files *.sage.py # Environments -.env -.venv env/ venv/ ENV/ env.bak/ venv.bak/ -# Spyder project settings -.spyderproject -.spyproject - -# Rope project settings -.ropeproject - # mkdocs documentation /site -# mypy -.mypy_cache/ - -# mac finder files -.DS_Store - # climada system data files # they get downloaded separately from the repo data/system/global_coast* diff --git a/CHANGELOG.md b/CHANGELOG.md index 502fa5609..0d25f81b6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,22 @@ Code freeze date: YYYY-MM-DD ### Dependency Changes +### Added + +### Changed + +### Fixed + +### Deprecated + +### Removed + +## 6.1.0 + +Release date: 2025-09-30 + +### Dependency Changes + Added: - `lxml` >=5 (was implicitly part of the dependency tree before) @@ -31,8 +47,6 @@ Updated: - Add `lxml` as explicit dependency [#168](https://github.com/CLIMADA-project/climada_petals/pull/168) -### Deprecated - ### Removed - Removed code of the copernicus interface module [#142](https://github.com/CLIMADA-project/climada_petals/pull/142), including code added in [#150](https://github.com/CLIMADA-project/climada_petals/pull/150), [#151](https://github.com/CLIMADA-project/climada_petals/pull/151) and [#156](https://github.com/CLIMADA-project/climada_petals/pull/156). The module has been moved to a separate [repository](https://github.com/DahyannAraya/copernicus-seasonal-forecast-tools). diff --git a/climada_petals/_version.py b/climada_petals/_version.py index 7857c0db3..a44a37d9e 100644 --- a/climada_petals/_version.py +++ b/climada_petals/_version.py @@ -1 +1 @@ -__version__ = '6.0.2-dev' +__version__ = '6.1.1-dev' diff --git a/climada_petals/conf/climada.conf b/climada_petals/conf/climada.conf index bc27f9e99..22c882612 100644 --- a/climada_petals/conf/climada.conf +++ b/climada_petals/conf/climada.conf @@ -43,7 +43,7 @@ "geoclaw_work_dir": "{hazard.tc_surge_geoclaw.local_data}/runs", "resources": { "clawpack_git": "https://github.com/clawpack/clawpack.git", - "clawpack_version": "v5.9.2" + "clawpack_version": "v5.13.1" } }, "tc_tracks_forecast": { diff --git a/climada_petals/hazard/rf_glofas/transform_ops.py b/climada_petals/hazard/rf_glofas/transform_ops.py index 46671fc4b..c4a506aca 100644 --- a/climada_petals/hazard/rf_glofas/transform_ops.py +++ b/climada_petals/hazard/rf_glofas/transform_ops.py @@ -699,7 +699,7 @@ def flood_depth( # Clip infinite return periods return_period = return_period.clip( min=1, max=flood_maps["return_period"].max(), keep_attrs=True - ) + ).astype(np.float32) # All but 'longitude' and 'latitude' are core dimensions for this operation core_dims = list(return_period.dims) diff --git a/climada_petals/hazard/tc_surge_geoclaw/geoclaw_runner.py b/climada_petals/hazard/tc_surge_geoclaw/geoclaw_runner.py index 6947785f6..91c3e069c 100644 --- a/climada_petals/hazard/tc_surge_geoclaw/geoclaw_runner.py +++ b/climada_petals/hazard/tc_surge_geoclaw/geoclaw_runner.py @@ -387,10 +387,10 @@ def _set_rundata_claw(self) -> None: clawdata.checkpt_style = -3 clawdata.checkpt_interval = 25 clawdata.lower = [ - lim - self.outer_pad_deg for lim in self.areas["wind_area"][:2] + float(lim - self.outer_pad_deg) for lim in self.areas["wind_area"][:2] ] clawdata.upper = [ - lim + self.outer_pad_deg for lim in self.areas["wind_area"][2:] + float(lim + self.outer_pad_deg) for lim in self.areas["wind_area"][2:] ] clawdata.num_cells = [ # coarsest resolution: appx. 0.25 degrees @@ -456,7 +456,7 @@ def _set_rundata_amr(self) -> None: for area in self.areas["surge_areas"]: x_1, y_1, x_2, y_2 = area regions.append([maxlevel - 1, maxlevel, t_1, t_2, x_1, x_2, y_1, y_2]) - refinedata.speed_tolerance = list(np.arange(1.0, maxlevel - 2)) + refinedata.speed_tolerance = np.arange(1.0, maxlevel - 2).tolist() refinedata.variable_dt_refinement_ratios = True refinedata.wave_tolerance = 1.0 @@ -506,7 +506,7 @@ def _set_rundata_geo(self) -> None: [ clawdata.lower, clawdata.upper, - [np.infty, 0.0, -np.infty], + [np.inf, 0.0, -np.inf], [0.050, 0.025], ] ) diff --git a/climada_petals/hazard/tc_tracks_forecast.py b/climada_petals/hazard/tc_tracks_forecast.py index aea61cdae..2df10dc16 100644 --- a/climada_petals/hazard/tc_tracks_forecast.py +++ b/climada_petals/hazard/tc_tracks_forecast.py @@ -188,10 +188,17 @@ def fetch_bufr_ftp(target_dir=None, remote_dir=None): ------- [filelike] """ - con = ftplib.FTP(host=ECMWF_FTP.host.str(), + def connect_to_ecmwf(remote_dir) -> ftplib.FTP: + con = ftplib.FTP(host=ECMWF_FTP.host.str(), user=ECMWF_FTP.user.str(), - passwd=ECMWF_FTP.passwd.str()) + passwd=ECMWF_FTP.passwd.str(), + timeout=10) + if remote_dir is not None: + con.cwd(remote_dir) + return con + try: + con= connect_to_ecmwf(remote_dir=remote_dir) if remote_dir is None: # Read list of directories on the FTP server remote = pd.Series(con.nlst()) @@ -200,9 +207,7 @@ def fetch_bufr_ftp(target_dir=None, remote_dir=None): # Select the most recent directory (names are formatted yyyymmddhhmmss) remote = remote.sort_values(ascending=False) remote_dir = remote.iloc[0] - - # Connect to the directory - con.cwd(remote_dir) + con.cwd(remote_dir) # Filter to files with 'tropical_cyclone' in the name: each file is a forecast # ensemble for one event @@ -224,16 +229,24 @@ def fetch_bufr_ftp(target_dir=None, remote_dir=None): lfile = Path(target_dir, rfile).open('w+b') else: lfile = tempfile.TemporaryFile(mode='w+b') - - con.retrbinary('RETR ' + rfile, lfile.write) - lfile.seek(0) - localfiles.append(lfile) + for attempt in range(3,0,-1): + try: + con.retrbinary('RETR ' + rfile, lfile.write) + lfile.seek(0) + localfiles.append(lfile) + break + except TimeoutError as toe: + if attempt == 1: + raise toe + LOGGER.info("got timeout from ftp connection, trying again") + con.quit() + con = connect_to_ecmwf(remote_dir=remote_dir) except ftplib.all_errors as err: - con.quit() raise type(err)('Error while downloading BUFR TC tracks: ' + str(err)) from err - _ = con.quit() + finally: + con.quit() return localfiles diff --git a/climada_petals/test/test_tc_surge_geoclaw.py b/climada_petals/test/test_tc_surge_geoclaw.py index be17f1c36..991641f5e 100644 --- a/climada_petals/test/test_tc_surge_geoclaw.py +++ b/climada_petals/test/test_tc_surge_geoclaw.py @@ -110,8 +110,10 @@ def test_surge_from_track(self): ) self.assertEqual(intensity.shape, (centroids.shape[0],)) - self.assertTrue(np.all(intensity[:6] > 0)) - self.assertTrue(np.all(intensity[6:] == 0)) + np.testing.assert_array_equal( + intensity[:6] > 0, [True, True, False, True, True, True] + ) + np.testing.assert_array_equal(intensity[6:] == 0, [True] * len(intensity[6:])) for gdata in gauge_data: self.assertTrue((gdata['time'][0][0] - track.time[0]) / np.timedelta64(1, 'h') >= 0) self.assertTrue((track.time[-1] - gdata['time'][0][-1]) / np.timedelta64(1, 'h') >= 0) diff --git a/pyproject.toml b/pyproject.toml index 214e893d9..237666d21 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "climada_petals" -version = "6.0.2-dev" +version = "6.1.1-dev" description = "CLIMADA Extensions" readme = "README.md" requires-python = ">=3.10,<3.13" diff --git a/requirements/env_climada.yml b/requirements/env_climada.yml index 38e145bef..6bf07fe74 100644 --- a/requirements/env_climada.yml +++ b/requirements/env_climada.yml @@ -18,3 +18,8 @@ dependencies: - ruamel.yaml>=0.18 - scikit-image>=0.25 - xesmf>=0.8 + - pip: + - openpyxl==3.1.0 # this is necessary as currently (2026-02-26) in conda-forge there is + # no openpyxl 3.1.* version compatible with python 3.12 and pymrio 0.6 + # so by updating the envrionment with this file openpyxl gets downgraded to 3.0.9 + # and the silently upgraded to 3.1.0 again diff --git a/requirements/env_docs.yml b/requirements/env_docs.yml index aedbc5e69..6fab654f3 100644 --- a/requirements/env_docs.yml +++ b/requirements/env_docs.yml @@ -20,7 +20,7 @@ dependencies: - nb_conda_kernels - pandoc - pip - - sphinx>=2.0 + - sphinx>=2.0,<9.0 - pip: - coverage>=4.5 - descartes