From 95b3d135f81260e6d6cab26554be99db2a3ed716 Mon Sep 17 00:00:00 2001 From: Oliver Kurz Date: Tue, 29 Apr 2025 09:32:27 +0200 Subject: [PATCH 1/4] Introduce python style checks with ruff --- Makefile | 6 +++++- racktables/get_unused_machines.py | 2 +- racktables/racktables.py | 19 ++++++++++--------- tests/test_trigger_bisect_jobs.py | 26 +++++++++++++------------- 4 files changed, 29 insertions(+), 24 deletions(-) diff --git a/Makefile b/Makefile index 4f890f94..b3e8a2e2 100644 --- a/Makefile +++ b/Makefile @@ -31,7 +31,7 @@ test-online: # Invalid JSON causes the job to abort with an error -tw_openqa_host=example.com dry_run=1 ./trigger-openqa_in_openqa -checkstyle: test-shellcheck test-yaml +checkstyle: test-shellcheck test-yaml checkstyle-python shfmt: shfmt -w . $$(file --mime-type test/*.t | sed -n 's/^\(.*\):.*text\/x-shellscript.*$$/\1/p') @@ -46,6 +46,10 @@ test-yaml: @which yamllint >/dev/null 2>&1 || echo "Command 'yamllint' not found, can not execute YAML syntax checks" yamllint --strict $$(git ls-files "*.yml" "*.yaml" ":!external/") +checkstyle-python: + @which ruff >/dev/null 2>&1 || echo "Command 'ruff' not found, can not execute python style checks" + ruff --check + update-deps: tools/update-deps --cpanfile cpanfile --specfile dist/rpm/os-autoinst-scripts-deps.spec diff --git a/racktables/get_unused_machines.py b/racktables/get_unused_machines.py index 35c315f0..ad212012 100644 --- a/racktables/get_unused_machines.py +++ b/racktables/get_unused_machines.py @@ -32,5 +32,5 @@ obj.from_path(url_path) try: print(obj.fqdn, flush=True) - except: + except Exception: print(obj.common_name, flush=True) diff --git a/racktables/racktables.py b/racktables/racktables.py index 9095eb0e..9a15bfac 100644 --- a/racktables/racktables.py +++ b/racktables/racktables.py @@ -14,20 +14,21 @@ def __init__(self, url, username, password): self.url = url def search(self, search_payload={}): - params="&".join("%s=%s" % (k, v) for k, v in search_payload.items()) - req = self.s.get( - join(self.url, "index.php"), - params=params - ) + params = "&".join("%s=%s" % (k, v) for k, v in search_payload.items()) + req = self.s.get(join(self.url, "index.php"), params=params) status = req.status_code if status == 401: - raise Exception("Racktables returned 401 Unauthorized. Are your credentials correct?") + raise Exception( + "Racktables returned 401 Unauthorized. Are your credentials correct?" + ) elif status >= 300: - raise Exception(f"Racktables returned statuscode {status} while trying to access {req.request.url}. Manual investigation needed.") + raise Exception( + f"Racktables returned statuscode {status} while trying to access {req.request.url}. Manual investigation needed." + ) soup = BeautifulSoup(req.text, "html.parser") result_table = soup.find("table", {"class": "cooltable"}) result_objs = result_table.find_all( - "tr", lambda tag: tag != None + "tr", lambda tag: tag is not None ) # Racktables does not use table-heads so we have to filter the header out (it has absolutely no attributes) return result_objs @@ -49,5 +50,5 @@ def from_path(self, url_path): value = row.find("td").text sane_name = re.sub(r"[^a-z_]+", "", name.lower().replace(" ", "_")) setattr(self, sane_name, value) - except Exception as e: + except Exception: pass diff --git a/tests/test_trigger_bisect_jobs.py b/tests/test_trigger_bisect_jobs.py index 000b1a2b..76b421a0 100644 --- a/tests/test_trigger_bisect_jobs.py +++ b/tests/test_trigger_bisect_jobs.py @@ -7,7 +7,6 @@ import importlib.util import json import os.path -import sys import re from unittest.mock import MagicMock, call, patch from urllib.parse import urlparse @@ -69,17 +68,16 @@ def mocked_call(cmds, dry_run=False): "OPENQA_INVESTIGATE_ORIGIN=https://openqa.opensuse.org/tests/7848818", ] + def test_catch_CalledProcessError(caplog): import subprocess + args = args_factory() args.url = "https://openqa.opensuse.org/tests/7848818" openqa.fetch_url = MagicMock(side_effect=mocked_fetch_url) - cmd_args = ["openqa-clone-job"] exp_err = "returned non-zero exit status 255." error = subprocess.CompletedProcess( - args=[], returncode=255, - stderr=exp_err, - stdout='' + args=[], returncode=255, stderr=exp_err, stdout="" ) with patch("subprocess.run", return_value=error): with pytest.raises(subprocess.CalledProcessError) as e: @@ -89,19 +87,21 @@ def test_catch_CalledProcessError(caplog): assert f"{exp_err}" in str(e.value.stderr) exp_err = "Current job 7848818 will fail, because the repositories for the below updates are unavailable" - error.stderr=exp_err + error.stderr = exp_err comment_process = subprocess.CompletedProcess( - args=[], returncode=0, - stderr='', - stdout=b'doo' + args=[], returncode=0, stderr="", stdout=b"doo" ) with patch("subprocess.run", side_effect=[error, comment_process]) as mocked: with pytest.raises(SystemExit) as e: openqa.main(args) - assert re.search("jobs/.*/comments.*text=.*updates are unavailable", str(mocked.call_args_list[-1][0])) + assert re.search( + "jobs/.*/comments.*text=.*updates are unavailable", + str(mocked.call_args_list[-1][0]), + ) assert e.value.code == 0 assert f"{exp_err}" in caplog.text + def test_clone(): openqa.call = MagicMock(side_effect=mocked_call) openqa.openqa_clone(cmds, dry_run=False) @@ -165,8 +165,8 @@ def test_triggers(): args = args_factory() args.url = "https://openqa.opensuse.org/tests/7848818" openqa.openqa_clone = MagicMock(return_value='{"7848818": 234567}') - openqa.openqa_comment = MagicMock(return_value='') - openqa.openqa_set_job_prio = MagicMock(return_value='') + openqa.openqa_comment = MagicMock(return_value="") + openqa.openqa_set_job_prio = MagicMock(return_value="") openqa.fetch_url = MagicMock(side_effect=mocked_fetch_url) openqa.main(args) calls = [ @@ -325,7 +325,7 @@ def test_network_problems(): try: openqa.main(args) assert False - except requests.exceptions.ConnectionError as e: + except requests.exceptions.ConnectionError: assert True From 0b68818d0f51e25f06ac109a7682014bb61d14c9 Mon Sep 17 00:00:00 2001 From: Oliver Kurz Date: Mon, 28 Apr 2025 16:03:43 +0200 Subject: [PATCH 2/4] openqa-trigger-bisect-jobs: Also evaluate "exclude_name_regex" It was observed that openqa-trigger-bisect-jobs were triggered even though according openqa-investigate jobs were not as the variable "exclude_name_regex" would exclude the job. However unlike "exclude_group_regex" the variable "exclude_name_regex" so far was only evaluated within openqa-investigate, not openqa-trigger-bisect-jobs. This commit adds according handling within openqa-trigger-bisect-jobs with according unit test. --- openqa-trigger-bisect-jobs | 5 +++++ tests/test_trigger_bisect_jobs.py | 18 +++++++++++++++--- 2 files changed, 20 insertions(+), 3 deletions(-) diff --git a/openqa-trigger-bisect-jobs b/openqa-trigger-bisect-jobs index f31f6e0e..732ff62d 100755 --- a/openqa-trigger-bisect-jobs +++ b/openqa-trigger-bisect-jobs @@ -264,6 +264,11 @@ def main(args): if re.search(exclude_group_regex, full_group): return + exclude_name_regex = os.environ.get("exclude_name_regex", "") + if len(exclude_name_regex) > 0 and re.search(exclude_name_regex, job["test"]): + log.debug("job name '%s' matches 'exclude_name_regex', skipping" % job["test"]) + return + log.debug("Received job data: %s" % test_data) test = job["settings"]["TEST"] prio = int(job["priority"]) + args.priority_add diff --git a/tests/test_trigger_bisect_jobs.py b/tests/test_trigger_bisect_jobs.py index 76b421a0..bfb90750 100644 --- a/tests/test_trigger_bisect_jobs.py +++ b/tests/test_trigger_bisect_jobs.py @@ -23,9 +23,6 @@ openqa = importlib.util.module_from_spec(spec) loader.exec_module(openqa) -# should only affect test_exclude_group_regex() as it does not match other tests -os.environ["exclude_group_regex"] = "s.*parent?-group / some-.*" - Incident = openqa.Incident @@ -301,10 +298,25 @@ def test_exclude_group_regex(): args = args_factory() openqa.openqa_clone = MagicMock(return_value="") openqa.fetch_url = MagicMock(side_effect=mocked_fetch_url) + # should only affect test_exclude_group_regex() as it does not match other tests + os.environ["exclude_group_regex"] = "s.*parent?-group / some-.*" + + args.url = "http://openqa.opensuse.org/tests/123457" + openqa.main(args) + openqa.openqa_clone.assert_not_called() + del os.environ["exclude_group_regex"] + + +def test_exclude_name_regex(): + args = args_factory() + openqa.openqa_clone = MagicMock(return_value="") + openqa.fetch_url = MagicMock(side_effect=mocked_fetch_url) + os.environ["exclude_name_regex"] = "with.*group" args.url = "http://openqa.opensuse.org/tests/123457" openqa.main(args) openqa.openqa_clone.assert_not_called() + del os.environ["exclude_name_regex"] def test_exclude_investigated(): From 3f07e1bdc4e51d8e78f23e55a469869cbf51d7a8 Mon Sep 17 00:00:00 2001 From: Oliver Kurz Date: Tue, 29 Apr 2025 09:37:02 +0200 Subject: [PATCH 3/4] openqa-trigger-bisect-jobs: Also trigger log line for exclude_group_regex --- openqa-trigger-bisect-jobs | 1 + 1 file changed, 1 insertion(+) diff --git a/openqa-trigger-bisect-jobs b/openqa-trigger-bisect-jobs index 732ff62d..c476a02d 100755 --- a/openqa-trigger-bisect-jobs +++ b/openqa-trigger-bisect-jobs @@ -262,6 +262,7 @@ def main(args): if "parent_group" in job: full_group = "%s / %s" % (job["parent_group"], full_group) if re.search(exclude_group_regex, full_group): + log.debug("job group '%s' matches 'exclude_group_regex', skipping" % full_group) return exclude_name_regex = os.environ.get("exclude_name_regex", "") From 9dc5d423ec763cdcfb700a1a6215d8a4af51913c Mon Sep 17 00:00:00 2001 From: Oliver Kurz Date: Tue, 29 Apr 2025 09:54:21 +0200 Subject: [PATCH 4/4] openqa-trigger-bisect-jobs: Add help text --- openqa-trigger-bisect-jobs | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/openqa-trigger-bisect-jobs b/openqa-trigger-bisect-jobs index c476a02d..34313b7a 100755 --- a/openqa-trigger-bisect-jobs +++ b/openqa-trigger-bisect-jobs @@ -13,6 +13,13 @@ from urllib.parse import urlparse, urlunparse import requests +""" +Trigger openQA tests bisecting which pending operating system maintenance +update, called "incident", might be thea reason for an openQA test failure. +Can be called manually and is also called by openQA hook scripts, see +http://open.qa/docs/#_enable_custom_hook_scripts_on_job_done_based_on_result +""" + USER_AGENT = 'openqa-trigger-bisect-jobs (https://github.com/os-autoinst/scripts)' logging.basicConfig()