ci(check-fixtures): parallelise fixture checking commands

tiborsimko · tiborsimko · commit 92eb08484bf7 · 2025-01-10T11:04:26.000+01:00
Introduces several fixture-checking commands to the `run-tests.sh`
checking script (`--check-jsonlint`, `--check-record-ids`,
`--check-record-dois`, `--check-docs-slugs`, `--check-secondary-typeS`,
`--check-trailing-whitespace`) in order to make the lengthy
`--check-fixtures` process paralellisable.

Removes `--check-pycodestyle` command since we are using the Black code
formatter.

Amends CI accordingly, and renames formatters and linters for clarity.

Amends `.editorconfig` to add rules for shell scripts and to remove ReST
file rules as we have switched to Markdown.

BREAKING CHANGE: Drops `run-tests.sh --check-fixtures` and `run-tests.sh
--check-pycodestyle` commands.
diff --git a/.editorconfig b/.editorconfig
@@ -1,7 +1,7 @@
 # -*- coding: utf-8 -*-
 #
 # This file is part of CERN Open Data Portal.
-# Copyright (C) 2016 CERN.
+# Copyright (C) 2016, 2025 CERN.
 #
 # CERN Open Data Portal is free software; you can redistribute it
 # and/or modify it under the terms of the GNU General Public License as
@@ -40,8 +40,8 @@ known_first_party = cernopendata
 multi_line_output = 2
 default_section = THIRDPARTY
 
-# RST files (used by sphinx)
-[*.rst]
+# Shell script files
+[*.sh]
 indent_size = 4
 
 # CSS, HTML, JS, JSON, YML
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -1,5 +1,5 @@
 # This file is part of CERN Open Data Portal.
-# Copyright (C) 2020, 2023, 2024 CERN.
+# Copyright (C) 2020, 2023, 2024, 2025 CERN.
 #
 # CERN Open Data Portal is free software; you can redistribute it
 # and/or modify it under the terms of the GNU General Public License as
@@ -20,23 +20,12 @@
 # waive the privileges and immunities granted to it by virtue of its status
 # as an Intergovernmental Organization or submit itself to any jurisdiction.
 
-name: CI
+name: ci
 
 on: [push, pull_request]
 
 jobs:
-  lint-shellcheck:
-    runs-on: ubuntu-20.04
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v2
-
-      - name: Runs shell script static analysis
-        run: |
-          sudo apt-get install shellcheck
-          ./run-tests.sh --check-shellscript
-
-  lint-black:
+  format-black:
     runs-on: ubuntu-20.04
     steps:
       - uses: actions/checkout@v2
@@ -51,7 +40,7 @@ jobs:
           pip install black
           ./run-tests.sh --check-black
 
-  lint-pycodestyle:
+  format-isort:
     runs-on: ubuntu-20.04
     steps:
       - name: Checkout
@@ -62,11 +51,25 @@ jobs:
         with:
           python-version: 3.9
 
-      - name: Check compliance with Python coding style conventions
+      - name: Check isort
         run: |
           pip install --upgrade pip
-          pip install pycodestyle
-          ./run-tests.sh --check-pycodestyle
+          pip install isort
+          ./run-tests.sh --check-isort
+
+  lint-jsonlint:
+    runs-on: ubuntu-24.04
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Setup Node
+        uses: actions/setup-node@v4
+
+      - name: Lint JSON files
+        run: |
+          npm install jsonlint --global
+          ./run-tests.sh --check-jsonlint
 
   lint-pydocstyle:
     runs-on: ubuntu-20.04
@@ -85,39 +88,28 @@ jobs:
           pip install pydocstyle
           ./run-tests.sh --check-pydocstyle
 
-  check-fixtures:
+  lint-shellcheck:
     runs-on: ubuntu-20.04
     steps:
       - name: Checkout
         uses: actions/checkout@v2
 
-      - name: Setup node
-        uses: actions/setup-node@v1
-        with:
-          node-version: "14"
-
-      - name: Install Node dependencies
-        run: npm install -g jsonlint
-
-      - name: Check fixtures
-        run: ./run-tests.sh --check-fixtures
+      - name: Runs shell script static analysis
+        run: |
+          sudo apt-get install shellcheck
+          ./run-tests.sh --check-shellcheck
 
-  check-isort:
-    runs-on: ubuntu-20.04
+  check-docs-slugs:
+    runs-on: ubuntu-24.04
     steps:
       - name: Checkout
         uses: actions/checkout@v2
 
-      - name: Setup Python
-        uses: actions/setup-python@v2
-        with:
-          python-version: 3.9
+      - name: Setup node
+        uses: actions/setup-node@v4
 
-      - name: Check isort
-        run: |
-          pip install --upgrade pip
-          pip install isort
-          ./run-tests.sh --check-isort
+      - name: Check docs slugs
+        run: ./run-tests.sh --check-docs-slugs
 
   check-licenses:
     runs-on: ubuntu-20.04
@@ -134,3 +126,51 @@ jobs:
         run: |
           pip install --upgrade pip
           ./run-tests.sh --check-licenses
+
+  check-record-dois:
+    runs-on: ubuntu-24.04
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+
+      - name: Setup node
+        uses: actions/setup-node@v4
+
+      - name: Check record DOIs
+        run: ./run-tests.sh --check-record-dois
+
+  check-record-ids:
+    runs-on: ubuntu-24.04
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+
+      - name: Setup node
+        uses: actions/setup-node@v4
+
+      - name: Check record IDs
+        run: ./run-tests.sh --check-record-ids
+
+  check-secondary-types:
+    runs-on: ubuntu-24.04
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+
+      - name: Setup node
+        uses: actions/setup-node@v4
+
+      - name: Check docs slugs
+        run: ./run-tests.sh --check-secondary-types
+
+  check-trailing-whitespace:
+    runs-on: ubuntu-24.04
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+
+      - name: Setup node
+        uses: actions/setup-node@v4
+
+      - name: Check docs slugs
+        run: ./run-tests.sh --check-trailing-whitespace
diff --git a/run-tests.sh b/run-tests.sh
@@ -1,7 +1,7 @@
 #!/usr/bin/env bash
 #
 # This file is part of CERN Open Data Portal.
-# Copyright (C) 2015, 2016, 2017, 2018, 2019, 2020, 2024 CERN.
+# Copyright (C) 2015, 2016, 2017, 2018, 2019, 2020, 2024, 2025 CERN.
 #
 # CERN Open Data Portal is free software; you can redistribute it
 # and/or modify it under the terms of the GNU General Public License as
@@ -22,62 +22,58 @@
 # waive the privileges and immunities granted to it by virtue of its status
 # as an Intergovernmental Organization or submit itself to any jurisdiction.
 
-# quit on errors and potentially unbound symbols:
 set -o errexit
 set -o nounset
 
-check_script () {
-    shellcheck run-tests.sh
-}
-
-check_black () {
+check_black() {
     black --check .
 }
 
-
-check_fixtures () {
-    # check for possibly incorrect JSON files:
-    find data/ -name "*.json" -exec jsonlint -q {} \;
-
-    # check record ID uniqueness:
-    dupes=$(jq '.[].recid' data/{records,skeletons}/*.json | sort | uniq -d)
+check_docs_slugs() {
+    # shellcheck disable=SC2044
+    dupes=$(for file in $(find data/docs -name "*.json"); do jq '.[].slug' "$file"; done | sort | grep -v null | uniq -d)
     if [ "x${dupes}" != "x" ]; then
-        echo "[ERROR] Found duplicate record IDs:"
+        echo "[ERROR] Found duplicate docs slugs:"
         echo "${dupes}"
         exit 1
     fi
+}
+
+check_isort() {
+    isort -rc -c -df --profile black -- **/*.py
+}
+
+check_jsonlint() {
+    find . -name "*.json" -exec jsonlint -q {} \+
+}
+
+check_licenses() {
+    scripts/check_licenses.py
+}
 
-    # check DOI uniqueness:
+check_pydocstyle() {
+    pydocstyle scripts
+}
+
+check_record_dois() {
     dupes=$(jq '.[].doi' data/{records,skeletons}/*.json | sort | grep -v null | uniq -d)
     if [ "x${dupes}" != "x" ]; then
         echo "[ERROR] Found duplicate record DOIs:"
         echo "${dupes}"
         exit 1
     fi
+}
 
-    # check docs slug uniqueness:
-    # shellcheck disable=SC2044
-    dupes=$(for file in $(find data/docs -name "*.json"); do jq '.[].slug' "$file"; done | sort | grep -v null | uniq -d)
+check_record_ids() {
+    dupes=$(jq '.[].recid' data/{records,skeletons}/*.json | sort | uniq -d)
     if [ "x${dupes}" != "x" ]; then
-        echo "[ERROR] Found duplicate docs slugs:"
+        echo "[ERROR] Found duplicate record IDs:"
         echo "${dupes}"
         exit 1
     fi
+}
 
-    # check trailing whitespace:
-    whitespace_found_p=0
-    for file in $(git ls-files | grep -E '.(py|html|css|json|md|sh|txt|yml)$'); do
-        if grep -q ' $' "$file"; then
-            whitespace_found_p=1
-            echo "[ERROR] Found trailing whitespace in ${file}."
-        fi
-    done
-
-    if [ "${whitespace_found_p}" != "0" ]; then
-        exit 1
-    fi
-
-    # check for empty secondary type in fixtures
+check_secondary_types() {
     # shellcheck disable=SC2044
     for file in $(find data/{records,docs}/ -name "*.json"); do
         secondaries=$(jq '.[].type.secondary' "$file" -c | sort | uniq)
@@ -87,46 +83,55 @@ check_fixtures () {
     done
 }
 
-check_pycodestyle () {
-    pycodestyle --max-line-length=120 scripts
-}
-
-check_pydocstyle () {
-    pydocstyle scripts
+check_shellcheck() {
+    shellcheck run-tests.sh
 }
 
-check_isort () {
-    isort -rc -c -df --profile black -- **/*.py
-}
+check_trailing_whitespace() {
+    whitespace_found_p=0
+    for file in $(git ls-files | grep -E '.(py|html|css|json|md|sh|txt|yml)$'); do
+        if grep -q ' $' "$file"; then
+            whitespace_found_p=1
+            echo "[ERROR] Found trailing whitespace in ${file}."
+        fi
+    done
 
-check_licenses () {
-    scripts/check_licenses.py
+    if [ "${whitespace_found_p}" != "0" ]; then
+        exit 1
+    fi
 }
 
-check_all () {
-    check_script
-    check_fixtures
-    check_pycodestyle
+check_all() {
     check_black
-    check_pydocstyle
+    check_docs_slugs
     check_isort
+    check_jsonlint
     check_licenses
+    check_pydocstyle
+    check_record_dois
+    check_record_ids
+    check_secondary_types
+    check_shellcheck
+    check_trailing_whitespace
 }
 
 if [ $# -eq 0 ]; then
     check_all
     exit 0
 fi
 
-for arg in "$@"
-do
-    case $arg in
-        --check-shellscript) check_script;;
-        --check-fixtures) check_fixtures;;
-        --check-pycodestyle) check_pycodestyle;;
-        --check-pydocstyle) check_pydocstyle;;
-        --check-isort) check_isort;;
-        --check-licenses) check_licenses;;
-        *)
-    esac
-done
+arg="$1"
+case $arg in
+--check-black) check_black ;;
+--check-docs-slugs) check_docs_slugs ;;
+--check-isort) check_isort ;;
+--check-jsonlint) check_jsonlint ;;
+--check-licenses) check_licenses ;;
+--check-pydocstyle) check_pydocstyle ;;
+--check-record-dois) check_record_dois ;;
+--check-record-ids) check_record_ids ;;
+--check-secondary-types) check_secondary_types ;;
+--check-shellcheck) check_shellcheck ;;
+--check-trailing-whitespace) check_trailing_whitespace ;;
+*) echo "[ERROR] Invalid argument '$arg'. Exiting." && exit 1 ;;
+esac
diff --git a/scripts/split_json_file.py b/scripts/split_json_file.py
@@ -28,7 +28,7 @@ def split_json_file(filename, split):
             )
             print("[INFO] Creating file %s..." % filenamepart)
             split_content = json.dumps(
-                records[split * i: split * (i + 1)],
+                records[split * i : split * (i + 1)],
                 indent=2,
                 sort_keys=True,
                 ensure_ascii=False,

Original file line number	Diff line number	Diff line change
`@@ -28,7 +28,7 @@ def split_json_file(filename, split):`
`28`	`28`	`)`
`29`	`29`	`print("[INFO] Creating file %s..." % filenamepart)`
`30`	`30`	`split_content = json.dumps(`
`31`		`- records[split * i: split * (i + 1)],`
	`31`	`+ records[split * i : split * (i + 1)],`
`32`	`32`	`indent=2,`
`33`	`33`	`sort_keys=True,`
`34`	`34`	`ensure_ascii=False,`