diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index 20f0254cc0..0a606602a1 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -115,9 +115,20 @@ jobs: runs-on: ubuntu-22.04 steps: - uses: actions/checkout@v4 + - run: | + echo "PYTHON_VERSION=$(cat .python-version)" >> $GITHUB_ENV - name: Run validator run: ./scripts/run_validator.sh - - name: Running schema tests + - name: Install Poetry + uses: snok/install-poetry@v1 + with: + version: 1.8.4 + virtualenvs-create: true + - uses: actions/setup-python@v5 + with: + python-version: ${{ env.PYTHON_VERSION }} + cache: "poetry" + - name: Running schema validation run: make validate-test-schemas test-functional: needs: [python-dependencies, node-dependencies] diff --git a/Makefile b/Makefile index 907e145505..8ca21ad59e 100644 --- a/Makefile +++ b/Makefile @@ -67,10 +67,10 @@ generate-spec: poetry run python -m tests.functional.generate_pages schemas/test/en/$(SCHEMA).json ./tests/functional/generated_pages/$(patsubst test_%,%,$(SCHEMA)) -r '../../base_pages' -s tests/functional/spec/$(SCHEMA).spec.js validate-test-schemas: - ./scripts/validate_test_schemas.sh + poetry run python -m scripts.validate_test_schemas validate-test-schema: - ./scripts/validate_test_schemas.sh $(SCHEMA_PATH)$(SCHEMA).json + poetry run python -m scripts.validate_test_schemas $(SCHEMA_PATH) translation-templates: poetry run python -m scripts.extract_translation_templates diff --git a/schemas/test/en/test_address.json b/schemas/test/en/test_address.json index f055666055..ba68f29d3a 100644 --- a/schemas/test/en/test_address.json +++ b/schemas/test/en/test_address.json @@ -19,6 +19,9 @@ { "name": "ru_name", "type": "string" + }, + { + "name": "Testing" } ], "questionnaire_flow": { diff --git a/schemas/test/en/test_default_with_skip.json b/schemas/test/en/test_default_with_skip.json index 08963ca5bf..b1609c9d72 100644 --- a/schemas/test/en/test_default_with_skip.json +++ b/schemas/test/en/test_default_with_skip.json @@ -22,7 +22,7 @@ } ], "questionnaire_flow": { - "type": "Linear", + "type": "Linear ", "options": { "summary": { "collapsible": false @@ -74,7 +74,7 @@ "when": { "!=": [ { - "source": "answers", + "source": "answers ", "identifier": "answer-one" }, 1 diff --git a/scripts/validate_test_schemas.py b/scripts/validate_test_schemas.py new file mode 100644 index 0000000000..da03072e0a --- /dev/null +++ b/scripts/validate_test_schemas.py @@ -0,0 +1,137 @@ +import json +import logging +import os +import re +import subprocess +import sys +import time +from concurrent.futures import ThreadPoolExecutor, as_completed + +logging.basicConfig( + level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s" +) + + +def check_connection(): + checks = 4 + while checks > 0: + response = subprocess.run( + [ + "curl", + "-so", + "/dev/null", + "-w", + "%{http_code}", + "http://localhost:5002/status", + ], + capture_output=True, + text=True, + check=False, + ).stdout.strip() + + if response != "200": + logging.error("\033[31m---Error: Schema Validator Not Reachable---\033[0m") + logging.error("\033[31mHTTP Status: %s\033[0m", response) + if checks != 1: + logging.info("Retrying...\n") + time.sleep(5) + else: + logging.info("Exiting...\n") + sys.exit(1) + checks -= 1 + else: + checks = 0 + + +def get_schemas() -> list[str]: + if len(sys.argv) == 1 or sys.argv[1] == "--local": + file_path = "./schemas/test/en" + schemas = [ + os.path.join(file_path, f) + for f in os.listdir(file_path) + if f.endswith(".json") + ] + logging.info("--- Testing Schemas in %s ---", file_path) + else: + file_path = sys.argv[1] + schemas = [sys.argv[1]] + logging.info("--- Testing %s Schema ---", file_path) + return schemas + + +def validate_schema(schema_path): + try: + result = subprocess.run( + [ + "curl", + "-s", + "-w", + "HTTPSTATUS:%{http_code}", + "-X", + "POST", + "-H", + "Content-Type: application/json", + "-d", + f"@{schema_path}", + "http://localhost:5001/validate", + ], + capture_output=True, + text=True, + check=True, + ) + return schema_path, result.stdout + except subprocess.CalledProcessError as e: + logging.info("Error validating schema %s: %s", schema_path, e) + return schema_path, None + + +def main(): + # pylint: disable=broad-exception-caught + error = False + passed = 0 + failed = 0 + + check_connection() + schemas = get_schemas() + + with ThreadPoolExecutor(max_workers=20) as executor: + future_to_schema = { + executor.submit(validate_schema, schema): schema for schema in schemas + } + for future in as_completed(future_to_schema): + schema = future_to_schema[future] + try: + schema_path, result = future.result() + # Extract HTTP body + http_body = re.sub(r"HTTPSTATUS:.*", "", result) + + # Convert HTTP body to JSON + http_body_json = json.loads(http_body) + + # Format JSON + formatted_json = json.dumps(http_body_json, indent=4) + + # Extract HTTP status code + result_response = re.search(r"HTTPSTATUS:(\d+)", result)[1] + + if result_response == "200" and http_body_json == {}: + logging.info("\033[32m%s: PASSED\033[0m", schema_path) + passed += 1 + else: + logging.error("\033[31m%s: FAILED\033[0m", schema_path) + logging.error( + "\033[31mHTTP Status @ /validate: %s\033[0m", result_response + ) + logging.error("\033[31mHTTP Status: %s\033[0m", formatted_json) + error = True + failed += 1 + except Exception as e: + logging.error("\033[31mError processing %s: %s\033[0m", schema, e) + + logging.info("\033[32m%s passed\033[0m - \033[31m%s failed\033[0m", passed, failed) + if error: + sys.exit(1) + + +if __name__ == "__main__": + main() diff --git a/scripts/validate_test_schemas.sh b/scripts/validate_test_schemas.sh deleted file mode 100755 index e585590bc6..0000000000 --- a/scripts/validate_test_schemas.sh +++ /dev/null @@ -1,76 +0,0 @@ -#!/usr/bin/env bash - -green="$(tput setaf 2)" -red="$(tput setaf 1)" -default="$(tput sgr0)" -checks=4 - - - - -until [ "$checks" == 0 ]; do - response="$(curl -so /dev/null -w '%{http_code}' http://localhost:5002/status)" - - if [ "$response" != "200" ]; then - echo "${red}---Error: Schema Validator Not Reachable---" - echo "HTTP Status: $response" - if [ "$checks" != 1 ]; then - echo -e "Retrying...${default}\\n" - sleep 5 - else - echo -e "Exiting...${default}\\n" - exit 1 - fi - (( checks-- )) - else - (( checks=0 )) - fi - -done - -exit=0 - -if [ $# -eq 0 ] || [ "$1" == "--local" ]; then - file_path="./schemas/test/en" -else - file_path="$1" -fi - -echo "--- Testing Schemas in $file_path ---" -failed=0 -passed=0 - -file_path_name=$(find "$file_path" -name '*.json') - -validate() { - schema=$1 - result="$(curl -s -w 'HTTPSTATUS:%{http_code}' -X POST -H "Content-Type: application/json" -d @"$schema" http://localhost:5001/validate | tr -d '\n')" - # shellcheck disable=SC2001 - HTTP_BODY=$(echo "${result}" | sed -e 's/HTTPSTATUS\:.*//g') - result_response="${result//*HTTPSTATUS:/}" - result_body=$(echo "$HTTP_BODY" | python -m json.tool) - - if [ "$result_response" == "200" ] && [ "$result_body" == "{}" ]; then - echo -e "${green}$schema - PASSED${default}" - (( passed++ )) - else - echo -e "\\n${red}$schema - FAILED" - echo "HTTP Status @ /validate: [$result_response]" - echo -e "Error: [$result_body]${default}\\n" - (( failed++ )) - exit=1 - fi -} - - -N_TIMES_IN_PARALLEL=20 - -for schema in ${file_path_name}; do - ((i=i%N_TIMES_IN_PARALLEL)); ((i++==0)) && wait -# Spawn multiple (N_TIMES_IN_PARALLEL) processes in subshells and send to background, but keep printing outputs. - validate "$schema" & -done - - - -exit "$exit"