From 7b9933bbf07e6318ec4b2912f1ae799121d255fa Mon Sep 17 00:00:00 2001 From: Brian Maltzan Date: Mon, 8 Jun 2026 13:46:47 -0400 Subject: [PATCH 1/6] SUBMISSION-16: test helpers in new/review --- .../ui/controllers/new/tests/test_review.py | 329 ++++++++++++++++++ 1 file changed, 329 insertions(+) diff --git a/submit_ce/ui/controllers/new/tests/test_review.py b/submit_ce/ui/controllers/new/tests/test_review.py index 17dd071c..d48f6280 100644 --- a/submit_ce/ui/controllers/new/tests/test_review.py +++ b/submit_ce/ui/controllers/new/tests/test_review.py @@ -1,8 +1,13 @@ """Tests for :mod:`submit_ce.ui.controllers.new.review`.""" from http import HTTPStatus as status +from unittest.mock import MagicMock +from werkzeug.datastructures import MultiDict +from submit_ce.domain.compilation import Compilation +from submit_ce.domain.event import SetSourceFormat +from submit_ce.domain.exceptions import InvalidEvent, SaveError from submit_ce.ui.controllers.new import review @@ -20,3 +25,327 @@ def test_review_files_get_warning_via_http(app, authorized_client, sub_files, assert resp.status_code == status.OK assert mock_flash.called assert "couldn't load preflight data" in mock_flash.call_args[0][0] + + +def _make_workspace(*paths): + """Build a stand-in workspace whose `.files` carry the given paths.""" + ws = MagicMock() + ws.files = [MagicMock(path=p) for p in paths] + return ws + + +def test_update_preflight_no_form_no_deletions_returns_false( + app, authorized_user, mocker): + """Empty POST (no form fields, no selected files) is a no-op.""" + mocker.patch.object(review, '_get_user_decisions_data', return_value=None) + with app.app_context(): + mock_save = mocker.patch.object(app.api, 'save') + result = review._update_preflight( + MultiDict(), 'sub1', _make_workspace('paper.tex'), + authorized_user, None, + ) + assert result is False + mock_save.assert_not_called() + + +def test_update_preflight_selected_files_not_in_workspace_returns_false( + app, authorized_user, mocker): + """Paths in selected_files that aren't in the workspace are filtered out; + with no form fields, the call is a no-op.""" + mocker.patch.object(review, '_get_user_decisions_data', return_value=None) + with app.app_context(): + mock_save = mocker.patch.object(app.api, 'save') + params = MultiDict([('selected_files', 'ghost.tex')]) + result = review._update_preflight( + params, 'sub1', _make_workspace('paper.tex'), + authorized_user, None, + ) + assert result is False + mock_save.assert_not_called() + + +def test_update_preflight_files_to_delete_triggers_save( + app, authorized_user, mocker): + """A valid file marked for deletion saves SetDecisions and returns True.""" + mocker.patch.object(review, '_get_user_decisions_data', return_value=None) + with app.app_context(): + mock_save = mocker.patch.object(app.api, 'save') + params = MultiDict([('selected_files', 'paper.tex')]) + result = review._update_preflight( + params, 'sub1', _make_workspace('paper.tex', 'other.tex'), + authorized_user, None, + ) + assert result is True + mock_save.assert_called_once() + cmd = mock_save.call_args[0][0] + assert cmd.files_to_delete == ['paper.tex'] + assert mock_save.call_args[1] == {'submission_id': 'sub1'} + + +def test_update_preflight_decisions_unchanged_returns_false( + app, authorized_user, mocker): + """Form fields identical to stored user_decisions: no-op, no save.""" + existing = { + 'sources': [{'filename': 'main.tex'}], + 'texlive_version': '2025', + 'process': {'compiler': 'pdflatex'}, + } + mocker.patch.object(review, '_get_user_decisions_data', + return_value=existing) + with app.app_context(): + mock_save = mocker.patch.object(app.api, 'save') + params = MultiDict([ + ('source_file', 'main.tex'), + ('compiler', 'pdflatex'), + ('compiler_version', '2025'), + ]) + result = review._update_preflight( + params, 'sub1', _make_workspace('main.tex'), + authorized_user, None, + ) + assert result is False + mock_save.assert_not_called() + + +def test_update_preflight_decisions_changed_triggers_save( + app, authorized_user, mocker): + """Form fields differ from stored user_decisions: SetDecisions saved.""" + existing = { + 'sources': [{'filename': 'main.tex'}], + 'texlive_version': '2024', + 'process': {'compiler': 'pdflatex'}, + } + mocker.patch.object(review, '_get_user_decisions_data', + return_value=existing) + with app.app_context(): + mock_save = mocker.patch.object(app.api, 'save') + params = MultiDict([ + ('source_file', 'main.tex'), + ('compiler', 'pdflatex'), + ('compiler_version', '2025'), + ]) + result = review._update_preflight( + params, 'sub1', _make_workspace('main.tex'), + authorized_user, None, + ) + assert result is True + mock_save.assert_called_once() + cmd = mock_save.call_args[0][0] + assert cmd.decisions['texlive_version'] == '2025' + assert cmd.files_to_delete == [] + + +def test_update_preflight_no_existing_decisions_triggers_save( + app, authorized_user, mocker): + """First-time form submission (no stored decisions) saves SetDecisions.""" + mocker.patch.object(review, '_get_user_decisions_data', return_value=None) + with app.app_context(): + mock_save = mocker.patch.object(app.api, 'save') + params = MultiDict([ + ('source_file', 'main.tex'), + ('compiler', 'pdflatex'), + ('compiler_version', '2025'), + ]) + result = review._update_preflight( + params, 'sub1', _make_workspace('main.tex'), + authorized_user, None, + ) + assert result is True + mock_save.assert_called_once() + + +def test_update_preflight_invalid_event_returns_false( + app, authorized_user, mocker): + """If api.save raises InvalidEvent, it's swallowed and the call returns False.""" + mocker.patch.object(review, '_get_user_decisions_data', return_value=None) + with app.app_context(): + mock_save = mocker.patch.object( + app.api, 'save', + side_effect=InvalidEvent(MagicMock(), "nope"), + ) + params = MultiDict([ + ('source_file', 'main.tex'), + ('compiler', 'pdflatex'), + ('compiler_version', '2025'), + ]) + result = review._update_preflight( + params, 'sub1', _make_workspace('main.tex'), + authorized_user, None, + ) + assert result is False + mock_save.assert_called_once() + + +def test_populate_form_choices_come_from_enums_and_preflight(app): + """compiler/compiler_version choices come from Compilation enums; + source_file choices come from preflight tex_files.""" + preflight = {'tex_files': [{'filename': 'main.tex'}, {'filename': 'sup.tex'}]} + with app.app_context(): + form = review.ReviewForm(MultiDict(), meta={'csrf': False}) + review._populate_form(form, preflight, None) + assert form.compiler.choices == [ + (c.value, c.value) for c in Compilation.SupportedCompiler + ] + assert form.compiler_version.choices == [ + (v.value, f'TeX Live {v.value}') for v in Compilation.CompilerVersion + ] + assert form.source_file.choices == [('main.tex', 'main.tex'), + ('sup.tex', 'sup.tex')] + + +def test_populate_form_source_file_prefers_user_decisions(app): + """user_decisions.sources[0].filename wins over preflight detected_toplevel_files.""" + preflight = { + 'tex_files': [{'filename': 'main.tex'}, {'filename': 'chosen.tex'}], + 'detected_toplevel_files': [{'filename': 'main.tex'}], + } + user_decisions = {'sources': [{'filename': 'chosen.tex'}]} + with app.app_context(): + form = review.ReviewForm(MultiDict(), meta={'csrf': False}) + review._populate_form(form, preflight, user_decisions) + assert form.source_file.data == 'chosen.tex' + + +def test_populate_form_source_file_falls_back_to_preflight_toplevel(app): + """With no user_decisions, source_file falls back to preflight detected_toplevel_files.""" + preflight = { + 'tex_files': [{'filename': 'main.tex'}], + 'detected_toplevel_files': [{'filename': 'main.tex'}], + } + with app.app_context(): + form = review.ReviewForm(MultiDict(), meta={'csrf': False}) + review._populate_form(form, preflight, None) + assert form.source_file.data == 'main.tex' + + +def test_populate_form_compiler_and_version_from_user_decisions(app): + """user_decisions.process.compiler and process.compiler_version are used when present.""" + preflight = {'tex_files': []} + user_decisions = { + 'process': {'compiler': 'xelatex', 'compiler_version': '2023'}, + } + with app.app_context(): + form = review.ReviewForm(MultiDict(), meta={'csrf': False}) + review._populate_form(form, preflight, user_decisions) + assert form.compiler.data == 'xelatex' + assert form.compiler_version.data == '2023' + + +def test_populate_form_compiler_version_falls_back_to_texlive_version(app): + """If process.compiler_version is missing, the top-level texlive_version is used.""" + preflight = {'tex_files': []} + user_decisions = { + 'process': {'compiler': 'pdflatex'}, + 'texlive_version': '2023', + } + with app.app_context(): + form = review.ReviewForm(MultiDict(), meta={'csrf': False}) + review._populate_form(form, preflight, user_decisions) + assert form.compiler_version.data == '2023' + + +def test_populate_form_defaults_when_no_user_decisions(app): + """With no user_decisions, compiler defaults to PDFLATEX and version to TEXLIVE_2025.""" + preflight = {'tex_files': []} + with app.app_context(): + form = review.ReviewForm(MultiDict(), meta={'csrf': False}) + review._populate_form(form, preflight, None) + assert form.compiler.data == Compilation.SupportedCompiler.PDFLATEX.value + assert form.compiler_version.data == Compilation.CompilerVersion.TEXLIVE_2025.value + + +def _mock_file_store(app, mocker, directives_exist): + """Wire app.api.get_file_store() to a stub whose does_directives_exist + returns the given bool.""" + store = MagicMock() + store.does_directives_exist.return_value = directives_exist + mocker.patch.object(app.api, 'get_file_store', return_value=store) + return store + + +def test_get_notifications_preflight_and_directives(app, mocker): + """Both preflight present and directives ready: two success notifications.""" + with app.app_context(): + _mock_file_store(app, mocker, directives_exist=True) + notes = review._get_notifications('sub1', {'tex_files': []}) + titles = [n['title'] for n in notes] + severities = [n['severity'] for n in notes] + assert titles == ['Preflight complete', 'Directives ready'] + assert severities == ['success', 'success'] + + +def test_get_notifications_preflight_only(app, mocker): + """Preflight present, directives missing: complete + pending.""" + with app.app_context(): + _mock_file_store(app, mocker, directives_exist=False) + notes = review._get_notifications('sub1', {'tex_files': []}) + assert [n['title'] for n in notes] == ['Preflight complete', 'Directives pending'] + assert [n['severity'] for n in notes] == ['success', 'info'] + + +def test_get_notifications_directives_only(app, mocker): + """No preflight but directives ready: pending warning + success.""" + with app.app_context(): + _mock_file_store(app, mocker, directives_exist=True) + notes = review._get_notifications('sub1', None) + assert [n['title'] for n in notes] == ['Preflight pending', 'Directives ready'] + assert [n['severity'] for n in notes] == ['warning', 'success'] + + +def test_get_notifications_nothing_ready(app, mocker): + """Neither preflight nor directives: two pending notifications.""" + with app.app_context(): + store = _mock_file_store(app, mocker, directives_exist=False) + notes = review._get_notifications('sub1', None) + assert [n['title'] for n in notes] == ['Preflight pending', 'Directives pending'] + assert [n['severity'] for n in notes] == ['warning', 'info'] + store.does_directives_exist.assert_called_once_with('sub1') + + +def test_store_source_format_none_preflight_is_noop(app, mocker): + """No preflight_data: nothing read, nothing saved.""" + mock_get_lang = mocker.patch.object(review.dm, 'get_lang_from_preflight') + with app.app_context(): + mock_save = mocker.patch.object(app.api, 'save') + review._store_source_format(None, MagicMock(), 'sub1') + mock_get_lang.assert_not_called() + mock_save.assert_not_called() + + +def test_store_source_format_none_lang_is_noop(app, mocker): + """Preflight present but no detected lang: no save.""" + mocker.patch.object(review.dm, 'get_lang_from_preflight', return_value=None) + mock_session = mocker.patch.object(review, 'user_and_client_from_session') + with app.app_context(): + mock_save = mocker.patch.object(app.api, 'save') + review._store_source_format({'tex_files': []}, MagicMock(), 'sub1') + mock_save.assert_not_called() + mock_session.assert_not_called() + + +def test_store_source_format_saves_command(app, authorized_user, mocker): + """Detected lang is saved as SetSourceFormat on the submission.""" + mocker.patch.object(review.dm, 'get_lang_from_preflight', return_value='tex') + mocker.patch.object(review, 'user_and_client_from_session', + return_value=(authorized_user, None)) + with app.app_context(): + mock_save = mocker.patch.object(app.api, 'save') + review._store_source_format({'tex_files': []}, MagicMock(), 'sub1') + mock_save.assert_called_once() + cmd = mock_save.call_args[0][0] + assert isinstance(cmd, SetSourceFormat) + assert cmd.source_format == 'tex' + assert mock_save.call_args[1] == {'submission_id': 'sub1'} + + +def test_store_source_format_swallows_save_error(app, authorized_user, mocker, caplog): + """SaveError from api.save is logged as a warning, not raised.""" + mocker.patch.object(review.dm, 'get_lang_from_preflight', return_value='tex') + mocker.patch.object(review, 'user_and_client_from_session', + return_value=(authorized_user, None)) + with app.app_context(): + mocker.patch.object(app.api, 'save', + side_effect=SaveError("boom")) + review._store_source_format({'tex_files': []}, MagicMock(), 'sub1') + assert any('Could not save SetSourceFormat for sub1' in r.message + for r in caplog.records) From c9f8df8b07f495e5a1d831b8198544be8bddfb09 Mon Sep 17 00:00:00 2001 From: Brian Maltzan Date: Mon, 8 Jun 2026 14:32:56 -0400 Subject: [PATCH 2/6] SUBMISSION-16: test review controller --- .../ui/controllers/new/tests/test_review.py | 108 ++++++++++++++++++ 1 file changed, 108 insertions(+) diff --git a/submit_ce/ui/controllers/new/tests/test_review.py b/submit_ce/ui/controllers/new/tests/test_review.py index d48f6280..ee0a3053 100644 --- a/submit_ce/ui/controllers/new/tests/test_review.py +++ b/submit_ce/ui/controllers/new/tests/test_review.py @@ -3,8 +3,11 @@ from http import HTTPStatus as status from unittest.mock import MagicMock +import pytest from werkzeug.datastructures import MultiDict +from submit_ce.ui.tests.csrf_util import parse_csrf_token + from submit_ce.domain.compilation import Compilation from submit_ce.domain.event import SetSourceFormat from submit_ce.domain.exceptions import InvalidEvent, SaveError @@ -27,6 +30,111 @@ def test_review_files_get_warning_via_http(app, authorized_client, sub_files, assert "couldn't load preflight data" in mock_flash.call_args[0][0] + +def test_review_files_empty_workspace_skips_preflight( + app, authorized_client, sub_files, mocker): + """End-to-end: when get_workspace returns None, the controller short-circuits + via return_to_parent_stage and never calls _load_or_create_preflight.""" + mock_store = MagicMock() + mock_store.get_workspace.return_value = None + mocker.patch.object(app.api, 'get_file_store', return_value=mock_store) + mock_load = mocker.patch.object(review, '_load_or_create_preflight') + + url = f"/{sub_files.submission_id}/review_files" + resp = authorized_client.get(url) + + assert resp.status_code == status.OK + mock_load.assert_not_called() + + +def test_review_files_unsupported_method_raises(mocker): + """Controller raises MethodNotAllowed for methods other than GET/POST.""" + mocker.patch.object(review, 'user_and_client_from_session', + return_value=(MagicMock(), None)) + with pytest.raises(review.MethodNotAllowed): + review.review_files('PUT', MultiDict(), MagicMock(), + 'sub1', 'tok') + + +def _get_csrf(authorized_client, url, mocker): + """GET the review page (with preflight stubbed out) and pull the CSRF token.""" + mocker.patch.object(review, '_load_or_create_preflight', + return_value=(None, None)) + resp = authorized_client.get(url) + return parse_csrf_token(resp) + + +def test_review_files_post_with_changes_redirects_to_parent( + app, authorized_client, sub_files, mocker): + """End-to-end POST: when _update_preflight reports changes, the controller + marks STAGE_PARENT and the flow redirects (303 SEE_OTHER).""" + url = f"/{sub_files.submission_id}/review_files" + csrf = _get_csrf(authorized_client, url, mocker) + + mock_update = mocker.patch.object(review, '_update_preflight', + return_value=True) + mock_load_dir = mocker.patch.object(review, '_load_or_create_directives') + + resp = authorized_client.post(url, data={'csrf_token': csrf, 'action': 'next'}) + + assert resp.status_code == status.SEE_OTHER + mock_update.assert_called_once() + mock_load_dir.assert_not_called() + + +def test_review_files_post_no_changes_no_preflight_flashes( + app, authorized_client, sub_files, mocker): + """End-to-end POST: when there are no changes but preflight is still + unavailable, the controller flashes a warning and stays on the stage.""" + url = f"/{sub_files.submission_id}/review_files" + csrf = _get_csrf(authorized_client, url, mocker) + + mocker.patch.object(review, '_update_preflight', return_value=False) + mocker.patch.object(review, '_load_or_create_directives') + # Re-patch _load_or_create_preflight: GET used (None, None) for CSRF, POST + # needs the same so we hit the "preflight unavailable" branch. + mocker.patch.object(review, '_load_or_create_preflight', + return_value=(None, None)) + mock_flash = mocker.patch.object(review.alerts, 'flash_warning') + + resp = authorized_client.post(url, data={'csrf_token': csrf, 'action': 'next'}) + + assert resp.status_code == status.OK + assert mock_flash.called + assert "Preflight data is not available" in mock_flash.call_args[0][0] + + +def test_review_files_post_no_changes_stores_zzrm_and_advances( + app, authorized_client, sub_files, mocker): + """End-to-end POST: when there are no changes and preflight is present, + the controller stores the merged zzrm and advances to the next stage.""" + url = f"/{sub_files.submission_id}/review_files" + csrf = _get_csrf(authorized_client, url, mocker) + + mocker.patch.object(review, '_update_preflight', return_value=False) + mocker.patch.object(review, '_load_or_create_directives') + mocker.patch.object(review, '_load_or_create_preflight', + return_value=({'tex_files': []}, {'sources': []})) + + # Stub the external preflight/zzrm types to avoid building real fixtures. + mocker.patch.object(review, 'PreflightResponse') + fake_zzrm = MagicMock() + fake_zzrm.to_dict.return_value = {'merged': True} + mocker.patch.object(review, 'ZeroZeroReadMe', return_value=fake_zzrm) + + mock_store = MagicMock() + mocker.patch.object(app.api, 'get_file_store', return_value=mock_store) + + resp = authorized_client.post(url, data={'csrf_token': csrf, 'action': 'next'}) + + assert resp.status_code == status.SEE_OTHER + fake_zzrm.from_dict.assert_called_once_with({'sources': []}) + fake_zzrm.update_from_preflight.assert_called_once() + mock_store.store_zzrm.assert_called_once_with( + str(sub_files.submission_id), {'merged': True} + ) + + def _make_workspace(*paths): """Build a stand-in workspace whose `.files` carry the given paths.""" ws = MagicMock() From d6086a906f8032c64a226a2d65e89a1de73ee434 Mon Sep 17 00:00:00 2001 From: Brian Maltzan Date: Mon, 8 Jun 2026 14:37:58 -0400 Subject: [PATCH 3/6] SUBMISSION-16: replaced by tex2pdf-api --- .../implementations/compile/compile_at_gcp.py | 833 ------------------ 1 file changed, 833 deletions(-) delete mode 100644 submit_ce/implementations/compile/compile_at_gcp.py diff --git a/submit_ce/implementations/compile/compile_at_gcp.py b/submit_ce/implementations/compile/compile_at_gcp.py deleted file mode 100644 index 3419a224..00000000 --- a/submit_ce/implementations/compile/compile_at_gcp.py +++ /dev/null @@ -1,833 +0,0 @@ -""" -Compile (La)TeX source submissions at GCP. - -This script provides an interface between the legacy submission -and the GCP compilation system. It makes a request to compile a submission -at GCP and then installs the resulting PDF and log in the submission -directory. -""" - -import os -import sys -import json -import tarfile -import glob -import tempfile -import shutil -import argparse -import time -import logging -import stat -import httpx -import urllib.parse -from typing import List, Optional -from enum import Enum - -from .common import (GCP_LOG_NAME, GCP_RESULTS_NAME, GCP_PREFLIGHT_NAME, - DEFAULT_SUBMISSION_LOG_NAME, DEFAULT_SYSTEM_LOG_NAME, - DEFAULT_COMPILATION_TIMEOUT, DEFAULT_MAX_APPEND_FILES, - DEFAULT_MAX_TEX_FILES, MAX_RETRIES, RETRY_DELAY) - - -DEFAULT_SYSTEM_LOGS_DIR = '/users/e-prints/httpd/logs' - -ENABLE_HTML_MARKUP = 0 - -# Enum for preflight options -class PreflightOption(str, Enum): - V1 = "v1" - V2 = "v2" - - -def parse_preflight_option(option): - if option is None: - return None - return PreflightOption(option) - - -# Create loggers -logger = logging.getLogger(__name__) - -# Create formatters -formatter = logging.Formatter('%(asctime)s - %(levelname)s - %(message)s', - datefmt='%Y-%m-%d %H:%M:%S') - -# Configure logging -logging.basicConfig(level=logging.DEBUG) - - -def get_submission_dir(identifier, base_submissions_dir=None): - """Generate the submission's home directory path.""" - base_submission_dir = os.path.join(base_submissions_dir, identifier[:4], identifier) - return base_submission_dir - - -def adjust_permissions(file_path): - """Add write permissions to file.""" - # Get the current file permissions - current_permissions = os.stat(file_path).st_mode - - # Add group write permissions (g+w) - new_permissions = current_permissions | stat.S_IWGRP - - # Update the file permissions - if not current_permissions & stat.S_IWGRP: - os.chmod(file_path, new_permissions) - - -def set_up_logging(identifier, submission_log_name=DEFAULT_SUBMISSION_LOG_NAME, - system_logs_dir=DEFAULT_SYSTEM_LOGS_DIR, - system_log_name=DEFAULT_SYSTEM_LOG_NAME, - base_submissions_dir=None, - log_to_console=False, - only_log_to_console=False): - """Set up submission, system, and console logging.""" - - # Remove all existing handlers from the logger - for handler in logger.handlers[:]: - logger.removeHandler(handler) - - if log_to_console: - - # Create a stream handler (logs to console) - console_handler = logging.StreamHandler(sys.stdout) - console_handler.setLevel(logging.DEBUG) - console_handler.setFormatter(formatter) - logger.addHandler(console_handler) - - if only_log_to_console: - return logger - - # Make sure various directories exist - if not base_submissions_dir or base_submissions_dir is None: - raise TypeError("Base submission directory is required.") - if not os.path.exists(base_submissions_dir): - raise FileNotFoundError(f"The base submission directory " - f"'{base_submissions_dir}' does not exist.") - if not os.path.exists(system_logs_dir): - raise FileNotFoundError(f"The system log directory '{system_logs_dir}' " - f"does not exist.") - - # Create a log for this submission in the submission's home directory - submission_log_dir = get_submission_dir(identifier, base_submissions_dir) - if not os.path.exists(submission_log_dir): - raise FileNotFoundError(f"The submission directory '{submission_log_dir}' does not exist.") - - submission_log_path = os.path.join(submission_log_dir, submission_log_name) - - # Create system log - system_log_path = os.path.join(system_logs_dir, system_log_name) - - # Create file handlers for this submission's compilation messaged - file_handler_submission = logging.FileHandler(submission_log_path) - file_handler_submission.setLevel(logging.DEBUG) - file_handler_submission.setFormatter(formatter) - file_handler_submission.mode = 0o664 - logger.addHandler(file_handler_submission) - - # Create file handlers for system level messages - file_handler_system = logging.FileHandler(system_log_path) - file_handler_system.setLevel(logging.INFO) - file_handler_system.setFormatter(formatter) - file_handler_system.mode = 0o664 - logger.addHandler(file_handler_system) - - adjust_permissions(system_log_path) - adjust_permissions(submission_log_path) - return logger - - -def find_and_process_log_file(output_files_dir): - """Process the log file in the out directory as a last resort. - - The 'out' directory contains a log file. We will use this - when we are unable to extract a log from the compilation metadata (json). - """ - log_files = glob.glob(os.path.join(output_files_dir, 'out', '*.log')) - if log_files: - log_file_path = log_files[0] # Take the first log file found - return log_file_path # Return log file path - - return None - - -def process_file_lists(list_of_listfiles: List, output_files_dir: str): - """Process the list of files involved it the compilation. - - """ - processed_files = {} - for current_file in list_of_listfiles: - path = os.path.join(output_files_dir, 'out', current_file) - if os.path.exists(path): - processed_files[current_file] = {"input": [], "output": []} - processed_input_files = set() - processed_output_files = set() - with open(path, 'r') as f: - lines = f.readlines() - for line in lines: - clean_line = line.strip() - if 'texmf' in clean_line: - continue - if clean_line.startswith('INPUT'): - input_file = clean_line[len('INPUT'):].strip() - if input_file.startswith("./"): - input_file = input_file[2:] - input_file = os.path.normpath(input_file) - if input_file not in processed_input_files: - processed_files[current_file]["input"].append(input_file) - processed_input_files.add(input_file) - elif clean_line.startswith('OUTPUT'): - output_file = clean_line[len('OUTPUT'):].strip() - output_file = os.path.normpath(output_file) - if output_file.startswith("./"): - output_file = input_file[2:] - if output_file not in processed_output_files: - processed_files[current_file]["output"].append(output_file) - processed_output_files.add(output_file) - - # After we finish processing the file we need to eliminate the set - # by serializing the data - processed_files[current_file]["input"] = list(processed_input_files) - processed_files[current_file]["output"] = list(processed_output_files) - - return processed_files - - -def add_html_class(css_class: str, content: str) -> str: - """Return HTML markup for display.""" - output = content - if ENABLE_HTML_MARKUP: - output = f"{content}" - return output - -def process_metadata_and_log(submission_dir, json_log_run_data, output_files_dir, - json_data=None, json_file_path: Optional[str] = None): - """Create a brief summary. Provide indication of success or failure. - - Use the log data from the last compilation run in - the compilation metadata (json). If this fails, grab the log - file in the out directory. - - :param submission_dir: Location of submission directory. - :param json_log_run_data: Used a log data when there is a fatal error and we don't have a log. - :param output_files_dir: The directory where the compilation results gzipped tar is unpacked. - :param json_data: The original compilation metadata (JSON). - :param json_file_path: The path for the installed compilation metadata (JSON). - """ - # Install the log file into the submission directory - new_log_path = os.path.join(submission_dir, GCP_LOG_NAME) - - if json_data and not json_file_path: - raise FileNotFoundError("File path is require to write out JSON compilation metadata.") - - if json_data is None: - # Create a minimal log file for a fatal error - with open(new_log_path, "w") as f: - display_status = add_html_class('tex-fatal', "FAILED") - f.write(f"Status: {display_status}\n\n") - f.write(f"{json_log_run_data}\n\n") - os.chmod(new_log_path, 0o664) - logger.debug("Creating compilation log file from supplied log data.") - elif json_data: - converter_full = json_data.get("converter", "") - converter_name = converter_full.split(":")[0] if ":" in converter_full else converter_full - converter_phrase = f" using {converter_name}." if converter_name else "." - - final_pdf_file = json_data.get("pdf_file", None) - logger.debug("Creating compilation log file from last run log in json " - "metadata: %s", new_log_path) - - with open(new_log_path, "w") as f: - f.write("Compilation Summary") - converters = json_data.get("converters", []) - num_conversions = len(converters) - num_failed = sum(1 for c in converters if isinstance(c, dict) and c.get("status") == "fail") - - if num_failed == 0: - display_status = add_html_class('tex-success', "[SUCCEEDED]") - f.write(f"\nWe successfully processed your submission{converter_phrase} Status: {display_status}\n\n") - else: - error_details = '' - if num_conversions > 1: - error_details =f"({num_failed} out of {num_conversions} conversions failed). " - display_status = add_html_class('tex-fatal', "[FAILED]") - f.write(f"\nOur system failed to process your submission" - f"{converter_phrase} {error_details} " - f"Status: {display_status}\n\n") - - - - f.write("\nFiles that were processed as part of this submission:\n\n") - for converter in json_data.get("converters", []): - if isinstance(converter, dict): - converter_status = converter.get("status", "N/A") - tex_file = converter.get("tex_file", "N/A") - pdf_file = converter.get("pdf_file", "N/A") - display_status = add_html_class('tex-success', "[SUCCEEDED]") - if converter_status == 'fail': - display_status = add_html_class('tex-fatal', "[FAILED]") - - f.write(f" {tex_file} => {pdf_file} {display_status}\n") - if converter_status == 'fail': - step = converter.get("step", "N/A") - reason = converter.get("reason", "No reason provided") - f.write(f"...at step \"{step}\"\n...reason for failure: \"{reason}\"\n") - - - elif isinstance(converter, str): - # TeX2PDF returns strings when run in preflight mode. - # TODO: The set of converters will be used to populate the - # TeX engine selection pulldown in the submission UI - pass - - f.write('\n') - - if num_failed == 0: - if len(json_data.get("converters", [])) > 1: - f.write(f"\nOur system has compiled the above LaTeX files into " - f"individual PDFs and merged them into a single " - f"PDF document: {final_pdf_file}.\n\n") - else: - f.write(f"\nOur system has compiled the above LaTeX file " - f"into a single PDF document: {final_pdf_file}.\n\n") - else: - f.write("\nOur system failed to generate a PDF.\n\n") - - # f.write(f"Selected Errors and Warnings\n") - - f.write("Logs for the last run of each file processed\n\n") - - for converter in json_data.get("converters", []): - if isinstance(converter, dict): - tex_file = converter.get("tex_file", "N/A") - step = converter.get("step", "N/A") - converter_status = converter.get("status", "N/A") - - display_status = add_html_class('tex-success', "[SUCCEEDED]") - if converter_status == 'fail': - display_status = add_html_class('tex-fatal', "[FAILED]") - - # Get the latest log in the series - latest_log = None - latest_step = "N/A" - for run in reversed(converter["runs"]): - if "log" in run: - latest_log = run["log"] - latest_step = run["step"] - break - - log = latest_log if latest_log else "Log not available" - step = latest_step if latest_log else "Step not available" - f.write(f"Log for {tex_file} at step '{step}': {display_status}\n\n") - f.write(f"--\n{log}\n--\n") - - logger.debug("\nCompilation: status: %s step: %s PDF file: %s TeX file: %s", - converter_status, step, pdf_file, tex_file) - - os.chmod(new_log_path, 0o664) - - # Collect list of files used in compilations - item: str - list_files = [] - for item in json_data.get("out_files", []): - if item.endswith('.fls'): - list_files.append(item) - - processed_files = process_file_lists(list_files, output_files_dir) - - json_data['processed_files'] = processed_files - - logger.debug(f"Writing new compilation metadata file (JSON): {json_file_path}") - new_metadata_path = os.path.join(submission_dir, GCP_RESULTS_NAME) - with open(new_metadata_path, "w") as f: - f.write(json.dumps(json_data, indent=4)) - os.chmod(new_metadata_path, 0o664) - else: - # Use existing log file (in 'out' directory) - log_file_path = find_and_process_log_file(output_files_dir) - if log_file_path and os.path.exists(log_file_path): - try: - shutil.copy2(log_file_path, new_log_path) - os.chmod(new_log_path, 0o664) - except OSError as e: - logger.error("Failed to copy log file to submission directory: %s", e) - else: - # No log file found - pass - - -def save_response_output(response, output_file_path): - """Save the response content from the compilation service. - - Return path when successful, otherwise return None. - """ - if response is not None and response.content: - # Write the content to the output file in the temp_dir - with open(output_file_path, 'wb') as out_file: - out_file.write(response.content) - os.chmod(output_file_path, 0o664) - - logger.debug("Output file created: %s", output_file_path) - - # Check if the output file (tarfile) exists. - if not os.path.exists(output_file_path) or os.path.getsize(output_file_path) == 0: - logger.critical("Output file '%s' not found or has zero size.", output_file_path) - return None # Return None for both files - - return output_file_path - return None - - -def extract_output_files(output_file_path, temp_dir): - """Extract files from the gzipped tar output file.""" - with tarfile.open(output_file_path, 'r:gz') as tar: - tar.extractall(temp_dir) - - -def copy_json_file_to_submission_directory(base_submissions_dir, json_file_path, identifier): - """Install the compilation response metadata into the submission directory. - - We preserve the json just in case we need to look at it or use it after - the temporary directory is cleaned up. - """ - base_submission_dir = os.path.join(base_submissions_dir, identifier[:4], identifier) - - # Use identifier as the new name for the JSON file - new_json_name = GCP_RESULTS_NAME - - # Install JSON file into base_submission_dir with the new name - new_json_path = os.path.join(base_submission_dir, new_json_name) - try: - shutil.copy2(json_file_path, new_json_path) - os.chmod(new_json_path, 0o664) - except OSError as e: - logger.error("Failed to copy JSON file to %s: %s", new_json_path, e) - - -def find_pdf_file(output_files_dir, pdf_file): - """Use the provided pdf_file value directly.""" - if pdf_file: - pdf_file_path = os.path.join(output_files_dir, 'out', pdf_file) - if os.path.exists(pdf_file_path): - return pdf_file_path - return None - - -def _compile_submission(args: argparse.Namespace): - """Compile (La)TeX source at GCP. - - Make a request to compile a submission at GCP, then install resulting - pdf, log, and json files into the submission's home directory. - - In standard production environment we only need the identifier to determine all - paths. - :param identifier: The submission identifier. - :param base_submissions_dir: Base directory to look for submissions and other related files. (Default is /data/new) - :param: max_append_files : Limit on extra files apended to final PDF. Default is 0. - :param: max_tex_files : Maximum number of (La)TeX source files to compile. Default is 1. - :param output_file: Name of file to store gzipped response from tex2pdf service. - :param preflight: Execute light-weight preflight check instead of compiling document. - :param source_file: A gzipped tarball containing the source to be compiled. - :param tex2pdf_url: The URL to the tex2pdf service. - :param timeout: A user specified timeout for tex2pdf service. (Default is 290 seconds) - :param: watermark_text : Text to use for watermark. - """ - identifier = args.identifier - source_file = args.source if args.source is not None else None - output_file = args.output if args.output is not None else None - # options - preflight = parse_preflight_option(args.preflight) - # thresholds - base_submissions_dir = args.base if args.base is not None else '/data/new' - max_append_files = args.max_append_files if args.max_append_files is not None else DEFAULT_MAX_APPEND_FILES - max_tex_files = args.max_tex_files if args.max_tex_files is not None else DEFAULT_MAX_TEX_FILES - tex2pdf_url = args.tex2pdf_url - timeout = args.timeout if args.timeout is not None else DEFAULT_COMPILATION_TIMEOUT - watermark_text = args.watermark_text if args.watermark_text is not None else None - - return compile_submission(identifier, source_file, output_file, tex2pdf_url, - base_submissions_dir, preflight, watermark_text, - max_append_files, max_tex_files, timeout) - -def compile_submission( - identifier: str, - source_file: str, - output_file: str, - tex2pdf_url: str, - base_submissions_dir: str = "/data/new", - - preflight: Optional[PreflightOption] = None, - watermark_text: Optional[str] = None, - max_append_files: int = DEFAULT_MAX_APPEND_FILES, - max_tex_files: int = DEFAULT_MAX_TEX_FILES, - timeout: int = DEFAULT_COMPILATION_TIMEOUT, -): - - - if preflight: - logger.info("Processing preflight request for '%s' at GCP", identifier) - else: - logger.info("Processing compilation request for '%s' at GCP", identifier) - - # additional parameters - query_params = { - 'timeout': timeout, - 'max_appending_files': max_append_files, - 'max_tex_files': max_tex_files, - } - if watermark_text: - query_params['watermark_text'] = watermark_text - - if preflight: - query_params['preflight'] = preflight - - if not tex2pdf_url: - raise FileNotFoundError("The tex2pdf_url is required. ") - - url = f'{tex2pdf_url}/convert/?{urllib.parse.urlencode(query_params)}' - logger.info("TeX2PDF request url '%s'", url) - headers = { - 'accept': 'application/json', - } - - # Create a unique temporary directory that only exists during the - # execution of the script. - # - # Note: newer versions of Python support delete=False option to - # preserve temporary directory for debugging purposes - temp_dir_prefix = f"temp_compile_{identifier}_" - with tempfile.TemporaryDirectory(prefix=temp_dir_prefix, suffix='_dir') as temp_dir: - - # Create the output file path within the temp_dir - if output_file.startswith(os.path.sep): - # Support this for debugging purposes - output_file_path = output_file - else: - output_file_path = os.path.join(temp_dir, output_file) - response = None - - # We need either the latest source directory or a gzipped tar file - # Specifying gzipped tar file will overide the default action of - # grabbing the source. - - # Locate source - base_submission_dir = os.path.join(base_submissions_dir, - str(identifier)[:4], str(identifier)) - if not os.path.exists(base_submission_dir): - raise FileNotFoundError(f"The base directory " - f"'{base_submission_dir}' does not exist.") - source_dir = os.path.join(base_submission_dir, "src") - - # Check if the incoming file or source directory exists - temp_tar_path = '' - - try: - if source_file: - if not os.path.exists(source_file): - # We will consider this a fatal error - emsg = f"Source specified but not found: {source_file}" - raise FileNotFoundError(emsg) - - incoming_file = source_file - logger.info("Using client-specified output " - "filename: %s", source_file) - else: - if not os.path.exists(source_dir): - emsg = "Source directory not found in submission home directory: %s" - logger.error(emsg, source_dir) - raise FileNotFoundError(emsg) - - logger.info("Using submission source directory: %s", source_dir) - temp_tar_path = os.path.join(temp_dir, f"{identifier}.tar.gz") - with tarfile.open(temp_tar_path, "w:gz") as tar: - tar.add(source_dir, arcname="") - os.chmod(temp_tar_path, 0o664) - incoming_file = temp_tar_path - - except FileNotFoundError as e: - logger.critical("The request to compile document '%s' " - "at GCP failed: %s", identifier, e) - return None, None - - files = {'incoming': (os.path.basename(incoming_file), - open(incoming_file, 'rb'), 'application/gzip')} - - # Use identifier as the new base name for PDF file - new_pdf_name = f"{identifier}.pdf" - new_pdf_path = os.path.join(base_submission_dir, new_pdf_name) - new_log_path = os.path.join(base_submission_dir, GCP_LOG_NAME) - new_json_path = os.path.join(base_submission_dir, GCP_RESULTS_NAME) - new_tar_path = os.path.join(base_submission_dir, f"{identifier}.tar.gz") - new_preflight_path = os.path.join(base_submission_dir, GCP_PREFLIGHT_NAME) - - # Remove all prior compilation output prior to making request - # for new compilation - - # Note: When running preflight, we must determine whether we need to - # delete any existing compilation output files. Calling preflight in itself - # does not guarantee that the resulting PDF will change. - # - # For now, we will delete existing compilation output files. - try: - if os.path.exists(new_pdf_path): - os.remove(new_pdf_path) - if os.path.exists(new_log_path): - os.remove(new_log_path) - if os.path.exists(new_json_path): - os.remove(new_json_path) - if preflight and os.path.exists(new_preflight_path): - os.remove(new_preflight_path) - - except PermissionError as e: - logger.error("You have insufficient permissions to delete file: %s.", e) - except Exception as e: - logger.error("There was an exception while deleting {file}: %s", e) - - json_data = None - - try: - # Create a temporary message and replace previous log. - log_msg = f"The system will now compile your (La)TeX source for document " \ - f"'{identifier}'\n\n" \ - "This may take a few minutes. Please be patient.\n\n" \ - "If the results are not displayed after several minutes, " \ - "please try again or contact the arXiv user support team at https://arxiv.org/support/submission_tex." - if preflight: - log_msg = f"The system will now run a 'preflight' process to check " \ - f"your (La)TeX source files for document '{identifier}'\n\n" \ - "This check will attempt to identify potential problems " \ - "with your submission.\n\n" - - process_metadata_and_log(base_submission_dir, log_msg, temp_dir) - - # Perform the POST request using requests library - for retry_attempt in range(MAX_RETRIES): - try: - with httpx.Client(timeout=timeout) as client: - response = client.post(url, headers=headers, files=files) - if response.status_code == 500: - logger.warning("Retry attempt %s/%s for identifier %s.", - retry_attempt + 1, MAX_RETRIES, identifier) - time.sleep(RETRY_DELAY) - else: - break # Exit the loop if the request succeeds - except httpx.HTTPStatusError as exc: - # Handle HTTP status errors - logger.error(f"HTTPX error occurred: {exc.response.text}") - raise exc - except httpx.RequestError as exc: - # Handle request errors (e.g., connection errors) - logger.error(f"Request error occurred: {exc}") - - if response is None: - raise RuntimeError("response is unexpectedly None") - - if response.status_code == 500: - raise httpx.HTTPStatusError(f"HTTP error {response.status_code}", request=response.request, response=response) - - response.raise_for_status() - - # Call the function to save the response output - output_file_path = save_response_output(response, output_file_path) - - # Determine the type of response - content_type = response.headers['Content-Type'] - - # We expect 'application/gzip' for most convert endpoint responses (including preflight v1). - # and 'application/json' for preflight v2. - - # Check if the content type is "application/gzip" - if output_file_path: - - # For preflight v2 we simply save the preflight response into - # the submission home directory. - if preflight == 'v2': - if content_type == 'application/json': - status = 'success' - logger.info("Preflight check completed successfully for submission 'submit/%s'.", identifier) - # Copy new preflight data into submission directory - try: - # Read the JSON data from the request output file - with open(output_file_path, 'r') as json_file: - summary_data = json.load(json_file) - - # Format JSON data with indentation - formatted_json = json.dumps(summary_data, indent=2) - - # Save the formatted JSON data directly to the new preflight file - with open(new_preflight_path, 'w') as json_file: - json_file.write(formatted_json) - - # Update file permissions - os.chmod(new_preflight_path, 0o664) - except OSError as e: - logger.critical("ERROR: copying preflight data " - "from %s to %s: %s", output_file_path, new_preflight_path, e) - msg = f"The preflight check completed successfully for submission 'submit/{identifier}'." - process_metadata_and_log(base_submission_dir, msg, temp_dir) - return status, json.loads(response.content.decode()) - else: - # We need to further investigate error cases for preflight v2 endpoint, - # but a non-JSON response for a preflight v2 request is - # an obvious error. - error_msg = "Our system encountered an error while executing " \ - "preflight check for document %s." - logger.error(error_msg, identifier) - process_metadata_and_log(base_submission_dir, error_msg, temp_dir) - return None, None - - # Call the function to extract files from the output - extract_output_files(output_file_path, temp_dir) - - # Find and parse the JSON file - json_files = glob.glob(os.path.join(temp_dir, '*.json')) - if json_files: - json_file_path = json_files[0] # Take the first JSON file found - with open(json_file_path, 'r') as json_file: - json_data = json.load(json_file) - - if preflight: - logger.debug("Completed preflight check for %s", identifier) - status = 'success' - else: - # Fetch 'status' and 'output_files' values from the JSON - status = json_data.get('status') - output_files = json_data.get('output_files', {}) - pdf_file = json_data.get('pdf_file', None) - - logger.debug("Compilation Status: %s", status) - logger.debug("Output Files: %s", output_files) - logger.debug("PDF File: %s", pdf_file) - - # Install PDF and log file into base_submission_dir - - # Use the value of pdf_file directly - if pdf_file is not None: - pdf_file_path = find_pdf_file(temp_dir, pdf_file) - if pdf_file_path and os.path.exists(pdf_file_path): - if os.path.getsize(pdf_file_path) == 0: - logger.error("The PDF file exists but is 0 bytes in size: %s", - pdf_file_path) - else: - try: - shutil.copy2(pdf_file_path, new_pdf_path) - os.chmod(new_pdf_path, 0o664) - except OSError as e: - logger.error("There was an error copying PDF file: %s", e) - - elif pdf_file_path: - # This is acceptable when compilation fails - logger.error("No 'pdf_file' found in response.") - else: - # This is acceptable when compilation fails - logger.info("No 'pdf_file' found in response.") - - process_metadata_and_log(base_submission_dir, " ", temp_dir, json_data, json_file_path) - - # We currently update the gzipped tar file when a compilation succeeds. - if not preflight and status == 'success' and not source_file and os.path.exists(temp_tar_path): - # Copy new gzipped tarfile into submission directory - try: - shutil.copy2(temp_tar_path, new_tar_path) - os.chmod(temp_tar_path, 0o664) - except OSError as e: - logger.critical("ERROR: copying gziped tar file " - "from %s to %s: %s", temp_tar_path, new_tar_path, e) - - else: - # We should always get a json file with the response - logger.critical("No JSON file found.") - - else: - logger.error("Unexpected content type '%s in response.", - response.headers.get('content-type')) - return None, None - - if preflight: - operation = 'Preflight' - else: - operation = 'Compilation' - - if status == 'success': - logger.info(f"{operation} of document %s succeeded at GCP", identifier) - else: - logger.error(f"{operation} of document %s failed at GCP", identifier) - - except httpx.HTTPStatusError as e: - - error_details = ( - f"HTTP error {response.status_code}\n" - f"Response headers: {response.headers}\n" - f"Response content: {response.text}\n" - ) - - logger.critical("The request to process document %s at GCP " - "failed: %s", identifier, error_details) - error_msg = "There was fatal error during our attempt to process your " \ - f"document's source files(s).\n\nERROR: {e}\n\nPlease try " \ - "again or contact the arXiv editorial team." - process_metadata_and_log(base_submission_dir, error_msg, temp_dir) - return None, None - except tarfile.TarError as e: - error_msg = "Our system encountered an error extracting source " \ - "files for submission %s: %s" - logger.error(error_msg, identifier, e) - process_metadata_and_log(base_submission_dir, error_msg, temp_dir) - return None, None - - return status, json_data - - -def main(args): - """Setup routine for command-line invocation.""" - set_up_logging(identifier=args.identifier, system_logs_dir=args.logs_dir, - system_log_name='compile_at_gcp.log', - log_to_console=args.console, base_submissions_dir=args.base) - - # Call the compile function with the provided arguments - status, json_data = _compile_submission(args) - - # dump json to console if both verbose and debug mode are enabled - if args.console and json_data and args.debug and args.verbose: - set_up_logging(identifier=args.identifier, log_to_console=args.console, - only_log_to_console=True) - logger.debug("JSON file content:") - logger.debug(json.dumps(json_data, indent=2)) - logger.debug("Status: %s", status) - - -if __name__ == "__main__": - # Create an argument parser - parser = argparse.ArgumentParser(description="Process LaTeX to PDF conversion request") - - # Define command-line options - parser.add_argument("-a", "--max-append-files", type=int, default=4, - help="Maximum number of extra files to append to PDF (default=0)") - parser.add_argument("-i", "--identifier", required=True, help="Identifier for the request") - parser.add_argument("-m", "--max-tex-files", type=int, default=1, - help="Maximum number of text files to process (default=1)") - parser.add_argument("-o", "--output", required=True, help="Output gzipped tar file") - parser.add_argument("-p", "--preflight", nargs='?', const='v1', - choices=list(PreflightOption), - help="Preflight check option: 'v1', 'v2'. Defaults to 'v1' if " - "no argument is provided.") - parser.add_argument("-s", "--source", default=None, help="Source LaTeX file (gzipped tar)") - parser.add_argument("-t", "--timeout", type=int, default=DEFAULT_COMPILATION_TIMEOUT, - help="Timeout for the request in seconds") - parser.add_argument("-w", "--watermark-text", default=None, help="Text string for PDF watermark.") - parser.add_argument("-v", "--verbose", action="store_true", help="Enable verbose mode") - parser.add_argument("-d", "--debug", action="store_true", help="Enable debug mode (dump JSON)") - parser.add_argument("-b", "--base", default="/data/new", - help="Root directory for submissions directories (default:/data/new)") - parser.add_argument("-c", "--console", default=False, action="store_true", - help="Write log output to console.") - parser.add_argument("-l", "--logs-dir", default="/users/e-prints/httpd/logs", - help="Directory to store system level compilation log.") - parser.add_argument("-u", '--tex2pdf-url', - help="URL for tex2pdf service.") - - # Parse command-line arguments - args = parser.parse_args() - - # Call main to dispatch request to compile_submission method - main(args) From 9e9ce0b17678abcf713af6ccc1528efe1333396b Mon Sep 17 00:00:00 2001 From: Brian Maltzan Date: Mon, 8 Jun 2026 14:45:23 -0400 Subject: [PATCH 4/6] add setting --- submit_ce/implementations/compile/compile_api_service.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/submit_ce/implementations/compile/compile_api_service.py b/submit_ce/implementations/compile/compile_api_service.py index e1f08089..a7046187 100644 --- a/submit_ce/implementations/compile/compile_api_service.py +++ b/submit_ce/implementations/compile/compile_api_service.py @@ -235,10 +235,10 @@ def check(self, process_id: str, user: User, client: Client) -> ProcessStatus: @override def is_available(self) -> bool: try: - resp = httpx.get(self.tex2pdf_url, timeout=1) + resp = httpx.get(settings.COMPILE_API_URL, timeout=1) return resp.status_code == 200 except httpx.RequestError as exc: - logger.error(f"Local compile service at '{self.tex2pdf_url}' is not available: {exc}") + logger.error(f"Local compile service at '{settings.COMPILE_API_URL}' is not available: {exc}") return False @override From b6a7dd1676839094687aff1d78e38e8ffac91224 Mon Sep 17 00:00:00 2001 From: Brian Maltzan Date: Mon, 8 Jun 2026 14:52:17 -0400 Subject: [PATCH 5/6] SUBMISSION-16: add compile api tests --- .../tests/test_compile_api_service.py | 297 ++++++++++++++++++ 1 file changed, 297 insertions(+) create mode 100644 submit_ce/implementations/tests/test_compile_api_service.py diff --git a/submit_ce/implementations/tests/test_compile_api_service.py b/submit_ce/implementations/tests/test_compile_api_service.py new file mode 100644 index 00000000..91d5bf26 --- /dev/null +++ b/submit_ce/implementations/tests/test_compile_api_service.py @@ -0,0 +1,297 @@ +"""Tests for :mod:`submit_ce.implementations.compile.compile_api_service`.""" + +from datetime import datetime, timezone, timedelta +from unittest.mock import MagicMock + +import httpx +import pytest + +from submit_ce.implementations.compile import compile_api_service +from submit_ce.implementations.compile.compile_api_service import ( + CompileApiService, + _auth_headers, + _get_id_token, +) +from submit_ce.ui.config import settings + + +@pytest.fixture(autouse=True) +def clear_token_cache(): + """Prevent the module-level ID token cache from leaking between tests.""" + compile_api_service._ID_TOKEN_CACHE.clear() + yield + compile_api_service._ID_TOKEN_CACHE.clear() + + +def _mock_httpx_post(mocker, *, status_code, raise_for_status=None): + """Patch httpx.Client so `with httpx.Client(...) as c: c.post(...)` yields + a response with the given status_code.""" + resp = MagicMock(status_code=status_code) + if raise_for_status is not None: + resp.raise_for_status.side_effect = raise_for_status + else: + resp.raise_for_status.return_value = None + mock_client = MagicMock() + mock_client.__enter__.return_value.post.return_value = resp + mocker.patch.object(compile_api_service.httpx, 'Client', + return_value=mock_client) + return resp + + +def _mock_httpx_post_sequence(mocker, status_codes): + """Cycle through `status_codes` across successive .post() calls (one per retry).""" + responses = [] + for code in status_codes: + r = MagicMock(status_code=code) + r.raise_for_status.return_value = None + responses.append(r) + mock_client = MagicMock() + mock_client.__enter__.return_value.post.side_effect = responses + mocker.patch.object(compile_api_service.httpx, 'Client', + return_value=mock_client) + return responses + + +def _mock_file_store(): + store = MagicMock() + store.get_full_source_package_path.return_value = 'gs://bucket/src.tgz' + store.get_full_preflight_package_path.return_value = 'gs://bucket/preflight.json' + store.get_full_submission_source_path.return_value = 'gs://bucket/src/' + store.get_full_outcome_path.return_value = 'gs://bucket/out/' + store.get_full_submission_path.return_value = 'gs://bucket/sub' + store.get_full_directives_package_path.return_value = 'gs://bucket/dir.json' + return store + + +def _patch_current_app(mocker, store=None): + """Replace current_app in the module with a stub whose api.get_file_store() + returns the given store (default: a fresh _mock_file_store).""" + mock_app = MagicMock() + mock_app.api.get_file_store.return_value = store or _mock_file_store() + mocker.patch.object(compile_api_service, 'current_app', mock_app) + return mock_app + + +def _submission(): + s = MagicMock() + s.submission_id = 'sub1' + return s + + +# ------------------------------------------------------------ _get_id_token + + +def test_get_id_token_cache_hit_returns_cached_token(mocker): + """Non-expired cache entry: no auth path runs, cached token returned.""" + audience = 'https://example.com' + future = datetime.now(timezone.utc) + timedelta(minutes=10) + compile_api_service._ID_TOKEN_CACHE[audience] = ('cached-tok', future) + fetch = mocker.patch('google.oauth2.id_token.fetch_id_token') + + assert _get_id_token(audience) == 'cached-tok' + fetch.assert_not_called() + + +def test_get_id_token_uses_impersonate_sa_when_set(mocker): + """COMPILE_API_IMPERSONATE_SA set → use the impersonated_credentials path.""" + mocker.patch.object(settings, 'COMPILE_API_IMPERSONATE_SA', + 'sa@x.iam.gserviceaccount.com') + mocker.patch.object(compile_api_service.google.auth, 'default', + return_value=(MagicMock(), 'project')) + target = mocker.patch.object( + compile_api_service.impersonated_credentials, 'Credentials') + mocker.patch.object( + compile_api_service.impersonated_credentials, 'IDTokenCredentials', + return_value=MagicMock(token='impersonated-tok')) + fetch = mocker.patch('google.oauth2.id_token.fetch_id_token') + + assert _get_id_token('https://aud') == 'impersonated-tok' + fetch.assert_not_called() + target.assert_called_once() + + +def test_get_id_token_falls_back_to_fetch_id_token(mocker): + """No impersonate SA → fetch_id_token is used.""" + mocker.patch.object(settings, 'COMPILE_API_IMPERSONATE_SA', '') + fetch = mocker.patch('google.oauth2.id_token.fetch_id_token', + return_value='fetched-tok') + + assert _get_id_token('https://aud') == 'fetched-tok' + fetch.assert_called_once() + + +def test_get_id_token_refetches_after_cache_expires(mocker): + """Expired cache entry triggers a refetch.""" + audience = 'https://aud' + past = datetime.now(timezone.utc) - timedelta(seconds=1) + compile_api_service._ID_TOKEN_CACHE[audience] = ('stale-tok', past) + mocker.patch.object(settings, 'COMPILE_API_IMPERSONATE_SA', '') + fetch = mocker.patch('google.oauth2.id_token.fetch_id_token', + return_value='fresh-tok') + + assert _get_id_token(audience) == 'fresh-tok' + fetch.assert_called_once() + + +# ------------------------------------------------------------ _auth_headers + + +def test_auth_headers_http_omits_authorization(mocker): + mocker.patch.object(settings, 'COMPILE_API_URL', 'http://localhost:9001') + headers = _auth_headers() + assert 'Authorization' not in headers + assert headers['accept'] == 'application/json' + + +def test_auth_headers_https_includes_bearer(mocker): + mocker.patch.object(settings, 'COMPILE_API_URL', 'https://example.run.app') + mocker.patch.object(compile_api_service, '_get_id_token', + return_value='tok-123') + headers = _auth_headers() + assert headers['Authorization'] == 'Bearer tok-123' + + +# ------------------------------------------------------------ __repr__ + + +def test_repr_includes_compile_api_url(mocker): + mocker.patch.object(settings, 'COMPILE_API_URL', 'http://test') + assert 'tex2pdf_url=http://test' in repr(CompileApiService()) + + +# ------------------------------------------------------------ check_* methods + + +def test_check_preflight_returns_succeeded(): + ps = CompileApiService().check_preflight('p1', MagicMock(), MagicMock()) + assert ps.status == ps.Status.SUCCEEDED + + +def test_check_returns_succeeded(): + ps = CompileApiService().check('p1', MagicMock(), MagicMock()) + assert ps.status == ps.Status.SUCCEEDED + + +def test_check_directives_returns_succeeded(): + ps = CompileApiService().check_directives('p1', MagicMock(), MagicMock()) + assert ps.status == ps.Status.SUCCEEDED + + +# ------------------------------------------------------------ is_available + + +def test_is_available_true_on_200(mocker): + mocker.patch.object(compile_api_service.httpx, 'get', + return_value=MagicMock(status_code=200)) + assert CompileApiService().is_available() is True + + +def test_is_available_false_on_non_200(mocker): + mocker.patch.object(compile_api_service.httpx, 'get', + return_value=MagicMock(status_code=503)) + assert CompileApiService().is_available() is False + + +def test_is_available_false_on_request_error(mocker): + mocker.patch.object(compile_api_service.httpx, 'get', + side_effect=httpx.RequestError("boom")) + assert CompileApiService().is_available() is False + + +# ------------------------------------------------------------ start_preflight + + +def test_start_preflight_success_returns_result(mocker): + mocker.patch.object(settings, 'COMPILE_API_URL', 'http://localhost:9001') + store = _mock_file_store() + _patch_current_app(mocker, store) + _mock_httpx_post(mocker, status_code=200) + + result = CompileApiService().start_preflight( + _submission(), MagicMock(), MagicMock(), MagicMock()) + + assert result.status.status == result.status.Status.SUCCEEDED + store.get_full_source_package_path.assert_called_once_with('sub1') + store.get_full_preflight_package_path.assert_called_once_with('sub1') + + +def test_start_preflight_retries_on_500_then_succeeds(mocker): + mocker.patch.object(settings, 'COMPILE_API_URL', 'http://localhost:9001') + mocker.patch.object(settings, 'COMPILE_API_MAX_RETRIES', 3) + mocker.patch.object(compile_api_service.time, 'sleep') + _patch_current_app(mocker) + _mock_httpx_post_sequence(mocker, [500, 200]) + + result = CompileApiService().start_preflight( + _submission(), MagicMock(), MagicMock(), MagicMock()) + + assert result.status.status == result.status.Status.SUCCEEDED + + +def test_start_preflight_raises_when_500_persists(mocker): + mocker.patch.object(settings, 'COMPILE_API_URL', 'http://localhost:9001') + mocker.patch.object(settings, 'COMPILE_API_MAX_RETRIES', 2) + mocker.patch.object(compile_api_service.time, 'sleep') + _patch_current_app(mocker) + _mock_httpx_post(mocker, status_code=500) + + with pytest.raises(httpx.HTTPStatusError): + CompileApiService().start_preflight( + _submission(), MagicMock(), MagicMock(), MagicMock()) + + +# ------------------------------------------------------------ start_compile + + +def test_start_compile_success_returns_result(mocker): + mocker.patch.object(settings, 'COMPILE_API_URL', 'http://localhost:9001') + store = _mock_file_store() + _patch_current_app(mocker, store) + _mock_httpx_post(mocker, status_code=200) + + result = CompileApiService().start_compile( + _submission(), MagicMock(), MagicMock(), MagicMock()) + + assert result.status.status == result.status.Status.SUCCEEDED + store.get_full_submission_source_path.assert_called_once_with('sub1') + store.get_full_outcome_path.assert_called_once_with('sub1') + + +def test_start_compile_raises_on_4xx(mocker): + mocker.patch.object(settings, 'COMPILE_API_URL', 'http://localhost:9001') + _patch_current_app(mocker) + err = httpx.HTTPStatusError("400", request=MagicMock(), response=MagicMock()) + _mock_httpx_post(mocker, status_code=400, raise_for_status=err) + + with pytest.raises(httpx.HTTPStatusError): + CompileApiService().start_compile( + _submission(), MagicMock(), MagicMock(), MagicMock()) + + +# ------------------------------------------------------------ start_directives + + +def test_start_directives_success_returns_result(mocker): + mocker.patch.object(settings, 'COMPILE_API_URL', 'http://localhost:9001') + store = _mock_file_store() + _patch_current_app(mocker, store) + _mock_httpx_post(mocker, status_code=200) + + result = CompileApiService().start_directives( + _submission(), MagicMock(), MagicMock(), MagicMock()) + + assert result.status.status == result.status.Status.SUCCEEDED + store.get_full_submission_path.assert_called_once_with('sub1') + store.get_full_directives_package_path.assert_called_once_with('sub1') + + +def test_start_directives_raises_when_500_persists(mocker): + mocker.patch.object(settings, 'COMPILE_API_URL', 'http://localhost:9001') + mocker.patch.object(settings, 'COMPILE_API_MAX_RETRIES', 2) + mocker.patch.object(compile_api_service.time, 'sleep') + _patch_current_app(mocker) + _mock_httpx_post(mocker, status_code=500) + + with pytest.raises(httpx.HTTPStatusError): + CompileApiService().start_directives( + _submission(), MagicMock(), MagicMock(), MagicMock()) From 51480cedfe2b1aceb527b80eb735d567d3e055f5 Mon Sep 17 00:00:00 2001 From: Brian Maltzan Date: Tue, 9 Jun 2026 09:01:40 -0400 Subject: [PATCH 6/6] SUBMISSION-16: fix messaging --- .../implementations/compile/compile_api_service.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/submit_ce/implementations/compile/compile_api_service.py b/submit_ce/implementations/compile/compile_api_service.py index a7046187..4cf645c4 100644 --- a/submit_ce/implementations/compile/compile_api_service.py +++ b/submit_ce/implementations/compile/compile_api_service.py @@ -158,7 +158,7 @@ def start_preflight(self, ), duration_sec=0, utc_start_time=datetime.now(timezone.utc), - url="FAKE_URL_LOCAL_PREFLIGHT" + url="FAKE_URL_PREFLIGHT" ) @override @@ -219,17 +219,17 @@ def start_compile(self, submission: Submission, ), duration_sec=0, utc_start_time=datetime.now(timezone.utc), - url="FAKE_URL_LOCAL_COMPILE" + url="FAKE_URL_COMPILE" ) @override def check(self, process_id: str, user: User, client: Client) -> ProcessStatus: - logger.info("Checking local compilation for process %s", process_id) + logger.info("Checking compilation for process %s", process_id) return ProcessStatus( status=ProcessStatus.Status.SUCCEEDED, creator=user, created=datetime.now(timezone.utc), - details={'message': 'Local compilation check completed'} + details={'message': 'Compilation check completed'} ) @override @@ -238,7 +238,7 @@ def is_available(self) -> bool: resp = httpx.get(settings.COMPILE_API_URL, timeout=1) return resp.status_code == 200 except httpx.RequestError as exc: - logger.error(f"Local compile service at '{settings.COMPILE_API_URL}' is not available: {exc}") + logger.error(f"Compile service at '{settings.COMPILE_API_URL}' is not available: {exc}") return False @override @@ -297,7 +297,7 @@ def start_directives(self, ), duration_sec=0, utc_start_time=datetime.now(timezone.utc), - url="FAKE_URL_LOCAL_DIRECTIVES" + url="FAKE_URL_DIRECTIVES" ) @override