diff --git a/submit_ce/ui/__init__.py b/submit_ce/ui/__init__.py index 10472969..a17e1568 100644 --- a/submit_ce/ui/__init__.py +++ b/submit_ce/ui/__init__.py @@ -42,9 +42,14 @@ def get_device_type(user_agent): return user_agent[:25] +# Bits of the "classic" capability code produced by +# arxiv.auth.legacy.util.compute_capabilities: +# flag_edit_users -> 2 (admin) +# flag_email_verified -> 4 +# flag_edit_system -> 8 (system/"god", i.e. dev) # TODO move these to somewhere under arxiv.auth.auth -ADMIN_MASK = 1 -DEV_MASK = 1<<2 +ADMIN_MASK = 1 << 1 +DEV_MASK = 1 << 3 def is_admin(session: Session)->bool: return bool(getattr(session.authorizations, "classic", 0) & ADMIN_MASK) diff --git a/submit_ce/ui/auth.py b/submit_ce/ui/auth.py index fbb5f400..3e7f3078 100644 --- a/submit_ce/ui/auth.py +++ b/submit_ce/ui/auth.py @@ -268,6 +268,7 @@ def user_and_client_from_session(session: auth_domian.Session) -> Tuple[User, Cl if is_admin(session): user = StaffUser( user_id=session.user.user_id, + username=session.user.username, name=name, email=session.user.email, endorsements = get_endorsements(session.user), diff --git a/submit_ce/ui/conftest.py b/submit_ce/ui/conftest.py index ff075ec9..c309efe7 100644 --- a/submit_ce/ui/conftest.py +++ b/submit_ce/ui/conftest.py @@ -188,6 +188,26 @@ def authorized_client(app, authorized_user_session): yield app.test_client(jwt=jwt) +@pytest.fixture +def admin_client(app, authorized_user_session): + """Authorized client whose user has the classic dev/system capability. + + Needed for the ``/debug/`` routes, which are gated by + ``is_admin_or_dev``. ``request_auth`` recomputes the ``classic`` capability + code from the database user (ignoring whatever is in the JWT), so we flip + ``flag_edit_system`` on the underlying TapirUser rather than editing the + token. Reuses the same user as ``authorized_client`` so ownership-based + fixtures still line up. + """ + session, jwt = authorized_user_session + with app.app_context(): + db_user = Session.get(classic.TapirUser, int(session.user.user_id)) + db_user.flag_edit_system = 1 + Session.commit() + app.test_client_class = ClientArxivAuth + yield app.test_client(jwt=jwt) + + #################### submissions in different stages #################### @pytest.fixture(scope="function") def sub_created(app, authorized_user): diff --git a/submit_ce/ui/controllers/__init__.py b/submit_ce/ui/controllers/__init__.py index 586725ce..f0f3f069 100644 --- a/submit_ce/ui/controllers/__init__.py +++ b/submit_ce/ui/controllers/__init__.py @@ -3,8 +3,10 @@ #ruff: noqa: F401 from http import HTTPStatus as status +from typing import Optional from arxiv.auth.domain import Session +from flask import current_app from werkzeug.datastructures import MultiDict from submit_ce.ui.routes.flow_control import advance_to_current @@ -23,16 +25,40 @@ from .new.verify_user import verify from .util import Response +_GS_CONSOLE = "https://console.cloud.google.com/storage/browser/" + + +def _gs_links(submission_id: str) -> tuple[Optional[str], Optional[str]]: + """Return ``(gs_path, console_url)`` for the submission dir. + + ``gs_path`` is the ``gs://{bucket}/{path}`` URI (used as the link text) + and ``console_url`` is the matching GCS console browser URL (the href). + Both are None unless the file store reports a gs:// path (i.e. the GS + store); local/null stores return "" and get ``(None, None)``. + """ + try: + path = current_app.api.get_file_store().get_full_submission_path( + str(submission_id)) + except Exception: + return None, None + if path and path.startswith("gs://"): + return path, _GS_CONSOLE + path[len("gs://"):] + return None, None + + def submission_status(method: str, params: MultiDict, session: Session, submission_id: str) -> Response: #user, client = util.user_and_client_from_session(session) # Will raise NotFound if there is no such submission. submission, submission_events = get_submission(submission_id) + gs_path, gs_console_url = _gs_links(submission_id) response_data = { 'submission': submission, 'submission_id': submission_id, - 'events': submission_events + 'events': submission_events, + 'gs_path': gs_path, + 'gs_console_url': gs_console_url, } return response_data, status.OK, {} @@ -41,9 +67,12 @@ def submission_edit(method: str, params: MultiDict, session: Session, submission_id: str) -> Response: """Cause flow_control to go to the current_stage of the Submission.""" submission, submission_events = get_submission(submission_id) + gs_path, gs_console_url = _gs_links(submission_id) response_data = { 'submission': submission, 'submission_id': submission_id, 'events': submission_events, + 'gs_path': gs_path, + 'gs_console_url': gs_console_url, } return advance_to_current((response_data, status.OK, {})) diff --git a/submit_ce/ui/routes/ui.py b/submit_ce/ui/routes/ui.py index 0be302ab..54517436 100644 --- a/submit_ce/ui/routes/ui.py +++ b/submit_ce/ui/routes/ui.py @@ -6,7 +6,7 @@ from arxiv.auth.auth import scopes from arxiv.auth.auth.decorators import scoped from arxiv.base import logging, alerts -from flask import Blueprint, make_response, redirect, request, render_template, url_for, send_file +from flask import Blueprint, make_response, redirect, request, render_template, url_for, send_file, current_app from flask import Response as FResponse from markupsafe import Markup from werkzeug import Response as WResponse @@ -223,21 +223,13 @@ def create_replacement(submission_id: str): @UI.route('/', methods=["GET"]) -@scoped(scopes.VIEW_SUBMISSION, authorizer=is_owner, - unauthorized=redirect_to_login) -def submission_status(submission_id: str) -> Response: - """Display the current state of the submission.""" - return handle(cntrls.submission_status, 'submit/status.html', - 'Submission status', submission_id) - - @UI.route('//edit', methods=['GET']) @scoped(scopes.VIEW_SUBMISSION, authorizer=is_owner, unauthorized=redirect_to_login) @flow_control() def submission_edit(submission_id: str) -> Response: """Redirects to current edit stage of the submission.""" - return handle(cntrls.submission_edit, 'submit/status.html', + return handle(cntrls.submission_edit, 'debug/status.html', 'Submission status', submission_id, flow_controlled=True) # # TODO: remove me!! @@ -651,6 +643,39 @@ def debug_logout() -> Response: return response +@UI.route('/debug/mail', methods=["GET"]) +@scoped(scopes.VIEW_SUBMISSION, authorizer=is_admin_or_dev, + unauthorized=redirect_to_login) +def get_debug_mail() -> Response: + """Dev-only: show email captured by the in-memory email service. + + Only available when ``EMAIL_MODE`` is ``TESTING`` and the configured + email service is the in-memory ``EmailInMemory`` capture. In any other + mode real mail was dispatched and there is nothing held in process to + show, so this returns 404. + """ + from submit_ce.implementations.email.email_in_memory import EmailInMemory + if settings.EMAIL_MODE != "TESTING": + raise NotFound() + service = current_app.api.get_email_service() + if not isinstance(service, EmailInMemory): + raise NotFound() + return make_response( + render_template('debug/debug_mail.html', + pagetitle='Debug Mail', + emails=service.sent), + 200) + + +@UI.route('/debug/', methods=["GET"]) +@scoped(scopes.VIEW_SUBMISSION, authorizer=is_admin_or_dev, + unauthorized=redirect_to_login) +def get_debug_submission(submission_id: Optional[str] = None) -> Response: + """Display the current state of the submission.""" + return handle(cntrls.submission_status, 'debug/status.html', + 'Submission status', submission_id) + + @UI.route('/debug//events', methods=["GET"]) @scoped(scopes.VIEW_SUBMISSION, authorizer=is_admin_or_dev, unauthorized=redirect_to_login) diff --git a/submit_ce/ui/templates/debug/debug_mail.html b/submit_ce/ui/templates/debug/debug_mail.html new file mode 100644 index 00000000..219c0872 --- /dev/null +++ b/submit_ce/ui/templates/debug/debug_mail.html @@ -0,0 +1,45 @@ +{% extends "base/base.html" %} + +{% block addl_head %} +{% endblock addl_head %} + +{% block content %} +

Debug Mail

+

+ Email captured in memory (EMAIL_MODE=TESTING). Nothing here was + actually dispatched. Newest first. {{ emails | length }} message(s). +

+ +{% if emails %} + + + + + + + + + + + {% for m in emails | reverse %} + + + + + + + {% endfor %} + +
ToSubjectHeadersBody
+
To: {{ m.to | join(', ') }}
+ {% if m.cc %}
Cc: {{ m.cc | join(', ') }}
{% endif %} + {% if m.bcc %}
Bcc: {{ m.bcc | join(', ') }}
{% endif %} +
Reply-To: {{ m.reply_to }}
+ 		{{ m.subject }} + {% if m.message_id %}
Message-ID: {{ m.message_id }}
{% endif %} + {% if m.references %}
References: {{ m.references }}
{% endif %} + 
{{ m.body }}
+{% else %} +

No email has been captured yet.

+{% endif %} +{% endblock content %} diff --git a/submit_ce/ui/templates/debug/status.html b/submit_ce/ui/templates/debug/status.html new file mode 100644 index 00000000..72828e36 --- /dev/null +++ b/submit_ce/ui/templates/debug/status.html @@ -0,0 +1,261 @@ +{% extends "submit/base.html" %} + +{% block title -%} +{% if submission.submission_type %}{{ submission.submission_type.name.replace('_', ' ').title() }}{% endif %} +Submission {{submission.submission_id}}: +{{ submission.status }}{%- endblock title %} + +{% block within_content %} + + +{% set handled_fields = ['creator', 'client', 'created', 'NAME', 'NAMED', 'committed', 'submission_id'] %} +{% set skip_if_none = ['proxy', 'cause'] %} +{% set sub_collections = ['annotations', 'comments', 'holds', 'flags', 'processes', 'proposals', 'user_requests', 'waivers'] %} +{% set sub_handled = ['creator', 'owner', 'client', 'metadata'] + sub_collections %} + + + + +{% if gs_console_url %} +

+ Submission directory in GS: {{ gs_path }} +

+{% endif %} + +
+ {# creator: show name, keep the rest collapsed #} + {% if submission.creator %} +
+ Creator: {{ submission.creator.name }} + + {% for key, val in submission.creator.model_dump().items()|sort %} + + {% endfor %} +
{{ key }}{{ val }}
+
+ {% endif %} + + {# owner: show name, keep the rest collapsed #} + {% if submission.owner %} +
+ Owner: {{ submission.owner.name }} + + {% for key, val in submission.owner.model_dump().items()|sort %} + + {% endfor %} +
{{ key }}{{ val }}
+
+ {% endif %} + + {# client: show name (or a sensible fallback), keep the rest collapsed #} + {% if submission.client %} +
+ + Client: + {% if submission.client.name is defined and submission.client.name %}{{ submission.client.name }} + {%- elif submission.client.remote_addr %}{{ submission.client.remote_addr }} + {%- else %}{{ submission.client.tool }}{% endif %} + + + {% for key, val in submission.client.model_dump().items()|sort %} + + {% endfor %} +
{{ key }}{{ val }}
+
+ {% endif %} + + {# metadata: collapsible but open by default #} +
+ Metadata + + {% for key, val in submission.metadata.__dict__.items()|sort %} + + + + + {% endfor %} +
{{ key }} + {% if val is mapping or val is sequence and val is not string %} + 
{{ val|pprint }}
+ {% else %} + {{ val }} + {% endif %} +
+
+ + {# collections: collapsible lists, default closed, with count next to the name #} + {% for name in sub_collections|sort %} + {% set coll = submission.__dict__.get(name) %} +
+ {{ name }} ({{ coll|length }}) + + {% if coll is mapping %} + {% for k, v in coll.items() %} + + {% endfor %} + {% else %} + {% for v in coll %} + + {% endfor %} + {% endif %} +
{{ k }}
{{ v|pprint }}
{{ loop.index0 }}
{{ v|pprint }}
+
+ {% endfor %} + + {# remaining properties, alphabetical, in a table #} + + + {% for key, val in submission.__dict__.items()|sort %} + {% if key not in sub_handled %} + + + + + {% endif %} + {% endfor %} + +
{{ key }} + {% if val is mapping or val is sequence and val is not string %} + 
{{ val|pprint }}
+ {% else %} + {{ val }} + {% endif %} +
+
+ +

Events

+
+ {% for event in events %} + {% set data = event.model_dump() %} +
+ + {{ event.NAMED.capitalize() }} + {{ event.created|timesince }} + {% if event.creator %}{{ event.creator.name }}{% endif %} + + + {# creator: show name, keep the rest collapsed #} + {% if event.creator %} +
+ Creator: {{ event.creator.name }} + + {% for key, val in data.get('creator', {}).items()|sort %} + + {% endfor %} +
{{ key }}{{ val }}
+
+ {% endif %} + + {# client: show name (or a sensible fallback), keep the rest collapsed #} + {% if event.client %} +
+ + Client: + {% if event.client.name is defined and event.client.name %}{{ event.client.name }} + {%- elif event.client.remote_addr %}{{ event.client.remote_addr }} + {%- else %}{{ event.client.tool }}{% endif %} + + + {% for key, val in data.get('client', {}).items()|sort %} + + {% endfor %} +
{{ key }}{{ val }}
+
+ {% endif %} + + {# remaining properties, sorted, in a table #} + + + {% for key, val in data.items()|sort %} + {% if key not in handled_fields and (key not in skip_if_none or val )%} + + + + + {% endif %} + {% endfor %} + +
{{ key }} + {% if val is mapping or val is sequence and val is not string %} + 
{{ val|pprint }}
+ {% else %} + {{ val }} + {% endif %} +
+ as json +
+ + {% endfor %} +
+ +{# Shared modal, populated by JS from the clicked event's data-json. #} + + + + +{% endblock within_content %} diff --git a/submit_ce/ui/templates/submit/status.html b/submit_ce/ui/templates/submit/status.html deleted file mode 100644 index 8d6c7770..00000000 --- a/submit_ce/ui/templates/submit/status.html +++ /dev/null @@ -1,24 +0,0 @@ -{% extends "submit/base.html" %} - -{% block title -%}{{ submission.status }}{%- endblock title %} - -{% block within_content %} -

- This is for dev/debugging. We need to implement this page with a real - layout that communicates what's going on with a user's submission. -

- -
-{{ submission|asdict|pprint }}
-
- -{% for event in events %} -
  • - {{ event.NAMED.capitalize() }} {{ event.created|timesince }} ({{event.event_id}}) - 
    -  {{ event|asdict|pprint }}
-
    -
    • -{% endfor %} - -{% endblock within_content %} diff --git a/submit_ce/ui/tests/test_debug_mail.py b/submit_ce/ui/tests/test_debug_mail.py new file mode 100644 index 00000000..f45027a4 --- /dev/null +++ b/submit_ce/ui/tests/test_debug_mail.py @@ -0,0 +1,56 @@ +"""Tests for the /debug/mail email viewer route.""" +from submit_ce.implementations.email.email_in_memory import EmailInMemory + + +def test_debug_mail_shows_captured_email(app, admin_client): + with app.app_context(): + service = app.api.get_email_service() + assert isinstance(service, EmailInMemory) + service.clear() + service.send_email(["someone@example.com"], "Hello there", + "This is the body of the message.", + "noreply@arxiv.org") + + resp = admin_client.get('/debug/mail') + assert resp.status_code == 200 + body = resp.get_data(as_text=True) + assert 'someone@example.com' in body + assert 'Hello there' in body + assert 'This is the body of the message.' in body + + +def test_debug_mail_empty(app, admin_client): + with app.app_context(): + app.api.get_email_service().clear() + + resp = admin_client.get('/debug/mail') + assert resp.status_code == 200 + assert 'No email has been captured yet.' in resp.get_data(as_text=True) + + +def test_debug_mail_404_when_not_testing_mode(app, admin_client): + from submit_ce.ui.config import settings + original = settings.EMAIL_MODE + settings.EMAIL_MODE = "HALON" + try: + resp = admin_client.get('/debug/mail') + assert resp.status_code == 404 + finally: + settings.EMAIL_MODE = original + + +def test_debug_mail_requires_auth(app): + """Unauthenticated requests are rejected by the global auth check.""" + resp = app.test_client().get('/debug/mail') + assert resp.status_code == 401 + + +def test_debug_mail_requires_auth_with_halon(app): + from submit_ce.ui.config import settings + original = settings.EMAIL_MODE + settings.EMAIL_MODE = "HALON" + try: + resp = app.test_client().get('/debug/mail') + assert resp.status_code == 401 + finally: + settings.EMAIL_MODE = original diff --git a/submit_ce/ui/tests/test_status_render.py b/submit_ce/ui/tests/test_status_render.py new file mode 100644 index 00000000..5d38fab6 --- /dev/null +++ b/submit_ce/ui/tests/test_status_render.py @@ -0,0 +1,22 @@ +"""Smoke test that the status page renders its event list.""" +from http import HTTPStatus as status + + +def test_status_page_renders(admin_client, sub_license): + client = admin_client + response = client.get(f'/debug/{sub_license.submission_id}') + assert response.status_code == status.OK + body = response.data.decode('utf-8') + # Header and collapsible sections should be present. + assert 'Events' in body + assert 'Creator:' in body + assert 'Client:' in body + # Client name from the InternalClient used in fixtures. + assert 'test_client_' in body + # "as json" link + shared modal for viewing raw event JSON. + assert 'event-json-link' in body + assert 'id="event-json-modal"' in body + # Structured Submission section with agent + metadata sub-sections. + assert 'Submission' in body + assert 'Owner:' in body + assert 'Metadata' in body diff --git a/submit_ce/ui/workflow/conditions.py b/submit_ce/ui/workflow/conditions.py index c4448ef2..ecc8ba21 100644 --- a/submit_ce/ui/workflow/conditions.py +++ b/submit_ce/ui/workflow/conditions.py @@ -1,9 +1,40 @@ -from typing import List +from typing import Callable, List from submit_ce.domain import Submission, Event -from submit_ce.domain.event.process import StartDirectives +from submit_ce.domain.event.file import ( + RemoveAllFiles, + RemoveFiles, + UploadArchive, + UploadFiles, +) +from submit_ce.domain.event.process import ( + PreflightStatus, + StartDirectives, + StartPreflight, +) from submit_ce.domain.uploads import SourceFormat +# Events that mutate the source workspace. Each of these invalidates the +# stored preflight via `_common_file_change_execute` in event/file.py. +_FILE_CHANGE_EVENTS = (UploadArchive, UploadFiles, RemoveFiles, RemoveAllFiles) + +# Events that record a preflight run against the current files. +_PREFLIGHT_EVENTS = (StartPreflight, PreflightStatus) + +Condition = Callable[[Submission, List[Event]], bool] +"""A workflow condition: true when a stage requirement is satisfied.""" + + +def OR(*conds: Condition) -> Condition: + """Combine conditions so the result is true when *any* of them is true. + + Short-circuits on the first satisfied condition. With no arguments the + combined condition is always false (an empty ``or``). + """ + def combined(submission: Submission, events: List[Event]) -> bool: + return any(cond(submission, events) for cond in conds) + return combined + def is_contact_verified(submission: Submission, events: List[Event]) -> bool: """Determine whether the submitter has verified their information.""" @@ -84,14 +115,32 @@ def is_finalized(submission: Submission, events: List[Event]) -> bool: return bool(submission.is_finalized) -def has_directives_started(submission: Submission, events: List[Event]) -> bool: - """Determine whether a StartDirectives event has been dispatched. +def has_current_directives(submission: Submission, events: List[Event]) -> bool: + """Determine whether directives are current for the uploaded files. Directives generation is a side-effect of the review-files stage: when the user advances past review, a `StartDirectives` event is saved and the compile service writes `directives.json`. The event history is the authoritative record that this happened. + + Any file-change event invalidates the generated directives (see + `_common_file_change_execute` in event/file.py), so a `StartDirectives` + only counts if no file-change event has occurred since the most recent + one. `events` is in chronological order (oldest first). """ - return (submission.source_format == SourceFormat.PDF - or any(isinstance(e, StartDirectives) for e in events) - ) + if submission.source_format == SourceFormat.PDF: + return True + last_directives = None + for i, event in enumerate(events): + if isinstance(event, StartDirectives): + last_directives = i + if last_directives is None: + return False + return not any(isinstance(event, _FILE_CHANGE_EVENTS) + for event in events[last_directives + 1:]) + +def source_format_pdf(submission: Submission, events: List[Event]) -> bool: + return submission.source_format == SourceFormat.PDF + +def source_format_html(submission: Submission, events: List[Event]) -> bool: + return submission.source_format == SourceFormat.HTML diff --git a/submit_ce/ui/workflow/stages.py b/submit_ce/ui/workflow/stages.py index eb5186a2..e85a87ef 100644 --- a/submit_ce/ui/workflow/stages.py +++ b/submit_ce/ui/workflow/stages.py @@ -107,7 +107,9 @@ class ReviewFiles(Stage): title = "Review Files" display = "Review Files" always_check = True - completed = [conditions.has_directives_started] + completed = [conditions.OR(conditions.source_format_pdf, + conditions.source_format_html, + conditions.has_current_directives)] class Process(Stage): """Uploaded files are processed; this is primarily to compile LaTeX.""" diff --git a/uv.lock b/uv.lock index b392cc68..28f788e5 100644 --- a/uv.lock +++ b/uv.lock @@ -60,7 +60,7 @@ dependencies = [ [[package]] name = "arxiv-tex2pdf-tools" version = "0.1.0" -source = { git = "https://github.com/arXiv/submission-tools.git?subdirectory=tex2pdf-tools&branch=master#0ef0c5a8e91cfafa119f46196ff15d9c1f2d4abf" } +source = { git = "https://github.com/arXiv/submission-tools.git?subdirectory=tex2pdf-tools&branch=master#83a908024c941ef2cd52503075b36be1184ade86" } dependencies = [ { name = "pydantic" }, { name = "ruamel-yaml" },