Merge develop -> master to release changes to qa

.github/workflows/lint-test-package.yml

@@ -47,6 +47,10 @@ jobs:
         run: uv run ruff check --output-format=github .
         working-directory: ${{ inputs.package_name }}
 
+      - name: Check formatting with Ruff
+        run: uv run ruff format --check
+        working-directory: ${{ inputs.package_name }}
+
       - name: Type with ty
         run: uv run ty check
         working-directory: ${{ inputs.package_name }}

LICENSE

@@ -1,4 +1,28 @@
-Copyright 2017-2025 Cornell University
+Copyright (c) 2026 arXiv, Inc.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of
+this software and associated documentation files (the "Software"), to deal in
+the Software without restriction, including without limitation the rights to
+use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
+of the Software, and to permit persons to whom the Software is furnished to do
+so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+
+---
+
+Contributions made on or before June 30, 2026 are:
+
+Copyright (c) 2017-2025 Cornell University
 
 Permission is hereby granted, free of charge, to any person obtaining a copy of
 this software and associated documentation files (the "Software"), to deal in

arxiv/db/arxiv_db_schema.sql

@@ -365,7 +365,6 @@ CREATE TABLE `arXiv_check_results` (
   KEY `check_results_submission_index` (`submission_id`),
   KEY `check_results_check_index` (`check_id`),
   KEY `check_results_user_index` (`user_id`),
-  CONSTRAINT `check_results_checks_fk` FOREIGN KEY (`check_id`) REFERENCES `arXiv_checks` (`check_id`),
   CONSTRAINT `check_results_sub_fk` FOREIGN KEY (`submission_id`) REFERENCES `arXiv_submissions` (`submission_id`),
   CONSTRAINT `check_results_user_fk` FOREIGN KEY (`user_id`) REFERENCES `tapir_users` (`user_id`)
 ) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=latin1;

arxiv/db/models.py

@@ -2261,7 +2261,7 @@ class CheckTargets(Base):
     def __repr__(self):
         return f"{ type(self).__name__ }/{ self.check_target_id }:{self.name}"
 
-
+# This class is no longer used
 class Checks(Base):
     __tablename__ = "arXiv_checks"
     __table_args__ = {"mysql_charset": "latin1"}
@@ -2283,7 +2283,6 @@ class Checks(Base):
     role: Mapped["CheckRoles"] = relationship("CheckRoles")
 
     view: Mapped["CheckResultViews"] = relationship("CheckResultViews")
-    arXiv_check_results: Mapped[List["CheckResults"]] = relationship("CheckResults", back_populates="check")
 
     def __repr__(self):
         return f"{ type(self).__name__ }/{ self.check_id }:{self.name};{'enabled' if self.enable_check else ''}"
@@ -2297,15 +2296,14 @@ class CheckResults(Base):
     submission_id: Mapped[int] = mapped_column(ForeignKey("arXiv_submissions.submission_id"), nullable=False, index=True)
     data_version: Mapped[int] = mapped_column(Integer, nullable=False, server_default=text("'0'"))
     metadata_version: Mapped[int] = mapped_column(Integer, nullable=False, server_default=text("'0'"))
-    check_id: Mapped[int] = mapped_column(ForeignKey("arXiv_checks.check_id"), nullable=False, index=True)
+    check_id: Mapped[int] = mapped_column(Integer, nullable=True)
     user_id: Mapped[int] = mapped_column(ForeignKey("tapir_users.user_id"), nullable=False, index=True)
     ok: Mapped[int] = mapped_column(Integer, nullable=False)
     created: Mapped[datetime] = mapped_column(DateTime, nullable=False, server_default=func.now())
     message: Mapped[Optional[str]] = mapped_column(String(200))
     data: Mapped[Optional[str]] = mapped_column(Text)
     submission: Mapped["Submission"] = relationship("Submission", back_populates="arXiv_check_results")
 
-    check: Mapped["Checks"] = relationship("Checks", back_populates="arXiv_check_results")
     user: Mapped["TapirUser"] = relationship("TapirUser", back_populates="arXiv_check_results")
     check_responses: Mapped[List["CheckResponses"]] = relationship("CheckResponses", back_populates="check_result")
     """ 

qa/pyproject.toml

@@ -25,6 +25,7 @@ arxiv-base = { git = "https://github.com/arXiv/arxiv-base", rev = "8135d213c8409
 # do not install arxiv-base deps
 [[tool.uv.dependency-metadata]]
 name = "arxiv-base"
+version = "1.0.1"   # required
 requires-dist = []
 
 [tool.ruff]

qa/qa/checks/__init__.py

@@ -11,7 +11,13 @@
 from qa.checks.metadata.report_num import ReportNumIsValid  # noqa
 from qa.checks.metadata.title import TitleIsValid  # noqa
 
-checks: list[BaseCheck] = [
+from qa.checks.submission.files import DoesNotExceedTheFileSizeLimit  # noqa
+from qa.checks.submission.type import IsNotAWithdrawal  # noqa
+
+from qa.checks.fulltext.extraction import TextExtractionSuccessful  # noqa
+from qa.checks.fulltext.structure import FulltextNotTooShort  # noqa
+
+submit_event_checks: list[BaseCheck] = [
     TitleIsValid(),
     AuthorsAreValid(),
     AbstractIsValid(),
@@ -21,4 +27,13 @@
     DoiIsValid(),
     MscClassIsValid(),
     AcmClassIsValid(),
+    DoesNotExceedTheFileSizeLimit(),
+    IsNotAWithdrawal(),
 ]
+
+fulltext_checks: list[BaseCheck] = [
+    TextExtractionSuccessful(),
+    FulltextNotTooShort(),
+]
+
+checks: list[BaseCheck] = submit_event_checks + fulltext_checks

qa/qa/checks/fulltext/extraction.py

@@ -0,0 +1,38 @@
+from qa.checks.base import BaseCheck
+
+from qa.checks.models import FulltextReport, OnFailurePolicy, QaDataRegistry, Result
+
+
+class TextExtractionSuccessful(BaseCheck):
+    name = "text_extraction_successful"
+    display_name = "Text Extraction Successful"
+    id = 14
+    version = "1.0.0"
+    description = "Text extraction was successful."
+    on_failure_policy = OnFailurePolicy.WARN
+    failure_message = "Text extraction failed."
+
+    required_data = {"fulltext_report"}
+
+    failure_flag_id = "text-extraction-failed"
+
+    @classmethod
+    def check(cls, fulltext_report: FulltextReport) -> Result:
+        return cls().run(QaDataRegistry(fulltext_report=fulltext_report))
+
+    @property
+    def config(self) -> dict:
+        return {
+            **super().config,
+            "failure_flag_id": self.failure_flag_id,
+        }
+
+    def _run(self, data_registry: QaDataRegistry) -> Result:
+        fulltext_report = data_registry.fulltext_report
+        assert fulltext_report is not None
+
+        extraction_failed = any(flag.id == self.failure_flag_id for flag in fulltext_report.flags)
+
+        if extraction_failed:
+            return self._result(passed=False, message=self.failure_message)
+        return self._result(passed=True)

qa/qa/checks/fulltext/structure.py

@@ -0,0 +1,39 @@
+from qa.checks.base import BaseCheck
+
+from qa.checks.models import OnFailurePolicy, QaDataRegistry, Result
+
+
+class FulltextNotTooShort(BaseCheck):
+    name = "fulltext_not_too_short"
+    display_name = "Fulltext Not Too Short"
+    id = 15
+    version = "1.0.0"
+    description = "The full text extracted is not too short."
+    on_failure_policy = OnFailurePolicy.WARN
+    failure_message = "Text too short."
+
+    required_inputs = {"fulltext"}
+
+    min_chars = 10000
+    min_words = 1400
+
+    @classmethod
+    def check(cls, fulltext: str) -> Result:
+        return cls().run(QaDataRegistry(fulltext=fulltext))
+
+    @property
+    def config(self) -> dict:
+        return {
+            **super().config,
+            "min_chars": self.min_chars,
+            "min_words": self.min_words,
+        }
+
+    def _run(self, data_registry: QaDataRegistry) -> Result:
+        fulltext = data_registry.fulltext
+        assert fulltext is not None
+
+        if len(fulltext) < self.min_chars and len(fulltext.split()) < self.min_words:
+            return self._result(passed=False, message=self.failure_message)
+        else:
+            return self._result(passed=True)

qa/qa/checks/generic/author_name.py

@@ -8,6 +8,8 @@
 from qa.checks.base import BaseGenericCheck
 from qa.checks.models import QaDataRegistry, OnFailurePolicy, Result
 
+# Note: the ids in this file should be the metadata check id + 10000,
+# to avoid collision with the check_ids previously used in the arxiv_checks table.
 
 _LLM_NAMES = {
     "llama",
@@ -75,7 +77,7 @@ def _check_author(self, keyname: str, firstname: str, suffix: str) -> bool:
 class AuthorsDoNotContainLoneSurname(BaseAuthorCheck):
     name = "authors_do_not_contain_lone_surname"
     display_name = "Authors Do Not Contain Lone Surname"
-    id = 51
+    id = 10051
     version = "1.0.0"
     description = "No author has only a surname without a given name, unless it is a known collaboration or LLM name."
     failure_message = "Contains lone surname."
@@ -112,7 +114,7 @@ def _check_author(self, keyname: str, firstname: str, suffix: str) -> bool:
 class AuthorsDoNotContainLlmAuthor(BaseAuthorCheck):
     name = "authors_do_not_contain_llm_author"
     display_name = "Authors Do Not Contain LLM Author"
-    id = 52
+    id = 10052
     version = "1.0.0"
     description = "No author's name appears to be an AI language model."
     failure_message = "Potential LLM author detected."
@@ -148,7 +150,7 @@ def _check_author(self, keyname: str, firstname: str, suffix: str) -> bool:
 class AuthorNamesDoNotContainSemicolon(BaseAuthorPatternCheck):
     name = "author_names_do_not_contain_semicolon"
     display_name = "Author Names Do Not Contain Semicolon"
-    id = 53
+    id = 10053
     version = "1.0.0"
     description = "No parsed author name contains a semicolon."
     failure_message = "Contains semicolon(s) - use ',' or 'and' to separate authors."
@@ -159,7 +161,7 @@ class AuthorNamesDoNotContainSemicolon(BaseAuthorPatternCheck):
 class AuthorNamesDoNotContainBrackets(BaseAuthorPatternCheck):
     name = "author_names_do_not_contain_brackets"
     display_name = "Author Names Do Not Contain Brackets"
-    id = 54
+    id = 10054
     version = "1.0.0"
     description = "No parsed author name contains square bracket characters."
     failure_message = "Unusual character detected."
@@ -170,7 +172,7 @@ class AuthorNamesDoNotContainBrackets(BaseAuthorPatternCheck):
 class AuthorNamesDoNotContainNumbers(BaseAuthorPatternCheck):
     name = "author_names_do_not_contain_numbers"
     display_name = "Author Names Do Not Contain Numbers"
-    id = 55
+    id = 10055
     version = "1.0.0"
     description = "No parsed author name contains numeric digits."
     failure_message = "Contains a number."
@@ -181,7 +183,7 @@ class AuthorNamesDoNotContainNumbers(BaseAuthorPatternCheck):
 class AuthorNamesDoNotContainAffiliation(BaseAuthorPatternCheck):
     name = "author_names_do_not_contain_affiliation"
     display_name = "Author Names Do Not Contain Affiliation"
-    id = 56
+    id = 10056
     version = "1.0.0"
     description = "No parsed author name contains institution or affiliation keywords."
     failure_message = "Contains a suffix that may be university affiliation or degree related."