Keep daily arXiv recommendations current

Author: chengtan9907

deployments/desktop/static/desktop.js

@@ -1572,6 +1572,22 @@
     return Math.max(0, Math.min(100, Math.round(percentage)));
   }
 
+  function paperDisplayTags(paper) {
+    const rawCategories = Array.isArray(paper?.categories)
+      ? paper.categories
+      : String(paper?.categories || "").split(/[,;]\s*/);
+    const categories = rawCategories
+      .map((item) => String(item || "").trim())
+      .filter((item) => item && /^[a-z-]+(\.[A-Z]{2})?$/i.test(item))
+      .slice(0, 4);
+    if (categories.length) return categories;
+    const fallback = [paper?.primary_category, paper?.arxiv_category, paper?.category]
+      .map((item) => String(item || "").trim())
+      .filter((item) => item && item !== "unknown" && item !== "cached");
+    if (fallback.length) return fallback.slice(0, 3);
+    return [paper?.source || ui().papers.sourceFallback].filter(Boolean);
+  }
+
   function setStatus(message) {
     $("statusText").textContent = statusText(message);
   }
@@ -2244,7 +2260,7 @@
       const reportId = state.paperReports.get(reportKey);
       const isReading = state.paperReading.has(reportKey);
       const score = normalizeScore(paper.score ?? paper.relevance ?? paper.relevance_score ?? 80);
-      const tags = paper.tags || [paper.source || ui().papers.sourceFallback, ui().papers.profileMatch];
+      const tags = paperDisplayTags(paper);
       return `
         <article class="paper-card" data-paper-number="${number}">
           <div class="score-ring" style="--score: ${Math.min(score, 100)}%">${score}%</div>

deployments/feishu/daily-push-agent/main.py

@@ -15,7 +15,7 @@
 import xml.etree.ElementTree as ET
 from email.utils import parsedate_to_datetime
 from concurrent.futures import ThreadPoolExecutor, as_completed
-from datetime import datetime, timedelta
+from datetime import datetime, timedelta, date
 from pathlib import Path
 from typing import Any, Callable, List, Dict, Optional, Set
 from dataclasses import dataclass
@@ -37,7 +37,6 @@
 # 导入多个数据源
 arxiv_fetcher = importlib.import_module("skills.arxiv-fetcher.scripts.fetch_arxiv")
 arxiv_fetch_by_date = arxiv_fetcher.fetch_by_date
-arxiv_fetch_latest = getattr(arxiv_fetcher, "fetch_latest", None)
 arxiv_fetch_recent_list_page = getattr(arxiv_fetcher, "fetch_recent_list_page", None)
 
 openreview_fetcher = importlib.import_module("skills.openreview-fetcher.scripts.fetch_openreview")
@@ -132,6 +131,25 @@ def _parse_feed_date(value: Any) -> str:
         return text[:10]
 
 
+def _parse_iso_date(value: Any) -> Optional[date]:
+    text = str(value or "").strip()
+    if not text:
+        return None
+    for fmt, width in (("%Y-%m-%d", 10), ("%Y%m%d", 8)):
+        try:
+            return datetime.strptime(text[:width], fmt).date()
+        except ValueError:
+            pass
+    return None
+
+
+def _paper_date_in_window(paper: Dict[str, Any], start_date: date, end_date: date) -> bool:
+    paper_date = _parse_iso_date((paper or {}).get("publish_date") or (paper or {}).get("published"))
+    if paper_date is None:
+        return True
+    return start_date <= paper_date <= end_date
+
+
 def _xml_text(node: ET.Element, *paths: str) -> str:
     namespaces = {
         "atom": "http://www.w3.org/2005/Atom",
@@ -1113,12 +1131,14 @@ def emit_fetched(source: str, papers: List[Dict]) -> None:
     except ValueError:
         today = datetime.now()
     end_date = today.strftime("%Y%m%d")
+    end_day = today.date()
 
     def fetch_arxiv_papers(fetch_days: int) -> List[Dict]:
         if not arxiv_categories:
             print("Skipping arXiv fetch: no categories selected")
             return []
-        start_date = (today - timedelta(days=fetch_days)).strftime("%Y%m%d")
+        start_day = (today - timedelta(days=fetch_days)).date()
+        start_date = start_day.strftime("%Y%m%d")
         papers: List[Dict] = []
         per_category_limit = max(1, int(math.ceil(limit_per_source / max(1, len(arxiv_categories)))))
         for category in arxiv_categories:
@@ -1129,12 +1149,10 @@ def fetch_arxiv_papers(fetch_days: int) -> List[Dict]:
                 categories=[category],
                 limit=per_category_limit,
             ) or []
-            if not category_papers and callable(arxiv_fetch_latest):
-                print(f"  Date-bounded arXiv fetch returned 0 papers for {category}; fetching latest via API...")
-                category_papers = arxiv_fetch_latest(
-                    categories=[category],
-                    limit=per_category_limit,
-                ) or []
+            category_papers = [
+                paper for paper in category_papers
+                if _paper_date_in_window(paper, start_day, end_day)
+            ]
             if not category_papers and callable(arxiv_fetch_recent_list_page):
                 print(f"  arXiv API returned 0 papers for {category}; fetching recent list page...")
                 category_papers = arxiv_fetch_recent_list_page(
@@ -1190,16 +1208,19 @@ def fetch_cached_arxiv_papers() -> List[Dict]:
         if not arxiv_categories:
             return []
         try:
+            start_day = (today - timedelta(days=max(1, int(days or 1)))).date()
             conn = db_ops.get_connection()
             cursor = conn.cursor()
             cursor.execute(
                 """
                 SELECT * FROM papers
                 WHERE arxiv_id IS NOT NULL AND TRIM(arxiv_id) != ''
+                  AND publish_date >= ?
+                  AND publish_date <= ?
                 ORDER BY fetched_at DESC, id DESC
                 LIMIT ?
                 """,
-                (max(1, int(limit_per_source)),),
+                (start_day.isoformat(), end_day.isoformat(), max(1, int(limit_per_source))),
             )
             rows = cursor.fetchall()
             conn.close()

skills/arxiv-fetcher/scripts/fetch_arxiv.py

@@ -93,17 +93,22 @@ def _parse_arxiv_list_page(html: str, category: str, limit: int) -> List[Dict]:
     pairs = re.findall(r"<dt>(.*?)</dt>\s*<dd>(.*?)</dd>", html or "", flags=re.DOTALL | re.IGNORECASE)
     papers: List[Dict] = []
     for dt_html, dd_html in pairs:
-        id_match = re.search(r'href="/abs/([^"]+)"', dt_html)
+        id_match = re.search(r"""href\s*=\s*["']/abs/([^"']+)["']""", dt_html, flags=re.IGNORECASE)
         if not id_match:
             continue
         arxiv_id = unescape(id_match.group(1)).strip()
         title_match = re.search(
-            r'<div class="list-title[^"]*">\s*<span[^>]*>\s*Title:\s*</span>(.*?)</div>',
+            r"""<div\s+class=["']list-title[^"']*["']>\s*<span[^>]*>\s*Title:\s*</span>(.*?)</div>""",
             dd_html,
             flags=re.DOTALL | re.IGNORECASE,
         )
         author_block = re.search(
-            r'<div class="list-authors[^"]*">(.*?)</div>',
+            r"""<div\s+class=["']list-authors[^"']*["']>(.*?)</div>""",
+            dd_html,
+            flags=re.DOTALL | re.IGNORECASE,
+        )
+        subjects_block = re.search(
+            r"""<div\s+class=["']list-subjects[^"']*["']>(.*?)</div>""",
             dd_html,
             flags=re.DOTALL | re.IGNORECASE,
         )
@@ -115,13 +120,17 @@ def _parse_arxiv_list_page(html: str, category: str, limit: int) -> List[Dict]:
         title = _clean_html_text(title_match.group(1) if title_match else "")
         if not title:
             continue
+        subjects_text = _clean_html_text(subjects_block.group(1) if subjects_block else "")
+        categories = re.findall(r"\(([a-z-]+(?:\.[A-Z]{2})?)\)", subjects_text)
+        if category and category not in categories:
+            categories.insert(0, category)
         papers.append(
             {
                 "arxiv_id": arxiv_id,
                 "title": title,
                 "authors": [author for author in authors if author],
                 "abstract": "",
-                "categories": [category],
+                "categories": categories or [category],
                 "publish_date": "",
                 "url": f"https://arxiv.org/abs/{arxiv_id}",
                 "paper_url": f"https://arxiv.org/abs/{arxiv_id}",
@@ -134,12 +143,12 @@ def _parse_arxiv_list_page(html: str, category: str, limit: int) -> List[Dict]:
 
 
 def fetch_recent_list_page(category: str, limit: int = 100) -> List[Dict]:
-    """Fetch recent papers from arxiv.org/list/<category>/recent as an API fallback."""
+    """Fetch recent papers from arxiv.org/list/<archive>/recent as an API fallback."""
     category = str(category or "").strip()
     if not category:
         return []
     archive = category.split(".", 1)[0]
-    url = f"{ARXIV_LIST_URL}/{urllib.parse.quote(archive, safe='-')}/new"
+    url = f"{ARXIV_LIST_URL}/{urllib.parse.quote(archive, safe='-')}/recent?skip=0&show=2000"
     try:
         print(f"Fetching arXiv recent list page for {category}...")
         response = requests.get(url, timeout=DEFAULT_REQUEST_TIMEOUT, headers=DEFAULT_REQUEST_HEADERS)

tests/test_desktop_gui.py

@@ -13,6 +13,7 @@
 from deployments.desktop.shared import agents
 
 reading_agent = importlib.import_module("agents.reading-agent.main")
+arxiv_fetcher = importlib.import_module("skills.arxiv-fetcher.scripts.fetch_arxiv")
 
 PROJECT_ROOT = Path(__file__).resolve().parents[1]
 
@@ -1555,6 +1556,81 @@ def fake_daily_push(**kwargs):
     assert captured["custom_rss_urls"] == ["https://example.com/feed.xml"]
 
 
+def test_daily_arxiv_fetch_filters_out_old_papers(monkeypatch: pytest.MonkeyPatch) -> None:
+    monkeypatch.setattr(
+        agents.daily_agent,
+        "arxiv_fetch_by_date",
+        lambda **_kwargs: [
+            {
+                "title": "Old May Biomolecular Paper",
+                "arxiv_id": "2605.99999v1",
+                "publish_date": "2026-05-20",
+                "categories": ["q-bio.BM"],
+            }
+        ],
+    )
+    monkeypatch.setattr(agents.daily_agent, "arxiv_fetch_recent_list_page", lambda **_kwargs: [])
+    monkeypatch.setattr(agents.daily_agent, "openreview_fetch_by_date", lambda **_kwargs: [])
+    monkeypatch.setattr(agents.daily_agent, "journal_fetch_recent", lambda **_kwargs: [])
+    monkeypatch.setattr(agents.daily_agent, "prepare_paper_features", lambda papers: papers)
+
+    class EmptyRows:
+        def fetchall(self):
+            return []
+
+    class EmptyConnection:
+        def cursor(self):
+            return self
+
+        def execute(self, *_args, **_kwargs):
+            return EmptyRows()
+
+        def fetchall(self):
+            return []
+
+        def close(self):
+            return None
+
+    monkeypatch.setattr(agents.daily_agent.db_ops, "get_connection", lambda: EmptyConnection())
+
+    papers = agents.daily_agent.fetch_and_process_papers(
+        days=1,
+        arxiv_categories=["q-bio.BM"],
+        conferences=[],
+        journals=[],
+        enable_semantic_scholar=False,
+        enable_custom_rss=False,
+        target_date="2026-06-19",
+    )
+
+    assert papers == []
+
+
+def test_arxiv_recent_list_parser_handles_current_html_shape() -> None:
+    html = """
+<dt>
+  <a href ="/abs/2606.20489" title="Abstract" id="2606.20489">arXiv:2606.20489</a>
+</dt>
+<dd>
+  <div class='meta'>
+    <div class='list-title mathjax'><span class='descriptor'>Title:</span>
+      West Nile virus outbreak in Italy modelled with the quantum Game of Life
+    </div>
+    <div class='list-authors'><a href="/search/q-bio?query=Fontana">Andrea Fontana</a></div>
+    <div class='list-subjects'><span class='descriptor'>Subjects:</span>
+      <span class="primary-subject">Populations and Evolution (q-bio.PE)</span>; Applications (stat.AP)
+    </div>
+  </div>
+</dd>
+"""
+    papers = arxiv_fetcher._parse_arxiv_list_page(html, "q-bio.BM", limit=10)  # noqa: SLF001
+
+    assert papers[0]["arxiv_id"] == "2606.20489"
+    assert papers[0]["title"].startswith("West Nile virus")
+    assert papers[0]["authors"] == ["Andrea Fontana"]
+    assert papers[0]["categories"] == ["q-bio.BM", "q-bio.PE", "stat.AP"]
+
+
 def test_desktop_relevance_threshold_changes_daily_push_weights(monkeypatch: pytest.MonkeyPatch) -> None:
     base = {
         "threshold_high_relevant": 0.40,
@@ -1741,6 +1817,15 @@ def test_desktop_paper_source_defaults_are_arxiv_only() -> None:
     assert "selectedSettingConferences()" not in collect_daily_options
 
 
+def test_desktop_paper_cards_show_arxiv_category_tags() -> None:
+    script = (PROJECT_ROOT / "deployments/desktop/static/desktop.js").read_text(encoding="utf-8")
+
+    assert "function paperDisplayTags(paper)" in script
+    assert "paperDisplayTags(paper)" in script
+    assert "paper?.categories" in script
+    assert "const tags = paper.tags || [paper.source || ui().papers.sourceFallback, ui().papers.profileMatch]" not in script
+
+
 def test_desktop_settings_controls_use_compact_typography() -> None:
     css = (PROJECT_ROOT / "deployments/desktop/static/desktop.css").read_text(encoding="utf-8")