Skip to content

Fix dashboard backend leaks #33

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
Protocol-zero-0 merged 1 commit into from
Jun 8, 2026
Merged

Fix dashboard backend leaks #33

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
85 changes: 75 additions & 10 deletions tests/test_web_app.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,20 +22,28 @@ class WebAppTests(unittest.TestCase):
def setUp(self):
self.client = web_app.app.test_client()

def test_api_events_serializes_datetime_payloads(self):
def test_api_events_returns_short_polling_json_and_serializes_datetimes(self):
with mock.patch.object(
web_app,
"get_events",
side_effect=[
[{"seq": 1, "created_at": datetime(2026, 4, 21, 12, 0, 0)}],
[{"seq": 1, "created_at": datetime(2026, 4, 21, 12, 0, 0)}],
],
):
response = self.client.get("/api/events")
first_chunk = next(response.response).decode("utf-8")
return_value=[{"seq": 6, "created_at": datetime(2026, 4, 21, 12, 0, 0)}],
) as get_events:
response = self.client.get("/api/events?since=5")
payload = response.get_json()

get_events.assert_called_once_with(5)
self.assertEqual(response.status_code, 200)
self.assertEqual(response.mimetype, "application/json")
self.assertEqual(payload["events"][0]["created_at"], "2026-04-21 12:00:00")
self.assertEqual(payload["events"][0]["seq"], 6)
self.assertEqual(payload["next_seq"], 7)

self.assertIn("2026-04-21 12:00:00", first_chunk)
self.assertIn('"seq": 1', first_chunk)
def test_read_requests_roll_back_open_transaction_on_teardown(self):
with mock.patch.object(web_app.db, "rollback") as rollback:
response = self.client.get("/api/meta")

self.assertEqual(response.status_code, 200)
rollback.assert_called()

def test_api_meta_includes_database_backend_summary(self):
response = self.client.get("/api/meta")
Expand Down Expand Up @@ -127,6 +135,63 @@ def test_dashboard_i18n_keys_match_and_static_labels_are_marked(self):
):
self.assertIn(f'"{key}"', i18n)

def test_dashboard_stat_cards_have_i18n_tooltips(self):
expected_tips = {
"sourcePapers": (
"Papers extracted/parsed (excludes fetched-but-unprocessed)",
"已抽取/解析完成的文献数(不含仅入库未处理的)",
),
"results": (
"Benchmark result records produced by experiments",
"基准实验产出的结果记录条数",
),
"researchAreas": (
"Nodes in the research-area taxonomy tree",
"研究领域分类树的节点数",
),
"contradictions": (
"Pairs of conflicting claims found across papers",
"从文献中发现的互相矛盾的论断对数",
),
"researchInsights": (
"Research insights (insights table; distinct from Discoveries)",
"研究洞见条数(insights 表;与\"深度发现\"是不同的表)",
),
"tokens": (
"Total LLM tokens consumed processing papers",
"处理文献累计消耗的 LLM token 总量",
),
"experimentRuns": (
"Total experiment runs",
"实验运行的总次数",
),
"discoveries": (
"Discoveries that can grow into papers (deep_insights table; distinct from Research Insights)",
"可发展成论文的深度发现条数(deep_insights 表;与\"研究洞见\"是不同的表)",
),
"submissionBundles": (
"Completed paper bundles ready for submission",
"打包完成、可投稿的完整论文数",
),
}
html = _read("web/templates/index.html")
i18n = _read("web/static/js/i18n.js")

self.assertEqual(html.count('class="stat-card'), 9)
self.assertEqual(html.count("data-i18n-title=\"overview."), 9)
for key, (en_tip, zh_tip) in expected_tips.items():
i18n_key = f"overview.{key}.tip"
self.assertIn(f'data-i18n-title="{i18n_key}"', html)
self.assertIn(f'"{i18n_key}": {json.dumps(en_tip, ensure_ascii=False)}', i18n)
self.assertIn(f'"{i18n_key}": {json.dumps(zh_tip, ensure_ascii=False)}', i18n)

def test_dashboard_events_use_short_polling_not_eventsource(self):
app_js = _read("web/static/js/app.js")
self.assertNotIn("EventSource", app_js)
self.assertIn("function fetchEvents()", app_js)
self.assertIn("/api/events?since=", app_js)
self.assertIn("setInterval(fetchEvents, 2000)", app_js)

def test_dashboard_legacy_visible_labels_are_gone(self):
frontend = "\n".join(
_read(path)
Expand Down
39 changes: 23 additions & 16 deletions web/app.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@
import traceback
from pathlib import Path
from typing import Any
from flask import Flask, Response, abort, jsonify, render_template, request, send_file, url_for
from flask import Flask, abort, jsonify, render_template, request, send_file, url_for
from agents.workspace_layout import get_idea_workspace, list_paper_assets, plan_file_path, resolve_paper_asset
from config import APP_NAME, APP_SUBTITLE, PROFILE, ROOT_NODE_ID
from db import database as db
Expand Down Expand Up @@ -83,6 +83,14 @@ def block_manual_experiment_post_apis():
return _manual_api_removed_response()


@app.teardown_request
def rollback_request_transaction(_exc):
try:
db.rollback()
except Exception:
pass
Comment on lines +86 to +91


def _pick_canonical_run(runs: list[dict], canonical_run_id: int | None = None) -> dict | None:
if not runs:
return None
Expand Down Expand Up @@ -1144,24 +1152,23 @@ def api_matrix_gaps():
return jsonify(rows)


# ── Events (SSE) ──────────────────────────────────────────────────
# ── Events ────────────────────────────────────────────────────────

@app.route("/api/events")
def api_events():
"""SSE endpoint for real-time updates."""
def generate():
# Start from near the end - only send last 20 events on connect
all_events = get_events(0)
last_seq = max(0, all_events[-1]["seq"] - 20) if all_events else 0
while True:
events = get_events(last_seq)
for e in events:
yield f"data: {json.dumps(e, ensure_ascii=False, default=str)}\n\n"
last_seq = e["seq"] + 1
time.sleep(2) # slower polling = less browser load

return Response(generate(), mimetype="text/event-stream",
headers={"Cache-Control": "no-cache", "X-Accel-Buffering": "no"})
"""Short-poll pipeline events without holding a web worker thread."""
since = max(0, request.args.get("since", 0, type=int) or 0)
events = get_events(since)
payload_events = json.loads(json.dumps(events, ensure_ascii=False, default=str))
next_seq = since
Comment on lines +1159 to +1163
for event in payload_events:
try:
next_seq = max(next_seq, int(event.get("seq", since)) + 1)
except (TypeError, ValueError):
pass
response = jsonify({"events": payload_events, "next_seq": next_seq})
response.headers["Cache-Control"] = "no-cache"
return response


# ── Pipeline Control ──────────────────────────────────────────────
Expand Down
41 changes: 12 additions & 29 deletions web/static/js/app.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ const ROOT_NODE = document.body.dataset.rootNode || 'ml';
let activeTab = 'overview';
let exploreNodeId = ROOT_NODE;
let exploreData = null; // cached /api/taxonomy/<id> response
let eventSource = null;
let eventsSince = 0;
let events = []; // max 50
let activePapers = {}; // paper_id -> {title, step, startTime}
let statsCache = null;
Expand Down Expand Up @@ -176,41 +176,23 @@ async function refreshStats() {
}
}

// ── SSE Event Stream ─────────────────────────────────────────────────
// ── Event Polling ────────────────────────────────────────────────────

let sseRetryDelay = 2000;

function startSSE() {
if (eventSource) {
try { eventSource.close(); } catch(e) {}
eventSource = null;
}
eventSource = new EventSource('/api/events');

eventSource.onopen = () => {
sseRetryDelay = 2000;
};

eventSource.onmessage = (msg) => {
try {
const ev = JSON.parse(msg.data);
async function fetchEvents() {
try {
const payload = await api(`/api/events?since=${eventsSince}`);
eventsSince = payload.next_seq || eventsSince;
Comment on lines +179 to +184
for (const ev of payload.events || []) {
events.push(ev);
if (events.length > 50) events.shift();

trackPaperEvent(ev);
updateLiveBadge(ev);
appendFeedEvent(ev);
} catch (e) {
console.error('SSE parse error:', e);
}
};

eventSource.onerror = () => {
eventSource.close();
eventSource = null;
setTimeout(startSSE, sseRetryDelay);
sseRetryDelay = Math.min(sseRetryDelay * 1.5, 15000);
};
} catch (e) {
console.error('Event polling error:', e);
}
}

let pipelineRunning = false;
Expand Down Expand Up @@ -2608,7 +2590,8 @@ function init() {
// Initial data loads
refreshStats();
loadRecentlyDiscovered();
startSSE();
fetchEvents();
setInterval(fetchEvents, 2000);

// Stats refresh every 15s
statsTimer = setInterval(refreshStats, 15000);
Expand Down
18 changes: 18 additions & 0 deletions web/static/js/i18n.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,15 @@
"overview.experimentRuns": "Experiment Run",
"overview.discoveries": "Discovery",
"overview.submissionBundles": "Submission Bundle",
"overview.sourcePapers.tip": "Papers extracted/parsed (excludes fetched-but-unprocessed)",
"overview.results.tip": "Benchmark result records produced by experiments",
"overview.researchAreas.tip": "Nodes in the research-area taxonomy tree",
"overview.contradictions.tip": "Pairs of conflicting claims found across papers",
"overview.researchInsights.tip": "Research insights (insights table; distinct from Discoveries)",
"overview.tokens.tip": "Total LLM tokens consumed processing papers",
"overview.experimentRuns.tip": "Total experiment runs",
"overview.discoveries.tip": "Discoveries that can grow into papers (deep_insights table; distinct from Research Insights)",
"overview.submissionBundles.tip": "Completed paper bundles ready for submission",
"overview.latest": "Latest Activity",
"overview.latestEmpty": "Run the pipeline to discover gaps, contradictions, opportunities, and discoveries.",
"overview.processing": "Current Processing",
Expand Down Expand Up @@ -436,6 +445,15 @@
"overview.experimentRuns": "实验运行",
"overview.discoveries": "深度发现",
"overview.submissionBundles": "投稿包",
"overview.sourcePapers.tip": "已抽取/解析完成的文献数(不含仅入库未处理的)",
"overview.results.tip": "基准实验产出的结果记录条数",
"overview.researchAreas.tip": "研究领域分类树的节点数",
"overview.contradictions.tip": "从文献中发现的互相矛盾的论断对数",
"overview.researchInsights.tip": "研究洞见条数(insights 表;与\"深度发现\"是不同的表)",
"overview.tokens.tip": "处理文献累计消耗的 LLM token 总量",
"overview.experimentRuns.tip": "实验运行的总次数",
"overview.discoveries.tip": "可发展成论文的深度发现条数(deep_insights 表;与\"研究洞见\"是不同的表)",
"overview.submissionBundles.tip": "打包完成、可投稿的完整论文数",
"overview.latest": "最新动态",
"overview.latestEmpty": "运行流水线以发现空白、矛盾、机会点和深度发现。",
"overview.processing": "当前处理",
Expand Down
18 changes: 9 additions & 9 deletions web/templates/index.html
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -91,63 +91,63 @@
<div class="tab-scroll">
<!-- Stat cards -->
<div class="stat-grid" id="overviewStats">
<div class="stat-card">
<div class="stat-card" data-i18n-title="overview.sourcePapers.tip">
<div class="stat-icon stat-icon-blue">
<svg width="22" height="22" viewBox="0 0 20 20" fill="none"><path d="M4 3h8l4 4v10a1 1 0 01-1 1H4a1 1 0 01-1-1V4a1 1 0 011-1z" stroke="currentColor" stroke-width="1.5"/><path d="M12 3v4h4" stroke="currentColor" stroke-width="1.5"/></svg>
</div>
<div class="stat-number" id="statPapers">0</div>
<div class="stat-label" data-i18n="overview.sourcePapers">Source Paper</div>
</div>
<div class="stat-card">
<div class="stat-card" data-i18n-title="overview.results.tip">
<div class="stat-icon stat-icon-cyan">
<svg width="22" height="22" viewBox="0 0 20 20" fill="none"><path d="M3 5h14M3 10h14M3 15h14" stroke="currentColor" stroke-width="1.5" stroke-linecap="round"/></svg>
</div>
<div class="stat-number" id="statResults">0</div>
<div class="stat-label" data-i18n="overview.results">Result</div>
</div>
<div class="stat-card">
<div class="stat-card" data-i18n-title="overview.researchAreas.tip">
<div class="stat-icon stat-icon-gold">
<svg width="22" height="22" viewBox="0 0 20 20" fill="none"><path d="M10 2l2.5 5 5.5.8-4 3.9.9 5.3-4.9-2.6L5.1 17l.9-5.3-4-3.9 5.5-.8L10 2z" stroke="currentColor" stroke-width="1.5" stroke-linejoin="round"/></svg>
</div>
<div class="stat-number" id="statTaxonomy">0</div>
<div class="stat-label" data-i18n="overview.researchAreas">Research Area</div>
</div>
<div class="stat-card">
<div class="stat-card" data-i18n-title="overview.contradictions.tip">
<div class="stat-icon stat-icon-red">
<svg width="22" height="22" viewBox="0 0 20 20" fill="none"><path d="M10 3L2 17h16L10 3z" stroke="currentColor" stroke-width="1.5" stroke-linejoin="round"/><path d="M10 8v4M10 14v1" stroke="currentColor" stroke-width="1.5" stroke-linecap="round"/></svg>
</div>
<div class="stat-number" id="statContradictions">0</div>
<div class="stat-label" data-i18n="overview.contradictions">Contradiction</div>
</div>
<div class="stat-card">
<div class="stat-card" data-i18n-title="overview.researchInsights.tip">
<div class="stat-icon stat-icon-purple">
<svg width="22" height="22" viewBox="0 0 20 20" fill="none"><path d="M10 2C5.6 2 2 5.6 2 10s3.6 8 8 8 8-3.6 8-8-3.6-8-8-8zm0 3a1.5 1.5 0 110 3 1.5 1.5 0 010-3zm2 9H8v-1h1v-3H8v-1h3v4h1v1z" fill="currentColor"/></svg>
</div>
<div class="stat-number" id="statInsights">0</div>
<div class="stat-label" data-i18n="overview.researchInsights">Research Insight</div>
</div>
<div class="stat-card">
<div class="stat-card" data-i18n-title="overview.tokens.tip">
<div class="stat-icon stat-icon-purple">
<svg width="22" height="22" viewBox="0 0 20 20" fill="none"><path d="M3 14l4-4 3 3 7-7" stroke="currentColor" stroke-width="1.5" stroke-linecap="round" stroke-linejoin="round"/></svg>
</div>
<div class="stat-number" id="statTokens">0</div>
<div class="stat-label" data-i18n="overview.tokens">Token</div>
</div>
<div class="stat-card overview-secondary-stat">
<div class="stat-card overview-secondary-stat" data-i18n-title="overview.experimentRuns.tip">
<div class="stat-icon stat-icon-green">
<svg width="22" height="22" viewBox="0 0 20 20" fill="none"><path d="M10 2v6M7 5l3 3 3-3M4 10h12v7H4z" stroke="currentColor" stroke-width="1.5" stroke-linecap="round" stroke-linejoin="round"/><path d="M7 13h6" stroke="currentColor" stroke-width="1.5" stroke-linecap="round"/></svg>
</div>
<div class="stat-number" id="statExperiments">0</div>
<div class="stat-label" data-i18n="overview.experimentRuns">Experiment Run</div>
</div>
<div class="stat-card overview-secondary-stat">
<div class="stat-card overview-secondary-stat" data-i18n-title="overview.discoveries.tip">
<div class="stat-icon stat-icon-gold">
<svg width="22" height="22" viewBox="0 0 20 20" fill="none"><path d="M10 1L12.5 6.5 18.5 7.5 14 11.5 15 17.5 10 14.5 5 17.5 6 11.5 1.5 7.5 7.5 6.5Z" stroke="currentColor" stroke-width="1.5" stroke-linejoin="round"/></svg>
</div>
<div class="stat-number" id="statDeepDiscoveries">0</div>
<div class="stat-label" data-i18n="overview.discoveries">Discovery</div>
</div>
<div class="stat-card overview-secondary-stat">
<div class="stat-card overview-secondary-stat" data-i18n-title="overview.submissionBundles.tip">
<div class="stat-icon stat-icon-blue">
<svg width="22" height="22" viewBox="0 0 20 20" fill="none"><path d="M4 3h8l4 4v10a1 1 0 01-1 1H4a1 1 0 01-1-1V4a1 1 0 011-1z" stroke="currentColor" stroke-width="1.5"/><path d="M12 3v4h4M7 10l2 2 4-4" stroke="currentColor" stroke-width="1.5" stroke-linecap="round" stroke-linejoin="round"/></svg>
</div>
Expand Down
Loading