diff --git a/inspector/templates/inspector/index.html b/inspector/templates/inspector/index.html index 27bd47f..f8079f2 100644 --- a/inspector/templates/inspector/index.html +++ b/inspector/templates/inspector/index.html @@ -19,12 +19,15 @@ button:hover { background:var(--primary-2); } button.secondary { background:#fff; color:var(--text); border-color:var(--line); } .status { margin-left:auto; font-size:13px; color:var(--muted); font-weight:600; } - .status.ok { color:var(--ok); } .status.err { color:var(--danger); } + .status.ok { color:var(--ok); } + .status.err { color:var(--danger); } .table-wrap { overflow:auto; max-height:45vh; border:1px solid var(--line); border-radius:8px; } table { width:100%; border-collapse:collapse; background:#fff; min-width:980px; } th,td { border-bottom:1px solid var(--line); padding:8px; text-align:left; font-size:13px; vertical-align:top; } th { position:sticky; top:0; z-index:2; background:#f9fbff; font-size:12px; text-transform:uppercase; letter-spacing:.03em; color:var(--muted); } - tbody tr { cursor:pointer; } tbody tr:hover { background:#f3f7ff; } tbody tr.selected { background:#e8f0ff; } + tbody tr { cursor:pointer; } + tbody tr:hover { background:#f3f7ff; } + tbody tr.selected { background:#e8f0ff; } .detail-grid { display:grid; gap:10px; grid-template-columns:2fr 1fr; } .meta { font-size:13px; color:var(--muted); } .checks { display:grid; gap:8px; grid-template-columns:repeat(4,minmax(130px,1fr)); } @@ -37,8 +40,24 @@ .select-cell { width:38px; text-align:center; } .select-cell input { min-width:auto; min-height:auto; } .flash { font-size:13px; padding:8px 10px; border-radius:8px; border:1px solid var(--line); background:#f8faff; color:var(--muted); display:none; } - .flash.show { display:block; } .flash.err { border-color:#f5c2c7; background:#fff4f5; color:var(--danger);} .flash.ok { border-color:#b7ebd3; background:#f1fbf6; color:var(--ok);} - @media (max-width:980px) { .detail-grid,.query-grid { grid-template-columns:1fr; } .checks { grid-template-columns:repeat(2,minmax(130px,1fr)); } } + .flash.show { display:block; } + .flash.err { border-color:#f5c2c7; background:#fff4f5; color:var(--danger); } + .flash.ok { border-color:#b7ebd3; background:#f1fbf6; color:var(--ok); } + .tab-bar { display:flex; flex-wrap:wrap; gap:8px; } + .tab-btn { background:#fff; color:var(--muted); border-color:var(--line); } + .tab-btn.active { background:var(--primary); color:#fff; border-color:var(--primary); } + .tab-panel { display:none; } + .tab-panel.active { display:grid; gap:12px; } + .validator-grid { display:grid; gap:12px; grid-template-columns:2fr 1fr; align-items:start; } + .validator-actions { display:flex; flex-wrap:wrap; gap:8px; align-items:center; } + .pill { display:inline-flex; align-items:center; gap:6px; padding:5px 9px; border:1px solid var(--line); border-radius:999px; background:#f9fbff; font-size:12px; color:var(--muted); } + .hidden { display:none !important; } + @media (max-width:980px) { + .detail-grid, + .query-grid, + .validator-grid { grid-template-columns:1fr; } + .checks { grid-template-columns:repeat(2,minmax(130px,1fr)); } + } @@ -46,101 +65,189 @@ {{ query_specs|json_script:"query-specs" }}
-

Database Connection

-
- - - - - - - -
Not connected
+
+ +
-
-

Query

-
-
-
- - - - - - - +
+
+

Database Connection

+
+ + + + + + + +
Not connected
+
+
+ +
+

Query

+
+
+
+ + + + + + + + + +
+
+ +
+ + +
+
+
+
No identifier list loaded.
+
-
- -
-
+
+
+ +
+

Results

+
+ + + + + + + + + + + + + + +
scix_idBibcodeTitleScorerun_idvalidatedcollection
+
+
+ +
+

Details and Update

+
+
+
No row selected.
+
+ - +
+
+
+ +
-
-
No identifier list loaded.
+
+ +
-
Select a row to inspect details. Use the table checkboxes to choose one or more rows for update.
-
-
+
-
-

Results

-
- - - - - - - - - - - - - - -
scix_idBibcodeTitleScorerun_idvalidatedcollection
-
-
+
+
+

Pre-ingest Validator

+
+
+
+ + + + + + +
+
+ +
+
+
No validator file loaded.
+ +
+
+
Upload a classified TSV, inspect rows, and edit the override field. Edits stay in the browser until you export the updated file.
+
+
+
-
-

Details and Update

-
-
-
No row selected.
-
- +
+

Validator Results

+
+ + + + + + + + + + + + + + + + +
scix_idBibcodeTitleAstronomyPhysicsGeneralrun_idcollectionsoverride
+
+
+ +
+

Override Editor

+
+
+
No row selected.
+
+ Editing TSV override values only +
+
+
+ + +
-
-
- - +
+ +
-
- - -
-
+
@@ -150,6 +257,40 @@

Details and Update

const LARGE_BATCH_CONFIRM_THRESHOLD = 100; const BIBCODE_LIST_PRESET = "By bibcode list"; const SCIX_ID_LIST_PRESET = "By scix_id list"; + const VALIDATOR_SCORE_COLUMN_BY_CATEGORY = { + astrophysics: "astrophysics_score", + heliophysics: "heliophysics_score", + planetary: "planetary_science_score", + earthscience: "earth_science_score", + "NASA-funded Biophysics": "biology_score", + physics: "physics_score", + general: "general_score", + "Text Garbage": "garbage_score", + }; + const VALIDATOR_EXTRA_SCORE_COLUMNS = [ + ["astronomy", "astronomy_score"], + ["other", "other_score"], + ]; + const VALIDATOR_DETAIL_SCORE_ORDER = [ + "astrophysics", + "NASA-funded Biophysics", + "planetary", + "physics", + "general", + "heliophysics", + "earthscience", + "Text Garbage", + ]; + const VALIDATOR_GROSS_CATEGORIES = [ + { value: "astronomy", label: "Astronomy" }, + { value: "physics", label: "Physics" }, + { value: "general", label: "General" }, + ]; + const VALIDATOR_GROSS_CATEGORY_LABELS = Object.fromEntries( + VALIDATOR_GROSS_CATEGORIES.map((category) => [category.value, category.label]) + ); + + let allRowsData = []; let rowsData = []; let selectedIdx = -1; let selectedRowKeys = new Set(); @@ -157,6 +298,17 @@

Details and Update

let parsedIdentifierType = null; let scoreSortDir = "desc"; + let activeTab = "db_inspector"; + let validatorAllRows = []; + let validatorRows = []; + let validatorSelectedIdx = -1; + let validatorSelectedRowKeys = new Set(); + let validatorScoreSortKey = "astronomy_score"; + let validatorScoreSortDir = "desc"; + let validatorParsedFileName = ""; + let validatorDirty = false; + let validatorHeaders = []; + const el = (id) => document.getElementById(id); function dbPayload() { @@ -187,6 +339,28 @@

Details and Update

f.className = "flash"; } + function showValidatorFlash(text, type = "") { + const f = el("validator_flash"); + f.textContent = text; + f.className = `flash show ${type}`.trim(); + } + + function clearValidatorFlash() { + const f = el("validator_flash"); + f.textContent = ""; + f.className = "flash"; + } + + function setActiveTab(tabName) { + activeTab = tabName; + document.querySelectorAll(".tab-btn").forEach((btn) => { + btn.classList.toggle("active", btn.dataset.tab === tabName); + }); + document.querySelectorAll(".tab-panel").forEach((panel) => { + panel.classList.toggle("active", panel.id === `panel_${tabName}`); + }); + } + function parseIdentifiers(rawText, splitOnNewlinesOnly = false) { const pieces = splitOnNewlinesOnly ? rawText.split(/\r?\n/) : rawText.split(/[\r\n,]+/); const out = []; @@ -245,7 +419,69 @@

Details and Update

el("collection_label").textContent = "No row selected."; el("scores_text").value = ""; el("abstract_text").value = ""; - clearChecks(); + clearChecks("category_checks"); + } + + function parseOptionalBound(rawValue) { + const text = String(rawValue ?? "").trim(); + if (!text) { + return null; + } + const value = Number.parseFloat(text); + return Number.isFinite(value) ? value : null; + } + + function scoreWithinBounds(rawScore, minValue, maxValue) { + const numericScore = Number.parseFloat(rawScore); + if (!Number.isFinite(numericScore)) { + return minValue === null && maxValue === null; + } + if (minValue !== null && numericScore < minValue) { + return false; + } + if (maxValue !== null && numericScore > maxValue) { + return false; + } + return true; + } + + function syncSelectedIndexByKey(rows, selectedRowKey) { + if (!selectedRowKey) { + return -1; + } + return rows.findIndex((row) => row.row_key === selectedRowKey); + } + + function resetDbDetailPane() { + el("collection_label").textContent = "No row selected."; + el("scores_text").value = ""; + el("abstract_text").value = ""; + clearChecks("category_checks"); + } + + function applyDbScoreFilter() { + const selectedRowKey = rowsData[selectedIdx]?.row_key; + const minValue = parseOptionalBound(el("score_min").value); + const maxValue = parseOptionalBound(el("score_max").value); + rowsData = allRowsData.filter((row) => scoreWithinBounds(row.score, minValue, maxValue)); + selectedRowKeys = new Set([...selectedRowKeys].filter((rowKey) => rowsData.some((row) => row.row_key === rowKey))); + selectedIdx = syncSelectedIndexByKey(rowsData, selectedRowKey); + if (selectedIdx < 0) { + resetDbDetailPane(); + } + sortRowsByScore(); + renderRows(); + } + + function selectedRowsForUpdate() { + const checkedRows = rowsData.filter((row) => selectedRowKeys.has(row.row_key)); + if (checkedRows.length) { + return checkedRows; + } + if (selectedIdx >= 0 && selectedIdx < rowsData.length) { + return [rowsData[selectedIdx]]; + } + return []; } async function runQueryWithPreset(presetOverride = null) { @@ -268,13 +504,8 @@

Details and Update

ads_token: el("ads_token").value, }); - rowsData = data.rows || []; - if (preset !== BIBCODE_LIST_PRESET && preset !== SCIX_ID_LIST_PRESET) { - sortRowsByScore(); - } else { - el("score_col_header").textContent = "Score"; - } - renderRows(); + allRowsData = data.rows || []; + applyDbScoreFilter(); setStatus("Connected", "ok"); let msg = `Loaded ${data.count} rows`; @@ -314,17 +545,17 @@

Details and Update

}); } - function buildCategoryChecks() { - const box = el("category_checks"); + function buildCategoryChecks(containerId, categories = allowedCategories.map((category) => ({ value: category, label: category }))) { + const box = el(containerId); box.innerHTML = ""; - allowedCategories.forEach((cat) => { + categories.forEach((category) => { const label = document.createElement("label"); const cb = document.createElement("input"); cb.type = "checkbox"; - cb.value = cat; - cb.dataset.category = cat; + cb.value = category.value; + cb.dataset.category = category.value; const t1 = document.createElement("span"); - t1.textContent = cat; + t1.textContent = category.label; const t2 = document.createElement("span"); t2.className = "meta"; t2.textContent = "0.00"; @@ -333,12 +564,14 @@

Details and Update

}); } - function selectedCategories() { - return Array.from(document.querySelectorAll("#category_checks input[type=checkbox]:checked")).map((i) => i.value); + function selectedCategories(containerId) { + return Array.from(document.querySelectorAll(`#${containerId} input[type=checkbox]:checked`)).map((i) => i.value); } - function clearChecks() { - document.querySelectorAll("#category_checks input[type=checkbox]").forEach((cb) => { cb.checked = false; }); + function clearChecks(containerId) { + document.querySelectorAll(`#${containerId} input[type=checkbox]`).forEach((cb) => { + cb.checked = false; + }); } async function api(path, method = "GET", payload = null) { @@ -357,7 +590,9 @@

Details and Update

function rowToTr(row, idx) { const tr = document.createElement("tr"); - if (idx === selectedIdx) { tr.classList.add("selected"); } + if (idx === selectedIdx) { + tr.classList.add("selected"); + } const checked = selectedRowKeys.has(row.row_key) ? "checked" : ""; tr.innerHTML = ` @@ -399,7 +634,7 @@

Details and Update

function patchUpdatedRows(updatedRowKeys, categories, validated) { const categoryCopy = [...categories]; - rowsData = rowsData.map((row) => { + allRowsData = allRowsData.map((row) => { if (!updatedRowKeys.has(row.row_key)) { return row; } @@ -418,6 +653,7 @@

Details and Update

validated, }; }); + applyDbScoreFilter(); } function scoreToNumber(v) { @@ -456,29 +692,487 @@

Details and Update

el("scores_text").value = detail.scores_text; el("abstract_text").value = detail.abstract_text; - const infoByName = Object.fromEntries(detail.categories.map((c) => [c.name, c])); - const labelNodes = Array.from(document.querySelectorAll("#category_checks label")); - labelNodes.forEach((label) => { - const cb = label.querySelector("input"); - const name = cb.dataset.category; - const info = infoByName[name]; - const line1 = label.querySelector("span:nth-of-type(1)"); - const line2 = label.querySelector("span:nth-of-type(2)"); - cb.checked = !!info?.checked; - - let tags = ""; - if (info && info.tags && info.tags.length) { - tags = ` [${info.tags.join(" / ")}]`; - } - line1.textContent = `${name}${tags}`; - line2.textContent = info ? info.score : "0.00"; - }); + populateChecks("category_checks", detail.categories); } catch (err) { el("abstract_text").value = `(Failed to load details: ${err.message})`; showFlash(err.message, "err"); } } + function populateChecks(containerId, categories) { + const infoByName = Object.fromEntries(categories.map((c) => [c.name, c])); + const labelNodes = Array.from(document.querySelectorAll(`#${containerId} label`)); + labelNodes.forEach((label) => { + const cb = label.querySelector("input"); + const name = cb.dataset.category; + const info = infoByName[name]; + const line1 = label.querySelector("span:nth-of-type(1)"); + const line2 = label.querySelector("span:nth-of-type(2)"); + cb.checked = !!info?.checked; + + let tags = ""; + if (info && info.tags && info.tags.length) { + tags = ` [${info.tags.join(" / ")}]`; + } + line1.textContent = `${info?.label || name}${tags}`; + line2.textContent = info ? info.score : "0.00"; + }); + } + + function validatorScoreMap(rawRow) { + const scores = {}; + allowedCategories.forEach((category) => { + const column = VALIDATOR_SCORE_COLUMN_BY_CATEGORY[category]; + scores[category] = toNumericString(rawRow[column] || "0"); + }); + VALIDATOR_EXTRA_SCORE_COLUMNS.forEach(([label, column]) => { + scores[label] = toNumericString(rawRow[column] || "0"); + }); + return scores; + } + + function splitCategoryList(rawValue) { + return String(rawValue || "") + .split(",") + .map((item) => item.trim()) + .filter(Boolean); + } + + function splitGrossCategoryList(rawValue) { + const allowedValues = new Set(VALIDATOR_GROSS_CATEGORIES.map((category) => category.value)); + return splitCategoryList(rawValue) + .map((item) => item.toLowerCase()) + .filter((item) => allowedValues.has(item)); + } + + function formatGrossCategoryList(categories) { + return categories.map((category) => VALIDATOR_GROSS_CATEGORY_LABELS[category] || category).join(", "); + } + + function toNumericString(rawValue) { + const n = Number.parseFloat(String(rawValue || "").trim()); + return Number.isFinite(n) ? n.toFixed(2) : "0.00"; + } + + function formatScoreMap(scoresMap) { + return VALIDATOR_DETAIL_SCORE_ORDER + .map((name) => `${name}: ${scoresMap[name] || "0.00"}`) + .join("\n"); + } + + function buildValidatorRow(rawRow, idx) { + const scoresMap = validatorScoreMap(rawRow); + return { + row_key: `validator:${idx}`, + row_idx: idx, + raw: rawRow, + bibcode: String(rawRow.bibcode || "").trim(), + scix_id: String(rawRow.scix_id || "").trim(), + run_id: String(rawRow.run_id || "").trim(), + title: String(rawRow.title || "").trim(), + collections: splitGrossCategoryList(rawRow.gross_collection || rawRow.collections), + override: splitGrossCategoryList(rawRow.override), + scoresMap, + astronomy_score: scoresMap.astronomy || "0.00", + physics_score: scoresMap.physics || "0.00", + general_score: scoresMap.general || "0.00", + }; + } + + function parseDelimitedLine(line, delimiter) { + const out = []; + let current = ""; + let inQuotes = false; + for (let idx = 0; idx < line.length; idx += 1) { + const ch = line[idx]; + if (ch === '"') { + if (inQuotes && line[idx + 1] === '"') { + current += '"'; + idx += 1; + } else { + inQuotes = !inQuotes; + } + } else if (ch === delimiter && !inQuotes) { + out.push(current); + current = ""; + } else { + current += ch; + } + } + out.push(current); + return out; + } + + function parseTsv(text) { + const lines = text.split(/\r?\n/).filter((line) => line.trim().length > 0); + if (!lines.length) { + throw new Error("The uploaded file is empty."); + } + const headers = parseDelimitedLine(lines[0], "\t").map((header) => header.trim()); + const requiredHeaders = ["title", "collections", "override"]; + const missingHeaders = requiredHeaders.filter((header) => !headers.includes(header)); + if (missingHeaders.length) { + throw new Error(`Missing required TSV columns: ${missingHeaders.join(", ")}`); + } + + const rawRows = lines.slice(1).map((line, idx) => { + const cells = parseDelimitedLine(line, "\t"); + const row = {}; + headers.forEach((header, headerIdx) => { + row[header] = cells[headerIdx] ?? ""; + }); + row.__line_index = idx; + return row; + }); + + return { headers, rawRows }; + } + + function syncValidatorDirtyPill() { + el("validator_dirty_pill").classList.toggle("hidden", !validatorDirty); + } + + function syncValidatorSelectedIndex(selectedRowKey) { + if (!selectedRowKey) { + return -1; + } + return validatorRows.findIndex((row) => row.row_key === selectedRowKey); + } + + function clearValidatorSelectionState() { + validatorSelectedIdx = -1; + validatorSelectedRowKeys = new Set(); + el("validator_select_all_rows").checked = false; + el("validator_collection_label").textContent = "No row selected."; + el("validator_scores_text").value = ""; + el("validator_title_text").value = ""; + clearChecks("validator_category_checks"); + } + + function applyValidatorScoreFilter() { + const selectedRowKey = validatorRows[validatorSelectedIdx]?.row_key; + const minValue = parseOptionalBound(el("validator_score_min").value); + const maxValue = parseOptionalBound(el("validator_score_max").value); + validatorRows = validatorAllRows.filter((row) => scoreWithinBounds(row[validatorScoreSortKey], minValue, maxValue)); + validatorSelectedRowKeys = new Set( + [...validatorSelectedRowKeys].filter((rowKey) => validatorRows.some((row) => row.row_key === rowKey)) + ); + validatorSelectedIdx = syncValidatorSelectedIndex(selectedRowKey); + if (validatorSelectedIdx < 0) { + el("validator_collection_label").textContent = "No row selected."; + el("validator_scores_text").value = ""; + el("validator_title_text").value = ""; + clearChecks("validator_category_checks"); + } + sortValidatorRowsByScoreColumn(); + renderValidatorRows(); + } + + function selectedValidatorRowsForUpdate() { + const checkedRows = validatorRows.filter((row) => validatorSelectedRowKeys.has(row.row_key)); + if (checkedRows.length) { + return checkedRows; + } + if (validatorSelectedIdx >= 0 && validatorSelectedIdx < validatorRows.length) { + return [validatorRows[validatorSelectedIdx]]; + } + return []; + } + + function validatorRowToTr(row, idx) { + const tr = document.createElement("tr"); + if (idx === validatorSelectedIdx) { + tr.classList.add("selected"); + } + const checked = validatorSelectedRowKeys.has(row.row_key) ? "checked" : ""; + tr.innerHTML = ` + + ${row.scix_id || ""} + ${row.bibcode || ""} + ${row.title || ""} + ${row.astronomy_score || ""} + ${row.physics_score || ""} + ${row.general_score || ""} + ${row.run_id || ""} + ${formatGrossCategoryList(row.collections)} + ${formatGrossCategoryList(row.override)} + `; + const cb = tr.querySelector('input[type="checkbox"]'); + cb.addEventListener("click", (event) => { + event.stopPropagation(); + }); + cb.addEventListener("change", () => { + if (cb.checked) { + validatorSelectedRowKeys.add(row.row_key); + } else { + validatorSelectedRowKeys.delete(row.row_key); + } + syncValidatorSelectAllState(); + refreshValidatorSelectionDetail(); + }); + tr.addEventListener("click", () => selectValidatorRow(idx)); + return tr; + } + + function renderValidatorRows() { + const body = el("validator_rows"); + body.innerHTML = ""; + validatorRows.forEach((row, idx) => body.appendChild(validatorRowToTr(row, idx))); + syncValidatorSelectAllState(); + } + + function syncValidatorSelectAllState() { + const allChecked = validatorRows.length > 0 && validatorRows.every((row) => validatorSelectedRowKeys.has(row.row_key)); + el("validator_select_all_rows").checked = allChecked; + } + + function syncValidatorScoreHeaders() { + const headers = [ + ["validator_astronomy_score_col_header", "Astronomy", "astronomy_score"], + ["validator_physics_score_col_header", "Physics", "physics_score"], + ["validator_general_score_col_header", "General", "general_score"], + ]; + headers.forEach(([id, label, key]) => { + el(id).textContent = key === validatorScoreSortKey + ? `${label} ${validatorScoreSortDir === "asc" ? "▲" : "▼"}` + : label; + }); + } + + function sortValidatorRowsByScoreColumn() { + const factor = validatorScoreSortDir === "asc" ? 1 : -1; + validatorRows.sort((a, b) => (scoreToNumber(a[validatorScoreSortKey]) - scoreToNumber(b[validatorScoreSortKey])) * factor); + syncValidatorScoreHeaders(); + } + + function refreshValidatorSelectionDetail() { + if (validatorSelectedIdx < 0 || validatorSelectedIdx >= validatorRows.length) { + el("validator_collection_label").textContent = "No row selected."; + el("validator_scores_text").value = ""; + el("validator_title_text").value = ""; + clearChecks("validator_category_checks"); + return; + } + + const row = validatorRows[validatorSelectedIdx]; + const selectedCount = validatorSelectedRowKeys.size; + const suffix = selectedCount ? ` Rows selected for update: ${selectedCount}` : ""; + el("validator_collection_label").textContent = `Current gross collections: ${formatGrossCategoryList(row.collections) || "(none)"} Current override: ${formatGrossCategoryList(row.override) || "(none)"}${suffix}`; + el("validator_scores_text").value = formatScoreMap(row.scoresMap); + el("validator_title_text").value = row.title || "(No title)"; + + const collectionSet = new Set(row.collections); + const overrideSet = new Set(row.override); + const categories = VALIDATOR_GROSS_CATEGORIES.map((category) => ({ + name: category.value, + label: category.label, + score: row[`${category.value}_score`] || "0.00", + checked: overrideSet.has(category.value), + tags: [ + ...(collectionSet.has(category.value) ? ["C"] : []), + ...(overrideSet.has(category.value) ? ["O"] : []), + ], + })); + populateChecks("validator_category_checks", categories); + } + + function selectValidatorRow(idx) { + validatorSelectedIdx = idx; + renderValidatorRows(); + clearValidatorFlash(); + refreshValidatorSelectionDetail(); + } + + function applyValidatorOverrideUpdate() { + const selectedRows = selectedValidatorRowsForUpdate(); + if (!selectedRows.length) { + showValidatorFlash("Select a row first.", "err"); + return; + } + if (selectedRows.length >= LARGE_BATCH_CONFIRM_THRESHOLD) { + const confirmed = window.confirm(`You are about to update ${selectedRows.length} validator rows at once. Continue?`); + if (!confirmed) { + return; + } + } + + const categories = selectedCategories("validator_category_checks"); + const categoryString = categories.join(", "); + const selectedKeys = new Set(selectedRows.map((row) => row.row_key)); + const selectedRowKey = validatorRows[validatorSelectedIdx]?.row_key; + + validatorAllRows = validatorAllRows.map((row) => { + if (!selectedKeys.has(row.row_key)) { + return row; + } + const nextRaw = { ...row.raw, override: categoryString }; + return buildValidatorRow(nextRaw, row.row_idx); + }); + + validatorDirty = true; + validatorSelectedRowKeys = new Set(); + validatorSelectedIdx = syncValidatorSelectedIndex(selectedRowKey); + applyValidatorScoreFilter(); + if (validatorSelectedIdx >= 0 && validatorSelectedIdx < validatorRows.length) { + refreshValidatorSelectionDetail(); + } + syncValidatorDirtyPill(); + showValidatorFlash(`Updated override for ${selectedRows.length} row${selectedRows.length === 1 ? "" : "s"}.`, "ok"); + } + + function loadValidatorFile() { + const file = el("validator_file").files[0]; + if (!file) { + showValidatorFlash("Choose a TSV file first.", "err"); + return; + } + clearValidatorFlash(); + const reader = new FileReader(); + reader.onload = () => { + try { + const parsed = parseTsv(String(reader.result || "")); + validatorHeaders = parsed.headers; + validatorAllRows = parsed.rawRows.map((rawRow, idx) => buildValidatorRow(rawRow, idx)); + validatorParsedFileName = file.name; + el("validator_curated_filename").value = curatedFileNameFor(file.name); + validatorDirty = false; + validatorSelectedIdx = -1; + validatorSelectedRowKeys = new Set(); + applyValidatorScoreFilter(); + refreshValidatorSelectionDetail(); + syncValidatorDirtyPill(); + el("validator_file_status").textContent = `Loaded ${validatorAllRows.length} rows from ${validatorParsedFileName}. Showing ${validatorRows.length}.`; + showValidatorFlash(`Loaded ${validatorAllRows.length} validator rows. Showing ${validatorRows.length}.`, "ok"); + } catch (err) { + validatorAllRows = []; + validatorRows = []; + validatorHeaders = []; + validatorParsedFileName = ""; + clearValidatorSelectionState(); + renderValidatorRows(); + showValidatorFlash(err.message, "err"); + } + }; + reader.onerror = () => { + showValidatorFlash("Failed to read validator file.", "err"); + }; + reader.readAsText(file); + } + + function escapeTsvCell(value) { + const text = String(value ?? ""); + if (text.includes("\t") || text.includes("\n") || text.includes('"')) { + return `"${text.replace(/"/g, '""')}"`; + } + return text; + } + + function curatedFileNameFor(baseName) { + const text = String(baseName || "").trim(); + if (!text) { + return "validated_pre_ingest_curated.tsv"; + } + return text.endsWith(".tsv") ? text.replace(/\.tsv$/i, "_curated.tsv") : `${text}_curated.tsv`; + } + + function triggerTsvDownload(lines, fileName) { + const blob = new Blob([`${lines.join("\n")}\n`], { type: "text/tab-separated-values;charset=utf-8" }); + const url = URL.createObjectURL(blob); + const link = document.createElement("a"); + link.href = url; + link.download = fileName; + document.body.appendChild(link); + link.click(); + link.remove(); + URL.revokeObjectURL(url); + } + + async function saveTsvWithPicker(lines, fileName) { + if (typeof window.showSaveFilePicker !== "function") { + return false; + } + const handle = await window.showSaveFilePicker({ + id: "pre-ingest-curated-tsv", + suggestedName: fileName, + types: [ + { + description: "Tab-separated values", + accept: { "text/tab-separated-values": [".tsv"] }, + }, + ], + }); + const writable = await handle.createWritable(); + await writable.write(`${lines.join("\n")}\n`); + await writable.close(); + return true; + } + + function exportValidatorTsv() { + if (!validatorRows.length || !validatorHeaders.length) { + showValidatorFlash("Load a validator TSV first.", "err"); + return; + } + const lines = [ + validatorHeaders.join("\t"), + ...validatorAllRows.map((row) => + validatorHeaders.map((header) => { + if (header === "override") { + return escapeTsvCell(row.override.join(", ")); + } + return escapeTsvCell(row.raw[header] ?? ""); + }).join("\t") + ), + ]; + const baseName = validatorParsedFileName || "validated_pre_ingest.tsv"; + const fileName = baseName.endsWith(".tsv") ? baseName.replace(/\.tsv$/i, "_updated.tsv") : `${baseName}_updated.tsv`; + triggerTsvDownload(lines, fileName); + validatorDirty = false; + syncValidatorDirtyPill(); + showValidatorFlash("Saved updated TSV.", "ok"); + } + + async function exportCuratedValidatorTsv() { + if (!validatorAllRows.length) { + showValidatorFlash("Load a validator TSV first.", "err"); + return; + } + const lines = [ + ["scix_id", "bibcode", "title", "gross_collection"].join("\t"), + ...validatorAllRows.map((row) => { + const grossCollection = row.override.length ? row.override.join(", ") : String(row.raw.gross_collection || "").trim(); + return [ + escapeTsvCell(row.scix_id || ""), + escapeTsvCell(row.bibcode || ""), + escapeTsvCell(row.title || ""), + escapeTsvCell(grossCollection), + ].join("\t"); + }), + ]; + const requestedName = String(el("validator_curated_filename").value || "").trim(); + const fileName = requestedName || curatedFileNameFor(validatorParsedFileName); + try { + const savedWithPicker = await saveTsvWithPicker(lines, fileName); + if (!savedWithPicker) { + triggerTsvDownload(lines, fileName); + showValidatorFlash("Exported curated TSV.", "ok"); + return; + } + showValidatorFlash("Saved curated TSV.", "ok"); + } catch (err) { + if (err && err.name === "AbortError") { + return; + } + triggerTsvDownload(lines, fileName); + showValidatorFlash(`Save dialog unavailable, exported curated TSV instead. ${err.message || ""}`.trim(), "ok"); + } + } + + document.querySelectorAll(".tab-btn").forEach((btn) => { + btn.addEventListener("click", () => { + setActiveTab(btn.dataset.tab); + }); + }); + el("score_col_header").addEventListener("click", () => { if (!rowsData.length) { return; @@ -489,10 +1183,35 @@

Details and Update

el("collection_label").textContent = "No row selected."; el("scores_text").value = ""; el("abstract_text").value = ""; - clearChecks(); + clearChecks("category_checks"); renderRows(); }); + function toggleValidatorScoreSort(sortKey) { + if (!validatorRows.length) { + return; + } + if (validatorScoreSortKey === sortKey) { + validatorScoreSortDir = validatorScoreSortDir === "asc" ? "desc" : "asc"; + } else { + validatorScoreSortKey = sortKey; + validatorScoreSortDir = "desc"; + } + applyValidatorScoreFilter(); + } + + el("validator_astronomy_score_col_header").addEventListener("click", () => { + toggleValidatorScoreSort("astronomy_score"); + }); + + el("validator_physics_score_col_header").addEventListener("click", () => { + toggleValidatorScoreSort("physics_score"); + }); + + el("validator_general_score_col_header").addEventListener("click", () => { + toggleValidatorScoreSort("general_score"); + }); + el("connect_btn").addEventListener("click", async () => { clearFlash(); setStatus("Connecting..."); @@ -510,6 +1229,7 @@

Details and Update

try { await runQueryWithPreset(); } catch (err) { + allRowsData = []; rowsData = []; renderRows(); setStatus("Connected"); @@ -530,6 +1250,7 @@

Details and Update

const preset = parsedIdentifierType === "scix_id" ? SCIX_ID_LIST_PRESET : BIBCODE_LIST_PRESET; await runQueryWithPreset(preset); } catch (err) { + allRowsData = []; rowsData = []; renderRows(); setStatus("Connected"); @@ -538,10 +1259,10 @@

Details and Update

}); el("submit_btn").addEventListener("click", async () => { - const selectedRows = rowsData.filter((row) => selectedRowKeys.has(row.row_key)); + const selectedRows = selectedRowsForUpdate(); const selectedRecords = selectedRows.map((row) => row.record); if (!selectedRecords.length) { - showFlash("Select at least one record first.", "err"); + showFlash("Select a record first.", "err"); return; } if (selectedRecords.length >= LARGE_BATCH_CONFIRM_THRESHOLD) { @@ -555,7 +1276,7 @@

Details and Update

try { const updatedRowKeys = new Set(selectedRows.map((row) => row.row_key)); - const categories = selectedCategories(); + const categories = selectedCategories("category_checks"); const validated = !!el("validated").checked; const data = await api("/api/update", "POST", { ...dbPayload(), @@ -582,7 +1303,7 @@

Details and Update

}); el("clear_btn").addEventListener("click", () => { - clearChecks(); + clearChecks("category_checks"); }); el("select_all_rows").addEventListener("change", (event) => { @@ -598,6 +1319,14 @@

Details and Update

syncIdentifierState(); }); + el("score_min").addEventListener("input", () => { + applyDbScoreFilter(); + }); + + el("score_max").addEventListener("input", () => { + applyDbScoreFilter(); + }); + el("identifier_file").addEventListener("change", async (event) => { const file = event.target.files[0]; if (!file) { @@ -614,9 +1343,49 @@

Details and Update

} }); + el("validator_load_btn").addEventListener("click", () => { + loadValidatorFile(); + }); + + el("validator_export_btn").addEventListener("click", () => { + exportValidatorTsv(); + }); + + el("validator_curated_export_btn").addEventListener("click", () => { + exportCuratedValidatorTsv(); + }); + + el("validator_submit_btn").addEventListener("click", () => { + applyValidatorOverrideUpdate(); + }); + + el("validator_clear_btn").addEventListener("click", () => { + clearChecks("validator_category_checks"); + }); + + el("validator_select_all_rows").addEventListener("change", (event) => { + if (event.target.checked) { + validatorSelectedRowKeys = new Set(validatorRows.map((row) => row.row_key)); + } else { + validatorSelectedRowKeys = new Set(); + } + renderValidatorRows(); + refreshValidatorSelectionDetail(); + }); + + el("validator_score_min").addEventListener("input", () => { + applyValidatorScoreFilter(); + }); + + el("validator_score_max").addEventListener("input", () => { + applyValidatorScoreFilter(); + }); + initSelects(); - buildCategoryChecks(); + buildCategoryChecks("category_checks"); + buildCategoryChecks("validator_category_checks", VALIDATOR_GROSS_CATEGORIES); syncIdentifierState(); + syncValidatorDirtyPill(); diff --git a/inspector/tests.py b/inspector/tests.py new file mode 100644 index 0000000..d1106c6 --- /dev/null +++ b/inspector/tests.py @@ -0,0 +1,259 @@ +import json +from unittest.mock import patch + +from django.test import RequestFactory, SimpleTestCase + +from inspector.views import ADSClient, DatabaseClient, QUERY_SPEC_BY_LABEL, api_query + + +class _FakeCursor: + def __init__(self, rows): + self.rows = rows + self.executed = [] + self.rowcount = 0 + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc, tb): + return False + + def execute(self, sql, params): + self.executed.append((sql, params)) + if sql.lstrip().upper().startswith("UPDATE"): + self.rowcount = 0 + elif sql.lstrip().upper().startswith("INSERT"): + self.rowcount = 1 + + def fetchall(self): + return self.rows + + +class _FakeConnection: + def __init__(self, rows): + self.cursor_instance = _FakeCursor(rows) + + def cursor(self, cursor_factory=None): + return self.cursor_instance + + +class DatabaseClientRunQueryTests(SimpleTestCase): + def test_base_select_prefers_score_id_then_bibcode_then_scix_id_for_final_collection(self): + client = DatabaseClient() + client.metadata_table = None + + sql = client._base_select() + + self.assertIn("OR (bibcode IS NOT NULL AND bibcode = s.bibcode)", sql) + self.assertIn("OR (scix_id IS NOT NULL AND scix_id = s.scix_id)", sql) + self.assertIn("WHEN score_id = s.id THEN 0", sql) + self.assertIn("WHEN bibcode IS NOT NULL AND bibcode = s.bibcode THEN 1", sql) + self.assertIn("WHEN scix_id IS NOT NULL AND scix_id = s.scix_id THEN 2", sql) + + def test_base_select_prefers_scix_id_then_bibcode_for_overrides(self): + client = DatabaseClient() + client.metadata_table = None + + sql = client._base_select() + + self.assertIn("WHERE (scix_id IS NOT NULL AND scix_id = s.scix_id)", sql) + self.assertIn("OR (bibcode IS NOT NULL AND bibcode = s.bibcode)", sql) + self.assertIn("WHEN scix_id IS NOT NULL AND scix_id = s.scix_id THEN 0", sql) + self.assertIn("WHEN bibcode IS NOT NULL AND bibcode = s.bibcode THEN 1", sql) + + def test_bibcode_list_query_uses_text_casts_for_filter_and_order(self): + client = DatabaseClient() + client.conn = _FakeConnection(rows=[]) + client.metadata_table = None + + client.run_query( + spec=QUERY_SPEC_BY_LABEL["By bibcode list"], + run_id="", + bibcode_term="", + scix_id_term="", + bibcode_list=["2024Natur.635..755S", "2024Natur.632..287O"], + scix_id_list=[], + limit=20, + ) + + sql, params = client.conn.cursor_instance.executed[0] + self.assertIn("s.bibcode::text = ANY(%s::text[])", sql) + self.assertIn("array_position(%s::text[], s.bibcode::text)", sql) + self.assertEqual(params[-1], ["2024Natur.635..755S", "2024Natur.632..287O"]) + + def test_scix_id_list_query_uses_text_casts_for_filter_and_order(self): + client = DatabaseClient() + client.conn = _FakeConnection(rows=[]) + client.metadata_table = None + + client.run_query( + spec=QUERY_SPEC_BY_LABEL["By scix_id list"], + run_id="", + bibcode_term="", + scix_id_term="", + bibcode_list=[], + scix_id_list=["scix:abc", "scix:def"], + limit=20, + ) + + sql, params = client.conn.cursor_instance.executed[0] + self.assertIn("s.scix_id::text = ANY(%s::text[])", sql) + self.assertIn("array_position(%s::text[], s.scix_id::text)", sql) + self.assertEqual(params[-1], ["scix:abc", "scix:def"]) + + def test_update_collection_insert_includes_scix_id_for_final_collection(self): + client = DatabaseClient() + client.conn = _FakeConnection(rows=[]) + + client.update_collection( + final_collection_id=None, + score_id=18, + bibcode=None, + scix_id="scix:abc", + collection=["astrophysics"], + validated=False, + commit=False, + ) + + sql_statements = client.conn.cursor_instance.executed + final_collection_insert = next( + params for sql, params in sql_statements if "INSERT INTO final_collection" in sql + ) + self.assertEqual(final_collection_insert, (None, "scix:abc", 18, ["astrophysics"], False)) + + +class ADSClientTests(SimpleTestCase): + def test_fetch_titles_queries_ads_by_identifier(self): + client = ADSClient() + + class _Response: + def raise_for_status(self): + return None + + def json(self): + return { + "response": { + "docs": [ + { + "bibcode": "2022Natur.608..472D", + "identifier": ["scix:abc", "2022Natur.608..472D"], + "title": ["Europe's energy crisis - climate community must speak up"], + } + ] + } + } + + with patch("inspector.views.requests.get", return_value=_Response()) as mock_get: + titles = client.fetch_titles(["scix:abc"], "token") + + self.assertEqual(titles["scix:abc"], "Europe's energy crisis - climate community must speak up") + self.assertEqual(mock_get.call_args.kwargs["params"]["q"], 'identifier:("scix:abc")') + + def test_fetch_abstract_queries_ads_by_identifier(self): + client = ADSClient() + + class _Response: + def raise_for_status(self): + return None + + def json(self): + return {"response": {"docs": [{"abstract": "Example abstract"}]}} + + with patch("inspector.views.requests.get", return_value=_Response()) as mock_get: + abstract = client.fetch_abstract(["scix:abc"], "token") + + self.assertEqual(abstract, "Example abstract") + self.assertEqual(mock_get.call_args.kwargs["params"]["q"], 'identifier:"scix:abc"') + + +class ApiQueryTests(SimpleTestCase): + def setUp(self): + self.factory = RequestFactory() + + def test_bibcode_list_query_reports_missing_bibcodes(self): + request = self.factory.post( + "/api/query", + data=json.dumps( + { + "preset": "By bibcode list", + "bibcode_list": ["2024Natur.635..755S", "2024Natur.632..287O"], + "score_category": "astrophysics", + "limit": 20, + } + ), + content_type="application/json", + ) + + returned_rows = [ + { + "score_id": 1, + "final_collection_id": 2, + "scix_id": "scix:1", + "bibcode": "2024Natur.635..755S", + "scores": '{"scores": {"astrophysics": 0.91}}', + "title": "Example title", + "run_id": 7, + "validated": False, + "collection": [], + } + ] + + fake_client = _ApiQueryFakeClient(returned_rows) + with patch("inspector.views.open_db", return_value=fake_client): + response = api_query(request) + + payload = json.loads(response.content) + self.assertTrue(payload["ok"]) + self.assertEqual(payload["missing_bibcodes"], ["2024Natur.632..287O"]) + self.assertEqual([row["bibcode"] for row in payload["rows"]], ["2024Natur.635..755S"]) + self.assertTrue(fake_client.closed) + + def test_scix_id_list_query_reports_missing_scix_ids(self): + request = self.factory.post( + "/api/query", + data=json.dumps( + { + "preset": "By scix_id list", + "scix_id_list": ["scix:1", "scix:2"], + "score_category": "astrophysics", + "limit": 20, + } + ), + content_type="application/json", + ) + + returned_rows = [ + { + "score_id": 1, + "final_collection_id": 2, + "scix_id": "scix:1", + "bibcode": "2024Natur.635..755S", + "scores": '{"scores": {"astrophysics": 0.91}}', + "title": "Example title", + "run_id": 7, + "validated": False, + "collection": [], + } + ] + + fake_client = _ApiQueryFakeClient(returned_rows) + with patch("inspector.views.open_db", return_value=fake_client): + response = api_query(request) + + payload = json.loads(response.content) + self.assertTrue(payload["ok"]) + self.assertEqual(payload["missing_scix_ids"], ["scix:2"]) + self.assertEqual([row["scix_id"] for row in payload["rows"]], ["scix:1"]) + self.assertTrue(fake_client.closed) + + +class _ApiQueryFakeClient: + def __init__(self, rows): + self.rows = rows + self.closed = False + + def run_query(self, **kwargs): + return self.rows + + def close(self): + self.closed = True diff --git a/inspector/views.py b/inspector/views.py index 30d13d7..26560ed 100644 --- a/inspector/views.py +++ b/inspector/views.py @@ -136,17 +136,30 @@ def _base_select(self): SELECT id, collection, validated FROM final_collection WHERE score_id = s.id - OR (score_id IS NULL AND bibcode = s.bibcode) + OR (bibcode IS NOT NULL AND bibcode = s.bibcode) + OR (scix_id IS NOT NULL AND scix_id = s.scix_id) ORDER BY - CASE WHEN score_id = s.id THEN 0 ELSE 1 END, + CASE + WHEN score_id = s.id THEN 0 + WHEN bibcode IS NOT NULL AND bibcode = s.bibcode THEN 1 + WHEN scix_id IS NOT NULL AND scix_id = s.scix_id THEN 2 + ELSE 3 + END, created DESC LIMIT 1 ) fc ON TRUE LEFT JOIN LATERAL ( SELECT override FROM overrides - WHERE bibcode = s.bibcode - ORDER BY created DESC + WHERE (scix_id IS NOT NULL AND scix_id = s.scix_id) + OR (bibcode IS NOT NULL AND bibcode = s.bibcode) + ORDER BY + CASE + WHEN scix_id IS NOT NULL AND scix_id = s.scix_id THEN 0 + WHEN bibcode IS NOT NULL AND bibcode = s.bibcode THEN 1 + ELSE 2 + END, + created DESC LIMIT 1 ) ov ON TRUE {metadata_join} @@ -186,13 +199,13 @@ def run_query( if spec.needs_bibcode_list: if not bibcode_list: raise ValueError("A bibcode list is required for this query.") - where_clauses.append("s.bibcode = ANY(%s)") + where_clauses.append("s.bibcode::text = ANY(%s::text[])") params.append(bibcode_list) if spec.needs_scix_id_list: if not scix_id_list: raise ValueError("A scix_id list is required for this query.") - where_clauses.append("s.scix_id = ANY(%s)") + where_clauses.append("s.scix_id::text = ANY(%s::text[])") params.append(scix_id_list) if spec.label == "Unvalidated records": @@ -205,10 +218,10 @@ def run_query( where_sql = " WHERE " + " AND ".join(where_clauses) if spec.needs_bibcode_list: - sql = self._base_select() + where_sql + " ORDER BY array_position(%s::text[], s.bibcode)" + sql = self._base_select() + where_sql + " ORDER BY array_position(%s::text[], s.bibcode::text)" params.append(bibcode_list) elif spec.needs_scix_id_list: - sql = self._base_select() + where_sql + " ORDER BY array_position(%s::text[], s.scix_id)" + sql = self._base_select() + where_sql + " ORDER BY array_position(%s::text[], s.scix_id::text)" params.append(scix_id_list) else: sql = self._base_select() + where_sql + " ORDER BY s.id DESC LIMIT %s" @@ -259,10 +272,10 @@ def update_collection( if updated == 0: cur.execute( """ - INSERT INTO final_collection (bibcode, score_id, collection, validated) - VALUES (%s, %s, %s, %s) + INSERT INTO final_collection (bibcode, scix_id, score_id, collection, validated) + VALUES (%s, %s, %s, %s, %s) """, - (bibcode, score_id, collection, validated), + (bibcode, scix_id, score_id, collection, validated), ) override_updated = 0 @@ -318,42 +331,56 @@ def _chunk(items, size): for idx in range(0, len(items), size): yield items[idx : idx + size] - def fetch_titles(self, bibcodes, token): + def fetch_titles(self, identifiers, token): if not token: return {} - unique_bibcodes = [b for b in dict.fromkeys(bibcodes) if b] - if not unique_bibcodes: + unique_identifiers = [identifier for identifier in dict.fromkeys(identifiers) if identifier] + if not unique_identifiers: return {} - titles_by_bibcode = {} + titles_by_identifier = {} headers = {"Authorization": f"Bearer {token.strip()}"} - for chunk in self._chunk(unique_bibcodes, 100): - query = " OR ".join(f'"{bibcode}"' for bibcode in chunk) - params = {"q": f"bibcode:({query})", "fl": "bibcode,title", "rows": len(chunk)} + for chunk in self._chunk(unique_identifiers, 100): + query = " OR ".join(f'"{identifier}"' for identifier in chunk) + params = {"q": f"identifier:({query})", "fl": "identifier,bibcode,title", "rows": len(chunk)} response = requests.get(self.base_url, headers=headers, params=params, timeout=20) response.raise_for_status() docs = response.json().get("response", {}).get("docs", []) for doc in docs: - bibcode = doc.get("bibcode") title = doc.get("title") if isinstance(title, list): title = title[0] if title else "" - if bibcode and title: - titles_by_bibcode[bibcode] = title + if not title: + continue + + doc_identifiers = doc.get("identifier") or [] + if isinstance(doc_identifiers, str): + doc_identifiers = [doc_identifiers] + if doc.get("bibcode"): + doc_identifiers = [*doc_identifiers, doc["bibcode"]] - return titles_by_bibcode + for identifier in chunk: + if identifier in doc_identifiers: + titles_by_identifier[identifier] = title - def fetch_abstract(self, bibcode, token): - if not token or not bibcode: + return titles_by_identifier + + def fetch_abstract(self, identifiers, token): + if not token or not identifiers: return "" headers = {"Authorization": f"Bearer {token.strip()}"} - params = {"q": f'bibcode:"{bibcode}"', "fl": "bibcode,abstract", "rows": 1} - response = requests.get(self.base_url, headers=headers, params=params, timeout=20) - response.raise_for_status() - docs = response.json().get("response", {}).get("docs", []) - return docs[0].get("abstract") if docs else "" + for identifier in identifiers: + if not identifier: + continue + params = {"q": f'identifier:"{identifier}"', "fl": "identifier,bibcode,abstract", "rows": 1} + response = requests.get(self.base_url, headers=headers, params=params, timeout=20) + response.raise_for_status() + docs = response.json().get("response", {}).get("docs", []) + if docs and docs[0].get("abstract"): + return docs[0]["abstract"] + return "" def summarize_exception(exc: Exception) -> str: @@ -456,6 +483,15 @@ def build_row_key(row, idx): ) +def ads_identifiers_for_row(row): + identifiers = [] + for value in (row.get("scix_id"), row.get("bibcode")): + text = str(value or "").strip() + if text and text not in identifiers: + identifiers.append(text) + return identifiers + + def open_db(payload) -> DatabaseClient: client = DatabaseClient() client.connect( @@ -558,14 +594,16 @@ def api_query(request): ) warning = None - bibcodes = [row.get("bibcode") for row in rows if row.get("bibcode")] - if bibcodes and ads_token: + row_identifiers = {idx: ads_identifiers_for_row(row) for idx, row in enumerate(rows)} + ads_identifiers = [identifier for identifiers in row_identifiers.values() for identifier in identifiers] + if ads_identifiers and ads_token: try: - titles = ads.fetch_titles(bibcodes, ads_token) - for row in rows: - bib = row.get("bibcode") - if bib in titles: - row["title"] = titles[bib] + titles = ads.fetch_titles(ads_identifiers, ads_token) + for idx, row in enumerate(rows): + for identifier in row_identifiers[idx]: + if identifier in titles: + row["title"] = titles[identifier] + break except Exception as exc: warning = summarize_exception(exc) @@ -653,9 +691,9 @@ def api_record(request): ) abstract = row.get("abstract") or "" - if not abstract and ads_token and row.get("bibcode"): + if not abstract and ads_token: try: - abstract = ADSClient().fetch_abstract(bibcode=row.get("bibcode"), token=ads_token) + abstract = ADSClient().fetch_abstract(identifiers=ads_identifiers_for_row(row), token=ads_token) except Exception as exc: abstract = f"(ADS abstract lookup failed: {summarize_exception(exc)})" if not abstract: @@ -680,7 +718,10 @@ def api_update(request): return JsonResponse({"ok": False, "error": "Method not allowed"}, status=405) payload = parse_json(request) - records = payload.get("records") or [] + records = payload.get("records") + if records is None: + single_record = payload.get("record") or {} + records = [single_record] if single_record else [] selected_categories = payload.get("selected_categories") or [] if not isinstance(records, list):