From 462afae4ec53a16f92719abc57e94a04171b7f8d Mon Sep 17 00:00:00 2001 From: Henry Estela Date: Fri, 17 Apr 2026 18:34:31 +0000 Subject: [PATCH 001/108] fix(AI): add null check to model name (#645) When the OpenAI-compatible fallback (/v1/models) is used, models are mapped as { name: m.id, size: 0 } with no details field. Accessing model.details.parameter_size throws `TypeError: Cannot read properties of undefined`, which crashes the React render and causes the entire page to go blank. --- admin/inertia/pages/settings/models.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/admin/inertia/pages/settings/models.tsx b/admin/inertia/pages/settings/models.tsx index d4124fe9..fc2b1dc1 100644 --- a/admin/inertia/pages/settings/models.tsx +++ b/admin/inertia/pages/settings/models.tsx @@ -369,7 +369,7 @@ export default function ModelsPage(props: { - {model.details.parameter_size || 'N/A'} + {model.details?.parameter_size || 'N/A'} From 10ba8000cf348a6a25f359392de44dbfcb473d85 Mon Sep 17 00:00:00 2001 From: Henry Estela Date: Fri, 17 Apr 2026 18:37:44 +0000 Subject: [PATCH 002/108] fix(AI): qwen2.5 loading on every chat message (#649) Use the currently loaded model for chat title generation and query rewrite. --- admin/app/controllers/ollama_controller.ts | 33 +++++++++++----------- admin/app/services/chat_service.ts | 33 +++++++++------------- admin/app/services/rag_service.ts | 10 +++++++ 3 files changed, 39 insertions(+), 37 deletions(-) diff --git a/admin/app/controllers/ollama_controller.ts b/admin/app/controllers/ollama_controller.ts index e58d2497..edd6f362 100644 --- a/admin/app/controllers/ollama_controller.ts +++ b/admin/app/controllers/ollama_controller.ts @@ -8,7 +8,7 @@ import { modelNameSchema } from '#validators/download' import { chatSchema, getAvailableModelsSchema } from '#validators/ollama' import { inject } from '@adonisjs/core' import type { HttpContext } from '@adonisjs/core/http' -import { DEFAULT_QUERY_REWRITE_MODEL, RAG_CONTEXT_LIMITS, SYSTEM_PROMPTS } from '../../constants/ollama.js' +import { RAG_CONTEXT_LIMITS, SYSTEM_PROMPTS } from '../../constants/ollama.js' import { SERVICE_NAMES } from '../../constants/service_names.js' import logger from '@adonisjs/core/services/logger' type Message = { role: 'system' | 'user' | 'assistant'; content: string } @@ -59,7 +59,7 @@ export default class OllamaController { // Query rewriting for better RAG retrieval with manageable context // Will return user's latest message if no rewriting is needed - const rewrittenQuery = await this.rewriteQueryWithContext(reqData.messages) + const rewrittenQuery = await this.rewriteQueryWithContext(reqData.messages, reqData.model) logger.debug(`[OllamaController] Rewritten query for RAG: "${rewrittenQuery}"`) if (rewrittenQuery) { @@ -157,7 +157,7 @@ export default class OllamaController { await this.chatService.addMessage(sessionId, 'assistant', fullContent) const messageCount = await this.chatService.getMessageCount(sessionId) if (messageCount <= 2 && userContent) { - this.chatService.generateTitle(sessionId, userContent, fullContent).catch((err) => { + this.chatService.generateTitle(sessionId, userContent, fullContent, reqData.model).catch((err) => { logger.error(`[OllamaController] Title generation failed: ${err instanceof Error ? err.message : err}`) }) } @@ -172,7 +172,7 @@ export default class OllamaController { await this.chatService.addMessage(sessionId, 'assistant', result.message.content) const messageCount = await this.chatService.getMessageCount(sessionId) if (messageCount <= 2 && userContent) { - this.chatService.generateTitle(sessionId, userContent, result.message.content).catch((err) => { + this.chatService.generateTitle(sessionId, userContent, result.message.content, reqData.model).catch((err) => { logger.error(`[OllamaController] Title generation failed: ${err instanceof Error ? err.message : err}`) }) } @@ -312,9 +312,18 @@ export default class OllamaController { } private async rewriteQueryWithContext( - messages: Message[] + messages: Message[], + model: string ): Promise { + const lastUserMessage = [...messages].reverse().find(msg => msg.role === 'user') + try { + // Skip the entire RAG pipeline if there are no documents to search + const hasDocuments = await this.ragService.hasDocuments() + if (!hasDocuments) { + return null + } + // Get recent conversation history (last 6 messages for 3 turns) const recentMessages = messages.slice(-6) @@ -322,7 +331,7 @@ export default class OllamaController { // little RAG benefit until there is enough context to matter. const userMessages = recentMessages.filter(msg => msg.role === 'user') if (userMessages.length <= 2) { - return userMessages[userMessages.length - 1]?.content || null + return lastUserMessage?.content || null } const conversationContext = recentMessages @@ -336,17 +345,8 @@ export default class OllamaController { }) .join('\n') - const installedModels = await this.ollamaService.getModels(true) - const rewriteModelAvailable = installedModels?.some(model => model.name === DEFAULT_QUERY_REWRITE_MODEL) - if (!rewriteModelAvailable) { - logger.warn(`[RAG] Query rewrite model "${DEFAULT_QUERY_REWRITE_MODEL}" not available. Skipping query rewriting.`) - const lastUserMessage = [...messages].reverse().find(msg => msg.role === 'user') - return lastUserMessage?.content || null - } - - // FUTURE ENHANCEMENT: allow the user to specify which model to use for rewriting const response = await this.ollamaService.chat({ - model: DEFAULT_QUERY_REWRITE_MODEL, + model, messages: [ { role: 'system', @@ -367,7 +367,6 @@ export default class OllamaController { `[RAG] Query rewriting failed: ${error instanceof Error ? error.message : error}` ) // Fallback to last user message if rewriting fails - const lastUserMessage = [...messages].reverse().find(msg => msg.role === 'user') return lastUserMessage?.content || null } } diff --git a/admin/app/services/chat_service.ts b/admin/app/services/chat_service.ts index 18b01081..2d97ad7a 100644 --- a/admin/app/services/chat_service.ts +++ b/admin/app/services/chat_service.ts @@ -4,7 +4,7 @@ import logger from '@adonisjs/core/services/logger' import { DateTime } from 'luxon' import { inject } from '@adonisjs/core' import { OllamaService } from './ollama_service.js' -import { DEFAULT_QUERY_REWRITE_MODEL, SYSTEM_PROMPTS } from '../../constants/ollama.js' +import { SYSTEM_PROMPTS } from '../../constants/ollama.js' import { toTitleCase } from '../utils/misc.js' @inject() @@ -232,29 +232,22 @@ export class ChatService { } } - async generateTitle(sessionId: number, userMessage: string, assistantMessage: string) { + async generateTitle(sessionId: number, userMessage: string, assistantMessage: string, model: string) { try { - const models = await this.ollamaService.getModels() - const titleModelAvailable = models?.some((m) => m.name === DEFAULT_QUERY_REWRITE_MODEL) - let title: string - if (!titleModelAvailable) { - title = userMessage.slice(0, 57) + (userMessage.length > 57 ? '...' : '') - } else { - const response = await this.ollamaService.chat({ - model: DEFAULT_QUERY_REWRITE_MODEL, - messages: [ - { role: 'system', content: SYSTEM_PROMPTS.title_generation }, - { role: 'user', content: userMessage }, - { role: 'assistant', content: assistantMessage }, - ], - }) + const response = await this.ollamaService.chat({ + model, + messages: [ + { role: 'system', content: SYSTEM_PROMPTS.title_generation }, + { role: 'user', content: userMessage }, + { role: 'assistant', content: assistantMessage }, + ], + }) - title = response?.message?.content?.trim() - if (!title) { - title = userMessage.slice(0, 57) + (userMessage.length > 57 ? '...' : '') - } + title = response?.message?.content?.trim() + if (!title) { + title = userMessage.slice(0, 57) + (userMessage.length > 57 ? '...' : '') } await this.updateSession(sessionId, { title }) diff --git a/admin/app/services/rag_service.ts b/admin/app/services/rag_service.ts index 0e9550e0..67e8627a 100644 --- a/admin/app/services/rag_service.ts +++ b/admin/app/services/rag_service.ts @@ -1013,6 +1013,16 @@ export class RagService { * Retrieve all unique source files that have been stored in the knowledge base. * @returns Array of unique full source paths */ + public async hasDocuments(): Promise { + try { + await this._ensureCollection(RagService.CONTENT_COLLECTION_NAME, RagService.EMBEDDING_DIMENSION) + const collectionInfo = await this.qdrant!.getCollection(RagService.CONTENT_COLLECTION_NAME) + return (collectionInfo.points_count ?? 0) > 0 + } catch { + return false + } + } + public async getStoredFiles(): Promise { try { await this._ensureCollection( From 10e8957b78b874450952e9ed727672c8df6f6c09 Mon Sep 17 00:00:00 2001 From: Jake Turner <52841588+jakeaturner@users.noreply.github.com> Date: Fri, 17 Apr 2026 11:54:04 -0700 Subject: [PATCH 003/108] fix: prevent ZIM corrupt file crash and deduplicate Ollama download logs (#741) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Corrupted ZIM files cause a native C++ abort (ZimFileFormatError) that bypasses JS try/catch and kills the process. Add magic number validation before passing files to @openzim/libzim so invalid files are skipped gracefully. Also deduplicate Ollama download progress broadcasts — both within a single stream (skip unchanged percentages) and across concurrent callers (share one download promise per model). Co-authored-by: aegisman Co-authored-by: Claude Opus 4.6 (1M context) --- admin/app/services/kiwix_library_service.ts | 30 ++++++++++++++------ admin/app/services/ollama_service.ts | 29 +++++++++++++++++-- admin/app/services/zim_extraction_service.ts | 9 +++++- admin/app/utils/fs.ts | 24 +++++++++++++++- 4 files changed, 80 insertions(+), 12 deletions(-) diff --git a/admin/app/services/kiwix_library_service.ts b/admin/app/services/kiwix_library_service.ts index 28e65766..c7aeabd4 100644 --- a/admin/app/services/kiwix_library_service.ts +++ b/admin/app/services/kiwix_library_service.ts @@ -2,7 +2,7 @@ import { XMLBuilder, XMLParser } from 'fast-xml-parser' import { readFile, writeFile, rename, readdir } from 'fs/promises' import { join } from 'path' import { Archive } from '@openzim/libzim' -import { KIWIX_LIBRARY_XML_PATH, ZIM_STORAGE_PATH, ensureDirectoryExists } from '../utils/fs.js' +import { KIWIX_LIBRARY_XML_PATH, ZIM_STORAGE_PATH, ensureDirectoryExists, isValidZimFile } from '../utils/fs.js' import logger from '@adonisjs/core/services/logger' import { randomUUID } from 'node:crypto' @@ -54,8 +54,12 @@ export class KiwixLibraryService { * * Returns null on any error so callers can fall back gracefully. */ - private _readZimMetadata(zimFilePath: string): Partial | null { + private async _readZimMetadata(zimFilePath: string): Promise | null> { try { + if (!(await isValidZimFile(zimFilePath))) { + logger.warn(`[KiwixLibraryService] Skipping invalid/corrupted ZIM file: ${zimFilePath}`) + return null + } const archive = new Archive(zimFilePath) const getMeta = (key: string): string | undefined => { @@ -197,17 +201,22 @@ export class KiwixLibraryService { const excludeSet = new Set(opts?.excludeFilenames ?? []) const zimFiles = entries.filter((name) => name.endsWith('.zim') && !excludeSet.has(name)) - const books: KiwixBook[] = zimFiles.map((filename) => { - const meta = this._readZimMetadata(join(dirPath, filename)) + const books: KiwixBook[] = [] + for (const filename of zimFiles) { + const meta = await this._readZimMetadata(join(dirPath, filename)) + if (meta === null) { + logger.warn(`[KiwixLibraryService] Skipping unreadable ZIM file: ${filename}`) + continue + } const containerPath = `${CONTAINER_DATA_PATH}/${filename}` - return { + books.push({ ...meta, // Override fields that must be derived locally, not from ZIM metadata id: meta?.id ?? filename.slice(0, -4), path: containerPath, title: meta?.title ?? this._filenameToTitle(filename), - } - }) + }) + } const xml = this._buildXml(books) await this._atomicWrite(xml) @@ -239,7 +248,12 @@ export class KiwixLibraryService { } const fullPath = join(process.cwd(), ZIM_STORAGE_PATH, zimFilename) - const meta = this._readZimMetadata(fullPath) + const meta = await this._readZimMetadata(fullPath) + + if (meta === null) { + logger.error(`[KiwixLibraryService] Cannot add ${zimFilename}: file is invalid or corrupted.`) + return + } existingBooks.push({ ...meta, diff --git a/admin/app/services/ollama_service.ts b/admin/app/services/ollama_service.ts index dacf1312..5e38cb63 100644 --- a/admin/app/services/ollama_service.ts +++ b/admin/app/services/ollama_service.ts @@ -53,6 +53,7 @@ export class OllamaService { private baseUrl: string | null = null private initPromise: Promise | null = null private isOllamaNative: boolean | null = null + private activeDownloads: Map> = new Map() constructor() {} @@ -95,6 +96,26 @@ export class OllamaService { async downloadModel( model: string, progressCallback?: (percent: number) => void + ): Promise<{ success: boolean; message: string; retryable?: boolean }> { + // Deduplicate concurrent downloads of the same model + const existing = this.activeDownloads.get(model) + if (existing) { + logger.info(`[OllamaService] Download already in progress for "${model}", waiting on existing download.`) + return existing + } + + const downloadPromise = this._doDownloadModel(model, progressCallback) + this.activeDownloads.set(model, downloadPromise) + try { + return await downloadPromise + } finally { + this.activeDownloads.delete(model) + } + } + + private async _doDownloadModel( + model: string, + progressCallback?: (percent: number) => void ): Promise<{ success: boolean; message: string; retryable?: boolean }> { await this._ensureDependencies() if (!this.baseUrl) { @@ -130,6 +151,7 @@ export class OllamaService { await new Promise((resolve, reject) => { let buffer = '' + let lastPercent = -1 pullResponse.data.on('data', (chunk: Buffer) => { buffer += chunk.toString() const lines = buffer.split('\n') @@ -140,8 +162,11 @@ export class OllamaService { const parsed = JSON.parse(line) if (parsed.completed && parsed.total) { const percent = parseFloat(((parsed.completed / parsed.total) * 100).toFixed(2)) - this.broadcastDownloadProgress(model, percent) - if (progressCallback) progressCallback(percent) + if (percent !== lastPercent) { + lastPercent = percent + this.broadcastDownloadProgress(model, percent) + if (progressCallback) progressCallback(percent) + } } } catch { // ignore parse errors on partial lines diff --git a/admin/app/services/zim_extraction_service.ts b/admin/app/services/zim_extraction_service.ts index e60042d8..b3594b64 100644 --- a/admin/app/services/zim_extraction_service.ts +++ b/admin/app/services/zim_extraction_service.ts @@ -5,6 +5,7 @@ import logger from '@adonisjs/core/services/logger' import { ExtractZIMChunkingStrategy, ExtractZIMContentOptions, ZIMContentChunk, ZIMArchiveMetadata } from '../../types/zim.js' import { randomUUID } from 'node:crypto' import { access } from 'node:fs/promises' +import { isValidZimFile } from '../utils/fs.js' export class ZIMExtractionService { @@ -51,7 +52,13 @@ export class ZIMExtractionService { logger.error(`[ZIMExtractionService]: ZIM file not accessible: ${filePath}`) throw new Error(`ZIM file not found or not accessible: ${filePath}`) } - + + // Validate ZIM magic number before opening with native library. + // A corrupted file causes a native C++ abort that cannot be caught by JS. + if (!(await isValidZimFile(filePath))) { + throw new Error(`ZIM file is invalid or corrupted: ${filePath}`) + } + const archive = new Archive(filePath) // Extract archive-level metadata once diff --git a/admin/app/utils/fs.ts b/admin/app/utils/fs.ts index c3ee398c..d6cb5e0c 100644 --- a/admin/app/utils/fs.ts +++ b/admin/app/utils/fs.ts @@ -1,4 +1,4 @@ -import { mkdir, readdir, readFile, stat, unlink } from 'fs/promises' +import { mkdir, open, readdir, readFile, stat, unlink } from 'fs/promises' import path, { join } from 'path' import { FileEntry } from '../../types/files.js' import { createReadStream } from 'fs' @@ -99,6 +99,28 @@ export async function getFileStatsIfExists( } } +/** + * Validates that a file has the ZIM magic number (0x44D495A). + * Must be called before passing a file to @openzim/libzim Archive, + * because a corrupted ZIM causes a native C++ abort that cannot be + * caught by JS try/catch. + */ +export async function isValidZimFile(filePath: string): Promise { + let fh + try { + fh = await open(filePath, 'r') + const buf = Buffer.alloc(4) + const { bytesRead } = await fh.read(buf, 0, 4, 0) + if (bytesRead < 4) return false + // ZIM magic number: 72 17 32 04 (little-endian 0x044D4953) + return buf[0] === 0x5a && buf[1] === 0x49 && buf[2] === 0x4d && buf[3] === 0x04 + } catch { + return false + } finally { + await fh?.close() + } +} + export async function deleteFileIfExists(path: string): Promise { try { await unlink(path) From b365130e767e3d2337e171509527b0275f38de26 Mon Sep 17 00:00:00 2001 From: Ben Gauger Date: Thu, 9 Apr 2026 14:17:54 -0600 Subject: [PATCH 004/108] fix(disk-collector): fix storage reporting for NFS mounts Co-Authored-By: Ben Smith --- admin/inertia/hooks/useDiskDisplayData.ts | 47 +++++++++++++++---- .../collect-disk-info.sh | 6 ++- 2 files changed, 42 insertions(+), 11 deletions(-) diff --git a/admin/inertia/hooks/useDiskDisplayData.ts b/admin/inertia/hooks/useDiskDisplayData.ts index 1d8b15d5..710ed620 100644 --- a/admin/inertia/hooks/useDiskDisplayData.ts +++ b/admin/inertia/hooks/useDiskDisplayData.ts @@ -19,16 +19,36 @@ export function getAllDiskDisplayItems( ): DiskDisplayItem[] { const validDisks = disks?.filter((d) => d.totalSize > 0) || [] + // If /app/storage is on a dedicated filesystem (e.g. NFS), it won't appear + // in the block-device list. Prepend it so NAS and OS disk are both shown. + const storageMount = fsSize?.find((fs) => fs.mount === '/app/storage' && fs.size > 0) + const storageMountItem: DiskDisplayItem[] = storageMount + ? [ + { + label: 'NAS Storage', + value: storageMount.use || 0, + total: formatBytes(storageMount.size), + used: formatBytes(storageMount.used), + subtext: `${formatBytes(storageMount.used)} / ${formatBytes(storageMount.size)}`, + totalBytes: storageMount.size, + usedBytes: storageMount.used, + }, + ] + : [] + if (validDisks.length > 0) { - return validDisks.map((disk) => ({ - label: disk.name || 'Unknown', - value: disk.percentUsed || 0, - total: formatBytes(disk.totalSize), - used: formatBytes(disk.totalUsed), - subtext: `${formatBytes(disk.totalUsed || 0)} / ${formatBytes(disk.totalSize || 0)}`, - totalBytes: disk.totalSize, - usedBytes: disk.totalUsed, - })) + return [ + ...storageMountItem, + ...validDisks.map((disk) => ({ + label: disk.name || 'Unknown', + value: disk.percentUsed || 0, + total: formatBytes(disk.totalSize), + used: formatBytes(disk.totalUsed), + subtext: `${formatBytes(disk.totalUsed || 0)} / ${formatBytes(disk.totalSize || 0)}`, + totalBytes: disk.totalSize, + usedBytes: disk.totalUsed, + })), + ] } if (fsSize && fsSize.length > 0) { @@ -59,6 +79,15 @@ export function getPrimaryDiskInfo( disks: NomadDiskInfo[] | undefined, fsSize: Systeminformation.FsSizeData[] | undefined ): { totalSize: number; totalUsed: number } | null { + // First, check if /app/storage is on a dedicated filesystem (e.g. NFS mount). + // This is the most accurate source since it reflects the actual backing + // store for NOMAD content, regardless of whether it's a local disk or + // network-attached storage. + const storageMount = fsSize?.find((fs) => fs.mount === '/app/storage' && fs.size > 0) + if (storageMount) { + return { totalSize: storageMount.size, totalUsed: storageMount.used } + } + const validDisks = disks?.filter((d) => d.totalSize > 0) || [] if (validDisks.length > 0) { const diskWithRoot = validDisks.find((d) => diff --git a/install/sidecar-disk-collector/collect-disk-info.sh b/install/sidecar-disk-collector/collect-disk-info.sh index 77051757..2a4a5c65 100755 --- a/install/sidecar-disk-collector/collect-disk-info.sh +++ b/install/sidecar-disk-collector/collect-disk-info.sh @@ -44,7 +44,9 @@ while true; do # These are not real filesystem roots and report misleading sizes [[ -f "/host${mountpoint}" ]] && continue - STATS=$(df -B1 "/host${mountpoint}" 2>/dev/null | awk 'NR==2{print $2,$3,$4,$5}') + # Use -P (POSIX) to force single-line output even when device names + # are long (e.g. NFS mounts), which otherwise wrap across two lines + STATS=$(df -P -B1 "/host${mountpoint}" 2>/dev/null | awk 'NR==2{print $2,$3,$4,$5}') [[ -z "$STATS" ]] && continue read -r size used avail pct <<< "$STATS" @@ -60,7 +62,7 @@ while true; do # The disk-collector container always has /storage bind-mounted from the host, # so df on /storage reflects the actual backing device and its capacity. if [[ "$FIRST" -eq 1 ]] && mountpoint -q /storage 2>/dev/null; then - STATS=$(df -B1 /storage 2>/dev/null | awk 'NR==2{print $1,$2,$3,$4,$5}') + STATS=$(df -P -B1 /storage 2>/dev/null | awk 'NR==2{print $1,$2,$3,$4,$5}') if [[ -n "$STATS" ]]; then read -r dev size used avail pct <<< "$STATS" pct="${pct/\%/}" From 898c4441b90675e9697a613592d000b917dbcec6 Mon Sep 17 00:00:00 2001 From: Ben Gauger Date: Thu, 9 Apr 2026 14:32:20 -0600 Subject: [PATCH 005/108] fix(disk-display): show NAS Storage label in fsSize fallback path Co-Authored-By: Ben Smith --- admin/inertia/hooks/useDiskDisplayData.ts | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/admin/inertia/hooks/useDiskDisplayData.ts b/admin/inertia/hooks/useDiskDisplayData.ts index 710ed620..331b323b 100644 --- a/admin/inertia/hooks/useDiskDisplayData.ts +++ b/admin/inertia/hooks/useDiskDisplayData.ts @@ -55,20 +55,24 @@ export function getAllDiskDisplayItems( const seen = new Set() const uniqueFs = fsSize.filter((fs) => { if (fs.size <= 0 || seen.has(fs.size)) return false + if (storageMount && fs.mount === '/app/storage') return false seen.add(fs.size) return true }) const realDevices = uniqueFs.filter((fs) => fs.fs.startsWith('/dev/')) const displayFs = realDevices.length > 0 ? realDevices : uniqueFs - return displayFs.map((fs) => ({ - label: fs.fs || 'Unknown', - value: fs.use || 0, - total: formatBytes(fs.size), - used: formatBytes(fs.used), - subtext: `${formatBytes(fs.used)} / ${formatBytes(fs.size)}`, - totalBytes: fs.size, - usedBytes: fs.used, - })) + return [ + ...storageMountItem, + ...displayFs.map((fs) => ({ + label: fs.fs || 'Unknown', + value: fs.use || 0, + total: formatBytes(fs.size), + used: formatBytes(fs.used), + subtext: `${formatBytes(fs.used)} / ${formatBytes(fs.size)}`, + totalBytes: fs.size, + usedBytes: fs.used, + })), + ] } return [] From b5d4804d5739f21b9f1d1bb68488cd7a2b769cba Mon Sep 17 00:00:00 2001 From: Aaron Bird Date: Sat, 21 Mar 2026 15:10:15 -0400 Subject: [PATCH 006/108] fix(downloads): stage downloads to .tmp to prevent Kiwix loading partial files Downloads are now written to `filepath + '.tmp'` and atomically renamed to the final path only on successful completion. Kiwix globs for `*.zim` and ZimService filters `.endsWith('.zim')`, so `.tmp` files are invisible to both during download. The same staging applies to `.pmtiles` map files. Ref #372 Co-Authored-By: Claude Sonnet 4.6 --- admin/app/utils/downloads.ts | 35 ++++++++++++++++++++++++++++------- 1 file changed, 28 insertions(+), 7 deletions(-) diff --git a/admin/app/utils/downloads.ts b/admin/app/utils/downloads.ts index 1c26a74c..b439be84 100644 --- a/admin/app/utils/downloads.ts +++ b/admin/app/utils/downloads.ts @@ -6,6 +6,7 @@ import axios from 'axios' import { Transform } from 'stream' import { deleteFileIfExists, ensureDirectoryExists, getFileStatsIfExists } from './fs.js' import { createWriteStream } from 'fs' +import { rename } from 'fs/promises' import path from 'path' /** @@ -27,13 +28,16 @@ export async function doResumableDownload({ const dirname = path.dirname(filepath) await ensureDirectoryExists(dirname) - // Check if partial file exists for resume + // Stage download to a .tmp file so consumers (e.g. Kiwix) never see a partial file + const tempPath = filepath + '.tmp' + + // Check if partial .tmp file exists for resume let startByte = 0 let appendMode = false - const existingStats = await getFileStatsIfExists(filepath) + const existingStats = await getFileStatsIfExists(tempPath) if (existingStats && !forceNew) { - startByte = existingStats.size + startByte = Number(existingStats.size) appendMode = true } @@ -55,14 +59,24 @@ export async function doResumableDownload({ } } - // If file is already complete and not forcing overwrite just return filepath + // If final file already exists at correct size, return early (idempotent) + const finalFileStats = await getFileStatsIfExists(filepath) + if (finalFileStats && Number(finalFileStats.size) === totalBytes && totalBytes > 0 && !forceNew) { + return filepath + } + + // If .tmp file is already at correct size (complete but never renamed), just rename it if (startByte === totalBytes && totalBytes > 0 && !forceNew) { + await rename(tempPath, filepath) + if (onComplete) { + await onComplete(url, filepath) + } return filepath } - // If server doesn't support range requests and we have a partial file, delete it + // If server doesn't support range requests and we have a partial .tmp file, delete it if (!supportsRangeRequests && startByte > 0) { - await deleteFileIfExists(filepath) + await deleteFileIfExists(tempPath) startByte = 0 appendMode = false } @@ -131,7 +145,7 @@ export async function doResumableDownload({ }, }) - const writeStream = createWriteStream(filepath, { + const writeStream = createWriteStream(tempPath, { flags: appendMode ? 'a' : 'w', }) @@ -157,6 +171,13 @@ export async function doResumableDownload({ writeStream.on('finish', async () => { clearStallTimer() + try { + // Atomically move the completed .tmp file to the final path + await rename(tempPath, filepath) + } catch (renameError) { + reject(renameError) + return + } if (onProgress) { onProgress({ downloadedBytes, From f1dd184f4d61078604777e3311b79a4c1cdbf4e5 Mon Sep 17 00:00:00 2001 From: Jake Turner Date: Fri, 17 Apr 2026 20:12:51 +0000 Subject: [PATCH 007/108] fix(Downloads): remove duplicate err listnr and improv Range req stability --- admin/app/utils/downloads.ts | 39 ++++++++++++++++++++++++++---------- 1 file changed, 28 insertions(+), 11 deletions(-) diff --git a/admin/app/utils/downloads.ts b/admin/app/utils/downloads.ts index b439be84..5d91d619 100644 --- a/admin/app/utils/downloads.ts +++ b/admin/app/utils/downloads.ts @@ -86,17 +86,29 @@ export async function doResumableDownload({ headers.Range = `bytes=${startByte}-` } - const response = await axios.get(url, { - responseType: 'stream', - headers, - signal, - timeout, - }) + const fetchStream = (hdrs: Record) => + axios.get(url, { responseType: 'stream', headers: hdrs, signal, timeout }) + + let response = await fetchStream(headers) if (response.status !== 200 && response.status !== 206) { throw new Error(`Failed to download: HTTP ${response.status}`) } + // If we requested a range but the server returned 200 (ignored the Range header), + // appending would corrupt the .tmp file — delete it and restart from byte 0. + if (headers.Range && response.status === 200) { + response.data.destroy() + await deleteFileIfExists(tempPath) + startByte = 0 + appendMode = false + delete headers.Range + response = await fetchStream(headers) + if (response.status !== 200 && response.status !== 206) { + throw new Error(`Failed to download: HTTP ${response.status}`) + } + } + return new Promise((resolve, reject) => { let downloadedBytes = startByte let lastProgressTime = Date.now() @@ -149,7 +161,6 @@ export async function doResumableDownload({ flags: appendMode ? 'a' : 'w', }) - // Handle errors and cleanup const cleanup = (error?: Error) => { clearStallTimer() progressStream.destroy() @@ -163,7 +174,6 @@ export async function doResumableDownload({ response.data.on('error', cleanup) progressStream.on('error', cleanup) writeStream.on('error', cleanup) - writeStream.on('error', cleanup) signal?.addEventListener('abort', () => { cleanup(new Error('Download aborted')) @@ -175,8 +185,15 @@ export async function doResumableDownload({ // Atomically move the completed .tmp file to the final path await rename(tempPath, filepath) } catch (renameError) { - reject(renameError) - return + // A parallel job may have completed the same file first — treat as success + // if the destination already exists at the expected size. + const existing = await getFileStatsIfExists(filepath) + if (existing && Number(existing.size) === totalBytes && totalBytes > 0) { + // fall through to resolve + } else { + reject(renameError) + return + } } if (onProgress) { onProgress({ @@ -228,7 +245,7 @@ export async function doResumableDownloadWithRetry({ }) return result // return on success - } catch (error) { + } catch (error: any) { attempt++ lastError = error as Error From 0d5b6f7927a9bfca5eb2a55b2c062290f0fa8772 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lu=C3=ADs=20Miguel?= <148079567+LuisMIguelFurlanettoSousa@users.noreply.github.com> Date: Fri, 17 Apr 2026 18:12:02 -0300 Subject: [PATCH 008/108] fix(security): SSRF validation for map downloads and error sanitization (CWE-918, CWE-209) (#552) * fix(security): add SSRF validation to map download URLs from manifest * fix(security): sanitize verbose error in rag controller scan endpoint * fix(security): sanitize verbose errors in benchmark controller * fix(security): sanitize verbose error in system controller version fetch * fix(security): sanitize verbose errors in chats controller (6 instances) * fix(security): sanitize verbose errors in docker service (6 instances) * fix(security): sanitize verbose error in system update service * fix(security): sanitize verbose errors in collection update service --------- Co-authored-by: Jake Turner <52841588+jakeaturner@users.noreply.github.com> --- admin/app/controllers/benchmark_controller.ts | 7 +++-- admin/app/controllers/chats_controller.ts | 19 +++++++++----- admin/app/controllers/rag_controller.ts | 4 ++- admin/app/controllers/system_controller.ts | 4 ++- .../app/services/collection_update_service.ts | 4 +-- admin/app/services/docker_service.ts | 26 +++++++++---------- admin/app/services/map_service.ts | 12 ++++++++- admin/app/services/system_update_service.ts | 4 +-- 8 files changed, 52 insertions(+), 28 deletions(-) diff --git a/admin/app/controllers/benchmark_controller.ts b/admin/app/controllers/benchmark_controller.ts index b3e5343c..da483c05 100644 --- a/admin/app/controllers/benchmark_controller.ts +++ b/admin/app/controllers/benchmark_controller.ts @@ -5,6 +5,7 @@ import { runBenchmarkValidator, submitBenchmarkValidator } from '#validators/ben import { RunBenchmarkJob } from '#jobs/run_benchmark_job' import type { BenchmarkType } from '../../types/benchmark.js' import { randomUUID } from 'node:crypto' +import logger from '@adonisjs/core/services/logger' @inject() export default class BenchmarkController { @@ -52,9 +53,10 @@ export default class BenchmarkController { result, }) } catch (error) { + logger.error({ err: error }, '[BenchmarkController] Benchmark run failed') return response.status(500).send({ success: false, - error: error.message, + error: 'An internal error occurred while running the benchmark.', }) } } @@ -181,9 +183,10 @@ export default class BenchmarkController { } catch (error) { // Pass through the status code from the service if available, otherwise default to 400 const statusCode = (error as any).statusCode || 400 + logger.error({ err: error }, '[BenchmarkController] Benchmark submit failed') return response.status(statusCode).send({ success: false, - error: error.message, + error: 'Failed to submit benchmark results.', }) } } diff --git a/admin/app/controllers/chats_controller.ts b/admin/app/controllers/chats_controller.ts index 005e60da..ff25f8b0 100644 --- a/admin/app/controllers/chats_controller.ts +++ b/admin/app/controllers/chats_controller.ts @@ -5,6 +5,7 @@ import { createSessionSchema, updateSessionSchema, addMessageSchema } from '#val import KVStore from '#models/kv_store' import { SystemService } from '#services/system_service' import { SERVICE_NAMES } from '../../constants/service_names.js' +import logger from '@adonisjs/core/services/logger' @inject() export default class ChatsController { @@ -45,8 +46,9 @@ export default class ChatsController { const session = await this.chatService.createSession(data.title, data.model) return response.status(201).json(session) } catch (error) { + logger.error({ err: error }, '[ChatsController] Failed to create session') return response.status(500).json({ - error: error instanceof Error ? error.message : 'Failed to create session', + error: 'Failed to create session', }) } } @@ -56,8 +58,9 @@ export default class ChatsController { const suggestions = await this.chatService.getChatSuggestions() return response.status(200).json({ suggestions }) } catch (error) { + logger.error({ err: error }, '[ChatsController] Failed to get suggestions') return response.status(500).json({ - error: error instanceof Error ? error.message : 'Failed to get suggestions', + error: 'Failed to get suggestions', }) } } @@ -69,8 +72,9 @@ export default class ChatsController { const session = await this.chatService.updateSession(sessionId, data) return session } catch (error) { + logger.error({ err: error }, '[ChatsController] Failed to update session') return response.status(500).json({ - error: error instanceof Error ? error.message : 'Failed to update session', + error: 'Failed to update session', }) } } @@ -81,8 +85,9 @@ export default class ChatsController { await this.chatService.deleteSession(sessionId) return response.status(204) } catch (error) { + logger.error({ err: error }, '[ChatsController] Failed to delete session') return response.status(500).json({ - error: error instanceof Error ? error.message : 'Failed to delete session', + error: 'Failed to delete session', }) } } @@ -94,8 +99,9 @@ export default class ChatsController { const message = await this.chatService.addMessage(sessionId, data.role, data.content) return response.status(201).json(message) } catch (error) { + logger.error({ err: error }, '[ChatsController] Failed to add message') return response.status(500).json({ - error: error instanceof Error ? error.message : 'Failed to add message', + error: 'Failed to add message', }) } } @@ -105,8 +111,9 @@ export default class ChatsController { const result = await this.chatService.deleteAllSessions() return response.status(200).json(result) } catch (error) { + logger.error({ err: error }, '[ChatsController] Failed to delete all sessions') return response.status(500).json({ - error: error instanceof Error ? error.message : 'Failed to delete all sessions', + error: 'Failed to delete all sessions', }) } } diff --git a/admin/app/controllers/rag_controller.ts b/admin/app/controllers/rag_controller.ts index 55b5ef66..149ba7e4 100644 --- a/admin/app/controllers/rag_controller.ts +++ b/admin/app/controllers/rag_controller.ts @@ -6,6 +6,7 @@ import app from '@adonisjs/core/services/app' import { randomBytes } from 'node:crypto' import { sanitizeFilename } from '../utils/fs.js' import { deleteFileSchema, getJobStatusSchema } from '#validators/rag' +import logger from '@adonisjs/core/services/logger' @inject() export default class RagController { @@ -92,7 +93,8 @@ export default class RagController { const syncResult = await this.ragService.scanAndSyncStorage() return response.status(200).json(syncResult) } catch (error) { - return response.status(500).json({ error: 'Error scanning and syncing storage', details: error.message }) + logger.error({ err: error }, '[RagController] Error scanning and syncing storage') + return response.status(500).json({ error: 'Error scanning and syncing storage' }) } } } diff --git a/admin/app/controllers/system_controller.ts b/admin/app/controllers/system_controller.ts index fbc872a3..8967e6dc 100644 --- a/admin/app/controllers/system_controller.ts +++ b/admin/app/controllers/system_controller.ts @@ -6,6 +6,7 @@ import { CheckServiceUpdatesJob } from '#jobs/check_service_updates_job' import { affectServiceValidator, checkLatestVersionValidator, installServiceValidator, subscribeToReleaseNotesValidator, updateServiceValidator } from '#validators/system'; import { inject } from '@adonisjs/core' import type { HttpContext } from '@adonisjs/core/http' +import logger from '@adonisjs/core/services/logger' @inject() export default class SystemController { @@ -144,7 +145,8 @@ export default class SystemController { ) response.send({ versions: updates }) } catch (error) { - response.status(500).send({ error: `Failed to fetch versions: ${error.message}` }) + logger.error({ err: error }, `[SystemController] Failed to fetch versions for ${serviceName}`) + response.status(500).send({ error: 'Failed to fetch available versions for this service.' }) } } diff --git a/admin/app/services/collection_update_service.ts b/admin/app/services/collection_update_service.ts index b1e06d19..fee6c148 100644 --- a/admin/app/services/collection_update_service.ts +++ b/admin/app/services/collection_update_service.ts @@ -65,7 +65,7 @@ export class CollectionUpdateService { return { updates: [], checked_at: new Date().toISOString(), - error: `Nomad API returned status ${error.response.status}`, + error: 'Failed to check for content updates. The update service may be temporarily unavailable.', } } const message = @@ -74,7 +74,7 @@ export class CollectionUpdateService { return { updates: [], checked_at: new Date().toISOString(), - error: `Failed to contact Nomad API: ${message}`, + error: 'Failed to contact the update service. Please try again later.', } } } diff --git a/admin/app/services/docker_service.ts b/admin/app/services/docker_service.ts index 2f0b6e8e..714cd112 100644 --- a/admin/app/services/docker_service.ts +++ b/admin/app/services/docker_service.ts @@ -110,10 +110,10 @@ export class DockerService { message: `Invalid action: ${action}. Use 'start', 'stop', or 'restart'.`, } } catch (error: any) { - logger.error(`Error starting service ${serviceName}: ${error.message}`) + logger.error({ err: error }, `[DockerService] Error controlling service ${serviceName}`) return { success: false, - message: `Failed to start service ${serviceName}: ${error.message}`, + message: `Failed to ${action} service ${serviceName}. Check server logs for details.`, } } } @@ -355,8 +355,8 @@ export class DockerService { ) } } catch (error: any) { - logger.warn(`Error during container cleanup: ${error.message}`) - this._broadcast(serviceName, 'cleanup-warning', `Warning during cleanup: ${error.message}`) + logger.warn({ err: error }, `[DockerService] Error during container cleanup for ${serviceName}`) + this._broadcast(serviceName, 'cleanup-warning', 'Warning during container cleanup. Check server logs for details.') } // Step 3: Clear volumes/data if needed @@ -382,11 +382,11 @@ export class DockerService { this._broadcast(serviceName, 'no-volumes', `No volumes found to clear`) } } catch (error: any) { - logger.warn(`Error during volume cleanup: ${error.message}`) + logger.warn({ err: error }, `[DockerService] Error during volume cleanup for ${serviceName}`) this._broadcast( serviceName, 'volume-cleanup-warning', - `Warning during volume cleanup: ${error.message}` + 'Warning during volume cleanup. Check server logs for details.' ) } @@ -411,11 +411,11 @@ export class DockerService { message: `Service ${serviceName} force reinstall initiated successfully. You can receive updates via server-sent events.`, } } catch (error: any) { - logger.error(`Force reinstall failed for ${serviceName}: ${error.message}`) + logger.error({ err: error }, `[DockerService] Force reinstall failed for ${serviceName}`) await this._cleanupFailedInstallation(serviceName) return { success: false, - message: `Failed to force reinstall service ${serviceName}: ${error.message}`, + message: `Failed to force reinstall service ${serviceName}. Check server logs for details.`, } } } @@ -664,10 +664,10 @@ export class DockerService { return { success: true, message: `Service ${serviceName} container removed successfully` } } catch (error: any) { - logger.error(`Error removing service container: ${error.message}`) + logger.error({ err: error }, `[DockerService] Error removing service container ${serviceName}`) return { success: false, - message: `Failed to remove service ${serviceName} container: ${error.message}`, + message: `Failed to remove service ${serviceName} container. Check server logs for details.`, } } } @@ -1204,10 +1204,10 @@ export class DockerService { this._broadcast( serviceName, 'update-rollback', - `Update failed: ${error.message}` + 'Update failed. Check server logs for details.' ) - logger.error(`[DockerService] Update failed for ${serviceName}: ${error.message}`) - return { success: false, message: `Update failed: ${error.message}` } + logger.error({ err: error }, `[DockerService] Update failed for ${serviceName}`) + return { success: false, message: 'Update failed. Check server logs for details.' } } } diff --git a/admin/app/services/map_service.ts b/admin/app/services/map_service.ts index 9e4363b6..c9902a3c 100644 --- a/admin/app/services/map_service.ts +++ b/admin/app/services/map_service.ts @@ -17,6 +17,7 @@ import { join, resolve, sep } from 'path' import urlJoin from 'url-join' import { RunDownloadJob } from '#jobs/run_download_job' import logger from '@adonisjs/core/services/logger' +import { assertNotPrivateUrl } from '#validators/common' import InstalledResource from '#models/installed_resource' import { CollectionManifestService } from './collection_manifest_service.js' import type { CollectionWithStatus, MapsSpec } from '../../types/collections.js' @@ -119,6 +120,13 @@ export class MapService implements IMapService { const downloadFilenames: string[] = [] for (const resource of toDownload) { + try { + assertNotPrivateUrl(resource.url) + } catch { + logger.warn(`[MapService] Blocked download from private/loopback URL: ${resource.url}`) + continue + } + const existing = await RunDownloadJob.getActiveByUrl(resource.url) if (existing) { logger.warn(`[MapService] Download already in progress for URL ${resource.url}, skipping.`) @@ -244,6 +252,7 @@ export class MapService implements IMapService { url: string ): Promise<{ filename: string; size: number } | { message: string }> { try { + assertNotPrivateUrl(url) const parsed = new URL(url) if (!parsed.pathname.endsWith('.pmtiles')) { throw new Error(`Invalid PMTiles file URL: ${url}. URL must end with .pmtiles`) @@ -267,7 +276,8 @@ export class MapService implements IMapService { return { filename, size } } catch (error: any) { - return { message: `Preflight check failed: ${error.message}` } + logger.error({ err: error }, '[MapService] Preflight check failed for URL') + return { message: 'Preflight check failed. Please verify the URL is valid and accessible.' } } } diff --git a/admin/app/services/system_update_service.ts b/admin/app/services/system_update_service.ts index 20031eb4..73f07913 100644 --- a/admin/app/services/system_update_service.ts +++ b/admin/app/services/system_update_service.ts @@ -47,10 +47,10 @@ export class SystemUpdateService { message: 'System update initiated. The admin container will restart during the process.', } } catch (error) { - logger.error('[SystemUpdateService]: Failed to request system update:', error) + logger.error({ err: error }, '[SystemUpdateService] Failed to request system update') return { success: false, - message: `Failed to request update: ${error.message}`, + message: 'Failed to request system update. Check server logs for details.', } } } From 53d143bb223d8f976b8a923d83dfdd501de7258c Mon Sep 17 00:00:00 2001 From: chriscrosstalk <49691103+chriscrosstalk@users.noreply.github.com> Date: Fri, 17 Apr 2026 14:43:41 -0700 Subject: [PATCH 009/108] fix(AI): allow cancelling in-progress model downloads and ensure consistent progress UI (#701) Adds a cancel button to in-progress Ollama model downloads and unifies the Active Model Downloads card layout with the Active Downloads card used for ZIMs, maps, and pmtiles (byte counts, progress bar, live speed, status indicator). Closes #676. --- admin/app/jobs/download_model_job.ts | 136 ++++++++--- admin/app/services/download_service.ts | 150 ++++++++---- admin/app/services/ollama_service.ts | 132 ++++++++-- .../components/ActiveModelDownloads.tsx | 228 +++++++++++++++--- .../inertia/hooks/useOllamaModelDownloads.ts | 46 +++- 5 files changed, 569 insertions(+), 123 deletions(-) diff --git a/admin/app/jobs/download_model_job.ts b/admin/app/jobs/download_model_job.ts index 4e27a498..f1890215 100644 --- a/admin/app/jobs/download_model_job.ts +++ b/admin/app/jobs/download_model_job.ts @@ -21,6 +21,25 @@ export class DownloadModelJob { return createHash('sha256').update(modelName).digest('hex').slice(0, 16) } + /** In-memory registry of abort controllers for active model download jobs */ + static abortControllers: Map = new Map() + + /** + * Redis key used to signal cancellation across processes. Uses a `model-cancel` prefix + * so it cannot collide with content download cancel signals (`nomad:download:cancel:*`). + */ + static cancelKey(jobId: string): string { + return `nomad:download:model-cancel:${jobId}` + } + + /** Signal cancellation via Redis so the worker process can pick it up on its next poll tick */ + static async signalCancel(jobId: string): Promise { + const queueService = new QueueService() + const queue = queueService.getQueue(this.queue) + const client = await queue.client + await client.set(this.cancelKey(jobId), '1', 'EX', 300) // 5 min TTL + } + async handle(job: Job) { const { modelName } = job.data as DownloadModelJobParams @@ -41,43 +60,96 @@ export class DownloadModelJob { `[DownloadModelJob] Ollama service is ready. Initiating download for ${modelName}` ) - // Services are ready, initiate the download with progress tracking - const result = await ollamaService.downloadModel(modelName, (progressPercent) => { - if (progressPercent) { - job.updateProgress(Math.floor(progressPercent)).catch((err) => { - if (err?.code !== -1) throw err - }) - logger.info( - `[DownloadModelJob] Model ${modelName}: ${progressPercent}%` - ) - } + // Register abort controller for this job — used both by in-process cancels (same process + // as the API server) and as the target of the Redis poll loop below. + const abortController = new AbortController() + DownloadModelJob.abortControllers.set(job.id!, abortController) - // Store detailed progress in job data for clients to query - job.updateData({ - ...job.data, - status: 'downloading', - progress: progressPercent, - progress_timestamp: new Date().toISOString(), - }).catch((err) => { - if (err?.code !== -1) throw err - }) - }) + // Get Redis client for checking cancel signals from the API process + const queueService = new QueueService() + const cancelRedis = await queueService.getQueue(DownloadModelJob.queue).client + + // Track whether cancellation was explicitly requested by the user. Only user-initiated + // cancels become UnrecoverableError — other failures (e.g., transient network errors) + // should still benefit from BullMQ's retry logic. + let userCancelled = false + + // Poll Redis for cancel signal every 2s — independent of progress events so cancellation + // works even when the pull is mid-blob and not emitting progress updates. + let cancelPollInterval: ReturnType | null = setInterval(async () => { + try { + const val = await cancelRedis.get(DownloadModelJob.cancelKey(job.id!)) + if (val) { + await cancelRedis.del(DownloadModelJob.cancelKey(job.id!)) + userCancelled = true + abortController.abort('user-cancel') + } + } catch { + // Redis errors are non-fatal; in-process AbortController covers same-process cancels + } + }, 2000) - if (!result.success) { - logger.error( - `[DownloadModelJob] Failed to initiate download for model ${modelName}: ${result.message}` + try { + // Services are ready, initiate the download with progress tracking + const result = await ollamaService.downloadModel( + modelName, + (progressPercent, bytes) => { + if (progressPercent) { + job.updateProgress(Math.floor(progressPercent)).catch((err) => { + if (err?.code !== -1) throw err + }) + } + + // Store detailed progress in job data for clients to query + job.updateData({ + ...job.data, + status: 'downloading', + progress: progressPercent, + downloadedBytes: bytes?.downloadedBytes, + totalBytes: bytes?.totalBytes, + progress_timestamp: new Date().toISOString(), + }).catch((err) => { + if (err?.code !== -1) throw err + }) + }, + abortController.signal, + job.id! ) - // Don't retry errors that will never succeed (e.g., Ollama version too old) - if (result.retryable === false) { - throw new UnrecoverableError(result.message) + + if (!result.success) { + logger.error( + `[DownloadModelJob] Failed to initiate download for model ${modelName}: ${result.message}` + ) + // User-initiated cancel — must be unrecoverable to avoid the 40-attempt retry storm. + // The downloadModel() catch block returns retryable: false for cancels, so this branch + // catches both Ollama version mismatches (existing) AND user cancels (new). + if (result.retryable === false) { + throw new UnrecoverableError(result.message) + } + throw new Error(`Failed to initiate download for model: ${result.message}`) } - throw new Error(`Failed to initiate download for model: ${result.message}`) - } - logger.info(`[DownloadModelJob] Successfully completed download for model ${modelName}`) - return { - modelName, - message: result.message, + logger.info(`[DownloadModelJob] Successfully completed download for model ${modelName}`) + return { + modelName, + message: result.message, + } + } catch (error: any) { + // Belt-and-suspenders: if downloadModel didn't recognize the cancel (e.g., the abort + // fired after the response stream completed but before our code returned), the cancel + // flag tells us this was a user action and should be unrecoverable. + if (userCancelled || abortController.signal.reason === 'user-cancel') { + if (!(error instanceof UnrecoverableError)) { + throw new UnrecoverableError(`Model download cancelled: ${error.message ?? error}`) + } + } + throw error + } finally { + if (cancelPollInterval !== null) { + clearInterval(cancelPollInterval) + cancelPollInterval = null + } + DownloadModelJob.abortControllers.delete(job.id!) } } diff --git a/admin/app/services/download_service.ts b/admin/app/services/download_service.ts index ac9d02dc..bd9076c6 100644 --- a/admin/app/services/download_service.ts +++ b/admin/app/services/download_service.ts @@ -5,6 +5,8 @@ import { DownloadModelJob } from '#jobs/download_model_job' import { DownloadJobWithProgress, DownloadProgressData } from '../../types/downloads.js' import { normalize } from 'path' import { deleteFileIfExists } from '../utils/fs.js' +import transmit from '@adonisjs/transmit/services/main' +import { BROADCAST_CHANNELS } from '../../constants/broadcast.js' @inject() export class DownloadService { @@ -111,14 +113,32 @@ export class DownloadService { } async cancelJob(jobId: string): Promise<{ success: boolean; message: string }> { + // Try the file download queue first (the original PR #554 path) const queue = this.queueService.getQueue(RunDownloadJob.queue) const job = await queue.getJob(jobId) - if (!job) { - // Job already completed (removeOnComplete: true) or doesn't exist - return { success: true, message: 'Job not found (may have already completed)' } + if (job) { + return await this._cancelFileDownloadJob(jobId, job, queue) } + // Fall through to the model download queue + const modelQueue = this.queueService.getQueue(DownloadModelJob.queue) + const modelJob = await modelQueue.getJob(jobId) + + if (modelJob) { + return await this._cancelModelDownloadJob(jobId, modelJob, modelQueue) + } + + // Not found in either queue + return { success: true, message: 'Job not found (may have already completed)' } + } + + /** Cancel a content download (zim, map, pmtiles, etc.) — original PR #554 logic */ + private async _cancelFileDownloadJob( + jobId: string, + job: any, + queue: any + ): Promise<{ success: boolean; message: string }> { const filepath = job.data.filepath // Signal the worker process to abort the download via Redis @@ -128,45 +148,8 @@ export class DownloadService { RunDownloadJob.abortControllers.get(jobId)?.abort('user-cancel') RunDownloadJob.abortControllers.delete(jobId) - // Poll for terminal state (up to 4s at 250ms intervals) — cooperates with BullMQ's lifecycle - // instead of force-removing an active job and losing the worker's failure/cleanup path. - const POLL_INTERVAL_MS = 250 - const POLL_TIMEOUT_MS = 4000 - const deadline = Date.now() + POLL_TIMEOUT_MS - let reachedTerminal = false - - while (Date.now() < deadline) { - await new Promise((resolve) => setTimeout(resolve, POLL_INTERVAL_MS)) - try { - const state = await job.getState() - if (state === 'failed' || state === 'completed' || state === 'unknown') { - reachedTerminal = true - break - } - } catch { - reachedTerminal = true // getState() throws if job is already gone - break - } - } - - if (!reachedTerminal) { - console.warn(`[DownloadService] cancelJob: job ${jobId} did not reach terminal state within timeout, removing anyway`) - } - - // Remove the BullMQ job - try { - await job.remove() - } catch { - // Lock contention fallback: clear lock and retry once - try { - const client = await queue.client - await client.del(`bull:${RunDownloadJob.queue}:${jobId}:lock`) - const updatedJob = await queue.getJob(jobId) - if (updatedJob) await updatedJob.remove() - } catch { - // Best effort - job will be cleaned up on next dismiss attempt - } - } + await this._pollForTerminalState(job, jobId) + await this._removeJobWithLockFallback(job, queue, RunDownloadJob.queue, jobId) // Delete the partial file from disk if (filepath) { @@ -195,4 +178,87 @@ export class DownloadService { return { success: true, message: 'Download cancelled and partial file deleted' } } + + /** Cancel an Ollama model download — mirrors the file cancel pattern but skips file cleanup */ + private async _cancelModelDownloadJob( + jobId: string, + job: any, + queue: any + ): Promise<{ success: boolean; message: string }> { + const modelName: string = job.data?.modelName ?? 'unknown' + + // Signal the worker process to abort the pull via Redis + await DownloadModelJob.signalCancel(jobId) + + // Also try in-memory abort (works if worker is in same process) + DownloadModelJob.abortControllers.get(jobId)?.abort('user-cancel') + DownloadModelJob.abortControllers.delete(jobId) + + await this._pollForTerminalState(job, jobId) + await this._removeJobWithLockFallback(job, queue, DownloadModelJob.queue, jobId) + + // Broadcast a cancelled event so the frontend hook clears the entry. We use percent: -2 + // (distinct from -1 = error) so the hook can route it to a 2s auto-clear instead of the + // 15s error display. The frontend ALSO removes the entry optimistically from the API + // response, so this is belt-and-suspenders for cases where the SSE arrives first. + transmit.broadcast(BROADCAST_CHANNELS.OLLAMA_MODEL_DOWNLOAD, { + model: modelName, + jobId, + percent: -2, + status: 'cancelled', + timestamp: new Date().toISOString(), + }) + + // Note on partial blob cleanup: Ollama manages model blobs internally at + // /root/.ollama/models/blobs/. We deliberately do NOT call /api/delete here — Ollama's + // expected behavior is to retain partial blobs so a re-pull resumes from where it left + // off. If the user wants to reclaim that space, they can re-pull and let it complete, + // or delete the partially-downloaded model from the AI Settings page. + return { success: true, message: 'Model download cancelled' } + } + + /** Wait up to 4s (250ms intervals) for the job to reach a terminal state */ + private async _pollForTerminalState(job: any, jobId: string): Promise { + const POLL_INTERVAL_MS = 250 + const POLL_TIMEOUT_MS = 4000 + const deadline = Date.now() + POLL_TIMEOUT_MS + + while (Date.now() < deadline) { + await new Promise((resolve) => setTimeout(resolve, POLL_INTERVAL_MS)) + try { + const state = await job.getState() + if (state === 'failed' || state === 'completed' || state === 'unknown') { + return + } + } catch { + return // getState() throws if job is already gone + } + } + + console.warn( + `[DownloadService] cancelJob: job ${jobId} did not reach terminal state within timeout, removing anyway` + ) + } + + /** Remove a BullMQ job, clearing a stale worker lock if the first attempt fails */ + private async _removeJobWithLockFallback( + job: any, + queue: any, + queueName: string, + jobId: string + ): Promise { + try { + await job.remove() + } catch { + // Lock contention fallback: clear lock and retry once + try { + const client = await queue.client + await client.del(`bull:${queueName}:${jobId}:lock`) + const updatedJob = await queue.getJob(jobId) + if (updatedJob) await updatedJob.remove() + } catch { + // Best effort - job will be cleaned up on next dismiss attempt + } + } + } } diff --git a/admin/app/services/ollama_service.ts b/admin/app/services/ollama_service.ts index 5e38cb63..27f5cac7 100644 --- a/admin/app/services/ollama_service.ts +++ b/admin/app/services/ollama_service.ts @@ -92,10 +92,21 @@ export class OllamaService { /** * Downloads a model from Ollama with progress tracking. Only works with Ollama backends. * Use dispatchModelDownload() for background job processing where possible. + * + * @param signal Optional AbortSignal — when triggered, the underlying axios stream is cancelled + * and the method returns a non-retryable failure so callers can mark the job + * unrecoverable in BullMQ and avoid the 40-attempt retry storm. + * @param jobId Optional BullMQ job id — included in progress broadcasts so the frontend can + * correlate Transmit events to a cancellable job. */ async downloadModel( model: string, - progressCallback?: (percent: number) => void + progressCallback?: ( + percent: number, + bytes?: { downloadedBytes: number; totalBytes: number } + ) => void, + signal?: AbortSignal, + jobId?: string ): Promise<{ success: boolean; message: string; retryable?: boolean }> { // Deduplicate concurrent downloads of the same model const existing = this.activeDownloads.get(model) @@ -104,7 +115,7 @@ export class OllamaService { return existing } - const downloadPromise = this._doDownloadModel(model, progressCallback) + const downloadPromise = this._doDownloadModel(model, progressCallback, signal, jobId) this.activeDownloads.set(model, downloadPromise) try { return await downloadPromise @@ -115,7 +126,12 @@ export class OllamaService { private async _doDownloadModel( model: string, - progressCallback?: (percent: number) => void + progressCallback?: ( + percent: number, + bytes?: { downloadedBytes: number; totalBytes: number } + ) => void, + signal?: AbortSignal, + jobId?: string ): Promise<{ success: boolean; message: string; retryable?: boolean }> { await this._ensureDependencies() if (!this.baseUrl) { @@ -142,16 +158,45 @@ export class OllamaService { } } - // Stream pull via Ollama native API + // Stream pull via Ollama native API. axios supports `signal` natively for AbortController + // integration — when triggered, the request errors with code 'ERR_CANCELED' which we detect + // in the catch block below to return a non-retryable cancel result. const pullResponse = await axios.post( `${this.baseUrl}/api/pull`, { model, stream: true }, - { responseType: 'stream', timeout: 0 } + { responseType: 'stream', timeout: 0, signal } ) + // Ollama's pull API reports progress per-digest (each blob). A single model can contain + // multiple blobs (weights, tokenizer, template, etc.) and each is reported in turn. + // Aggregate across all digests so the UI shows a single monotonically-increasing total, + // matching the behavior of the content download progress (Active Downloads section). + const digestProgress = new Map() + + // Throttle broadcasts to once per BROADCAST_THROTTLE_MS — Ollama can emit hundreds of + // progress events per second for fast connections, which would flood the Transmit SSE + // channel and cause jittery speed calculations on the frontend. + const BROADCAST_THROTTLE_MS = 500 + let lastBroadcastAt = 0 + await new Promise((resolve, reject) => { let buffer = '' - let lastPercent = -1 + // If the abort fires after headers are received but mid-stream, axios's signal handling + // destroys the stream which surfaces as an 'error' event — wire the signal listener so + // the promise rejects promptly with a recognizable cancel reason. + const onAbort = () => { + const err: any = new Error('Download cancelled') + err.code = 'ERR_CANCELED' + pullResponse.data.destroy(err) + } + if (signal) { + if (signal.aborted) { + onAbort() + return + } + signal.addEventListener('abort', onAbort, { once: true }) + } + pullResponse.data.on('data', (chunk: Buffer) => { buffer += chunk.toString() const lines = buffer.split('\n') @@ -160,12 +205,42 @@ export class OllamaService { if (!line.trim()) continue try { const parsed = JSON.parse(line) - if (parsed.completed && parsed.total) { - const percent = parseFloat(((parsed.completed / parsed.total) * 100).toFixed(2)) - if (percent !== lastPercent) { - lastPercent = percent - this.broadcastDownloadProgress(model, percent) - if (progressCallback) progressCallback(percent) + if (parsed.completed && parsed.total && parsed.digest) { + // Update this digest's progress — take the max seen value so transient + // out-of-order updates don't make the aggregate jump backwards. + const existing = digestProgress.get(parsed.digest) + digestProgress.set(parsed.digest, { + completed: Math.max(existing?.completed ?? 0, parsed.completed), + total: Math.max(existing?.total ?? 0, parsed.total), + }) + + // Compute aggregate across all known blobs + let aggCompleted = 0 + let aggTotal = 0 + for (const { completed, total } of digestProgress.values()) { + aggCompleted += completed + aggTotal += total + } + + const percent = aggTotal > 0 + ? parseFloat(((aggCompleted / aggTotal) * 100).toFixed(2)) + : 0 + + // Throttle broadcasts. Always call the progressCallback though — the worker + // uses it to update job state in Redis, which should reflect the latest view. + const now = Date.now() + if (now - lastBroadcastAt >= BROADCAST_THROTTLE_MS) { + lastBroadcastAt = now + this.broadcastDownloadProgress(model, percent, jobId, { + downloadedBytes: aggCompleted, + totalBytes: aggTotal, + }) + } + if (progressCallback) { + progressCallback(percent, { + downloadedBytes: aggCompleted, + totalBytes: aggTotal, + }) } } } catch { @@ -173,13 +248,31 @@ export class OllamaService { } } }) - pullResponse.data.on('end', resolve) - pullResponse.data.on('error', reject) + pullResponse.data.on('end', () => { + if (signal) signal.removeEventListener('abort', onAbort) + resolve() + }) + pullResponse.data.on('error', (err: any) => { + if (signal) signal.removeEventListener('abort', onAbort) + reject(err) + }) }) logger.info(`[OllamaService] Model "${model}" downloaded successfully.`) return { success: true, message: 'Model downloaded successfully.' } } catch (error) { + // Detect axios cancel (signal-triggered abort). Don't broadcast an error event for + // user-initiated cancels — the cancel handler in DownloadService already broadcasts + // a cancelled state. Returning retryable: false prevents BullMQ retries. + const isCancelled = + axios.isCancel(error) || + (error as any)?.code === 'ERR_CANCELED' || + (error as any)?.name === 'CanceledError' + if (isCancelled) { + logger.info(`[OllamaService] Model "${model}" download cancelled by user.`) + return { success: false, message: 'Download cancelled', retryable: false } + } + const errorMessage = error instanceof Error ? error.message : String(error) logger.error( `[OllamaService] Failed to download model "${model}": ${errorMessage}` @@ -653,10 +746,19 @@ export class OllamaService { }) } - private broadcastDownloadProgress(model: string, percent: number) { + private broadcastDownloadProgress( + model: string, + percent: number, + jobId?: string, + bytes?: { downloadedBytes: number; totalBytes: number } + ) { + // Conditional spread on jobId/bytes — Transmit's Broadcastable type rejects fields whose + // value is `undefined`, so we omit each key entirely when its value isn't available. transmit.broadcast(BROADCAST_CHANNELS.OLLAMA_MODEL_DOWNLOAD, { model, percent, + ...(jobId ? { jobId } : {}), + ...(bytes ? { downloadedBytes: bytes.downloadedBytes, totalBytes: bytes.totalBytes } : {}), timestamp: new Date().toISOString(), }) logger.info(`[OllamaService] Download progress for model "${model}": ${percent}%`) diff --git a/admin/inertia/components/ActiveModelDownloads.tsx b/admin/inertia/components/ActiveModelDownloads.tsx index c927126d..d9640ca7 100644 --- a/admin/inertia/components/ActiveModelDownloads.tsx +++ b/admin/inertia/components/ActiveModelDownloads.tsx @@ -1,50 +1,214 @@ +import { useCallback, useRef, useState } from 'react' import useOllamaModelDownloads from '~/hooks/useOllamaModelDownloads' -import HorizontalBarChart from './HorizontalBarChart' import StyledSectionHeader from './StyledSectionHeader' -import { IconAlertTriangle } from '@tabler/icons-react' +import StyledModal from './StyledModal' +import { IconAlertTriangle, IconLoader2, IconX } from '@tabler/icons-react' +import api from '~/lib/api' +import { useModals } from '~/context/ModalContext' +import { formatBytes } from '~/lib/util' interface ActiveModelDownloadsProps { withHeader?: boolean } +function formatSpeed(bytesPerSec: number): string { + if (bytesPerSec <= 0) return '0 B/s' + if (bytesPerSec < 1024) return `${Math.round(bytesPerSec)} B/s` + if (bytesPerSec < 1024 * 1024) return `${(bytesPerSec / 1024).toFixed(1)} KB/s` + return `${(bytesPerSec / (1024 * 1024)).toFixed(1)} MB/s` +} + const ActiveModelDownloads = ({ withHeader = false }: ActiveModelDownloadsProps) => { - const { downloads } = useOllamaModelDownloads() + const { downloads, removeDownload } = useOllamaModelDownloads() + const { openModal, closeAllModals } = useModals() + const [cancellingModels, setCancellingModels] = useState>(new Set()) + + // Track previous downloadedBytes for speed calculation — mirrors the approach in + // ActiveDownloads.tsx so content + model downloads feel identical. + const prevBytesRef = useRef>(new Map()) + const speedRef = useRef>(new Map()) + + const getSpeed = useCallback((model: string, currentBytes?: number): number => { + if (!currentBytes || currentBytes <= 0) return 0 + + const prev = prevBytesRef.current.get(model) + const now = Date.now() + + if (prev && prev.bytes > 0 && currentBytes > prev.bytes) { + const deltaBytes = currentBytes - prev.bytes + const deltaSec = (now - prev.time) / 1000 + if (deltaSec > 0) { + const instantSpeed = deltaBytes / deltaSec + + // Simple moving average (last 5 samples) + const samples = speedRef.current.get(model) || [] + samples.push(instantSpeed) + if (samples.length > 5) samples.shift() + speedRef.current.set(model, samples) + + const avg = samples.reduce((a, b) => a + b, 0) / samples.length + prevBytesRef.current.set(model, { bytes: currentBytes, time: now }) + return avg + } + } + + // Only set initial observation; never advance timestamp when bytes unchanged + if (!prev) { + prevBytesRef.current.set(model, { bytes: currentBytes, time: now }) + } + return speedRef.current.get(model)?.at(-1) || 0 + }, []) + + const runCancel = async (download: { model: string; jobId?: string }) => { + // Defensive guard: stale broadcasts during a hot upgrade may not include jobId. + // Without it we have nothing to call the cancel API with. + if (!download.jobId) return + + setCancellingModels((prev) => new Set(prev).add(download.model)) + try { + await api.cancelDownloadJob(download.jobId) + // Optimistically clear the entry — the Transmit cancelled broadcast usually + // arrives within a second but we don't want to leave the row hanging if it doesn't. + removeDownload(download.model) + // Clean up speed tracking refs for this model + prevBytesRef.current.delete(download.model) + speedRef.current.delete(download.model) + } finally { + setCancellingModels((prev) => { + const next = new Set(prev) + next.delete(download.model) + return next + }) + } + } + + const confirmCancel = (download: { model: string; jobId?: string }) => { + if (!download.jobId) return + + openModal( + { + closeAllModals() + runCancel(download) + }} + onCancel={closeAllModals} + open={true} + confirmText="Cancel Download" + cancelText="Keep Downloading" + > +
+

+ Stop downloading {download.model}? +

+

+ Any data already downloaded will remain on disk. If you re-download + this model later, it will resume from where it left off rather than + starting over. +

+
+
, + 'confirm-cancel-model-download-modal' + ) + } return ( <> {withHeader && }
{downloads && downloads.length > 0 ? ( - downloads.map((download) => ( -
- {download.error ? ( -
- -
-

{download.model}

-

{download.error}

+ downloads.map((download) => { + const isCancelling = cancellingModels.has(download.model) + const canCancel = !!download.jobId && !download.error + const speed = getSpeed(download.model, download.downloadedBytes) + const hasBytes = !!(download.downloadedBytes && download.totalBytes) + + return ( +
+ {download.error ? ( +
+ +
+

+ {download.model} +

+

{download.error}

+
+
+ ) : ( +
+ {/* Title + Cancel button row */} +
+
+

+ {download.model} +

+ + ollama + +
+ {canCancel && ( + isCancelling ? ( + + ) : ( + + ) + )} +
+ + {/* Size info */} +
+ + {hasBytes + ? `${formatBytes(download.downloadedBytes!, 1)} / ${formatBytes(download.totalBytes!, 1)}` + : `${download.percent.toFixed(1)}% / 100%`} + +
+ + {/* Progress bar */} +
+
+
+
+
15 + ? 'left-2 text-white drop-shadow-md' + : 'right-2 text-desert-green' + }`} + > + {Math.round(download.percent)}% +
+
+ + {/* Status indicator */} +
+
+ + Downloading...{speed > 0 ? ` ${formatSpeed(speed)}` : ''} + +
-
- ) : ( - - )} -
- )) + )} +
+ ) + }) ) : (

No active model downloads

)} diff --git a/admin/inertia/hooks/useOllamaModelDownloads.ts b/admin/inertia/hooks/useOllamaModelDownloads.ts index 8fc54606..d99e708b 100644 --- a/admin/inertia/hooks/useOllamaModelDownloads.ts +++ b/admin/inertia/hooks/useOllamaModelDownloads.ts @@ -1,11 +1,25 @@ -import { useEffect, useRef, useState } from 'react' +import { useCallback, useEffect, useRef, useState } from 'react' import { useTransmit } from 'react-adonis-transmit' export type OllamaModelDownload = { model: string percent: number timestamp: string + /** + * BullMQ job id — included on progress events from v1.32+ so the frontend can + * call the cancel API. Optional for backward compat with stale broadcasts during + * a hot upgrade. + */ + jobId?: string + /** + * Aggregate bytes across all blobs in the model pull, summed from Ollama's + * per-digest progress events on the backend. Optional for backward compat. + */ + downloadedBytes?: number + totalBytes?: number error?: string + /** Set to 'cancelled' alongside percent === -2 when the user cancels the download */ + status?: 'cancelled' } export default function useOllamaModelDownloads() { @@ -13,6 +27,19 @@ export default function useOllamaModelDownloads() { const [downloads, setDownloads] = useState>(new Map()) const timeoutsRef = useRef>>(new Set()) + /** + * Optimistically remove a download from local state — used by the cancel UI to clear + * the entry immediately on a successful API call, in case the Transmit cancelled + * broadcast arrives late or the SSE connection drops at exactly the wrong moment. + */ + const removeDownload = useCallback((model: string) => { + setDownloads((current) => { + const next = new Map(current) + next.delete(model) + return next + }) + }, []) + useEffect(() => { const unsubscribe = subscribe('ollama-model-download', (data: OllamaModelDownload) => { setDownloads((prev) => { @@ -30,6 +57,21 @@ export default function useOllamaModelDownloads() { }) }, 15000) timeoutsRef.current.add(errorTimeout) + } else if (data.percent === -2) { + // Download cancelled — clear quickly (matches the completion TTL). + // Component-level optimistic removal usually beats this branch, but it's + // here as a safety net for cases where the cancel comes from another tab + // or another client. + const cancelTimeout = setTimeout(() => { + timeoutsRef.current.delete(cancelTimeout) + setDownloads((current) => { + const next = new Map(current) + next.delete(data.model) + return next + }) + }, 2000) + timeoutsRef.current.add(cancelTimeout) + updated.delete(data.model) } else if (data.percent >= 100) { // If download is complete, keep it for a short time before removing to allow UI to show 100% progress updated.set(data.model, data) @@ -60,5 +102,5 @@ export default function useOllamaModelDownloads() { const downloadsArray = Array.from(downloads.values()) - return { downloads: downloadsArray, activeCount: downloads.size } + return { downloads: downloadsArray, activeCount: downloads.size, removeDownload } } From d8ee6f5ceb8f862dcfe0758a945d9bd753cb8035 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 17 Apr 2026 14:45:03 -0700 Subject: [PATCH 010/108] build(deps): bump follow-redirects from 1.15.11 to 1.16.0 in /admin (#729) Bumps [follow-redirects](https://github.com/follow-redirects/follow-redirects) from 1.15.11 to 1.16.0. - [Release notes](https://github.com/follow-redirects/follow-redirects/releases) - [Commits](https://github.com/follow-redirects/follow-redirects/compare/v1.15.11...v1.16.0) --- updated-dependencies: - dependency-name: follow-redirects dependency-version: 1.16.0 dependency-type: indirect ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- admin/package-lock.json | 16 +++------------- 1 file changed, 3 insertions(+), 13 deletions(-) diff --git a/admin/package-lock.json b/admin/package-lock.json index d34cc880..32b57178 100644 --- a/admin/package-lock.json +++ b/admin/package-lock.json @@ -4383,7 +4383,6 @@ "cpu": [ "arm64" ], - "dev": true, "license": "Apache-2.0 AND MIT", "optional": true, "os": [ @@ -4400,7 +4399,6 @@ "cpu": [ "x64" ], - "dev": true, "license": "Apache-2.0 AND MIT", "optional": true, "os": [ @@ -4417,7 +4415,6 @@ "cpu": [ "arm" ], - "dev": true, "license": "Apache-2.0", "optional": true, "os": [ @@ -4434,7 +4431,6 @@ "cpu": [ "arm64" ], - "dev": true, "license": "Apache-2.0 AND MIT", "optional": true, "os": [ @@ -4451,7 +4447,6 @@ "cpu": [ "arm64" ], - "dev": true, "license": "Apache-2.0 AND MIT", "optional": true, "os": [ @@ -4468,7 +4463,6 @@ "cpu": [ "x64" ], - "dev": true, "license": "Apache-2.0 AND MIT", "optional": true, "os": [ @@ -4485,7 +4479,6 @@ "cpu": [ "x64" ], - "dev": true, "license": "Apache-2.0 AND MIT", "optional": true, "os": [ @@ -4502,7 +4495,6 @@ "cpu": [ "arm64" ], - "dev": true, "license": "Apache-2.0 AND MIT", "optional": true, "os": [ @@ -4519,7 +4511,6 @@ "cpu": [ "ia32" ], - "dev": true, "license": "Apache-2.0 AND MIT", "optional": true, "os": [ @@ -4536,7 +4527,6 @@ "cpu": [ "x64" ], - "dev": true, "license": "Apache-2.0 AND MIT", "optional": true, "os": [ @@ -9068,9 +9058,9 @@ } }, "node_modules/follow-redirects": { - "version": "1.15.11", - "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.11.tgz", - "integrity": "sha512-deG2P0JfjrTxl50XGCDyfI97ZGVCxIpfKYmfyrQ54n5FO/0gfIES8C/Psl6kWVDolizcaaxZJnTS0QSMxvnsBQ==", + "version": "1.16.0", + "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.16.0.tgz", + "integrity": "sha512-y5rN/uOsadFT/JfYwhxRS5R7Qce+g3zG97+JrtFZlC9klX/W5hD7iiLzScI4nZqUS7DNUdhPgw4xI8W2LuXlUw==", "funding": [ { "type": "individual", From 2075a62b6031d1724a56d2d6071800d35ac23f53 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 17 Apr 2026 14:45:40 -0700 Subject: [PATCH 011/108] build(deps): bump axios from 1.13.5 to 1.15.0 in /admin (#708) Bumps [axios](https://github.com/axios/axios) from 1.13.5 to 1.15.0. - [Release notes](https://github.com/axios/axios/releases) - [Changelog](https://github.com/axios/axios/blob/v1.x/CHANGELOG.md) - [Commits](https://github.com/axios/axios/compare/v1.13.5...v1.15.0) --- updated-dependencies: - dependency-name: axios dependency-version: 1.15.0 dependency-type: direct:production ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- admin/package-lock.json | 21 ++++++++++++--------- admin/package.json | 2 +- 2 files changed, 13 insertions(+), 10 deletions(-) diff --git a/admin/package-lock.json b/admin/package-lock.json index 32b57178..5c18331f 100644 --- a/admin/package-lock.json +++ b/admin/package-lock.json @@ -38,7 +38,7 @@ "@vinejs/vine": "^3.0.1", "@vitejs/plugin-react": "^4.6.0", "autoprefixer": "^10.4.21", - "axios": "^1.13.5", + "axios": "^1.15.0", "better-sqlite3": "^12.1.1", "bullmq": "^5.65.1", "cheerio": "^1.2.0", @@ -6398,14 +6398,14 @@ } }, "node_modules/axios": { - "version": "1.13.5", - "resolved": "https://registry.npmjs.org/axios/-/axios-1.13.5.tgz", - "integrity": "sha512-cz4ur7Vb0xS4/KUN0tPWe44eqxrIu31me+fbang3ijiNscE129POzipJJA6zniq2C/Z6sJCjMimjS8Lc/GAs8Q==", + "version": "1.15.0", + "resolved": "https://registry.npmjs.org/axios/-/axios-1.15.0.tgz", + "integrity": "sha512-wWyJDlAatxk30ZJer+GeCWS209sA42X+N5jU2jy6oHTp7ufw8uzUTVFBX9+wTfAlhiJXGS0Bq7X6efruWjuK9Q==", "license": "MIT", "dependencies": { "follow-redirects": "^1.15.11", "form-data": "^4.0.5", - "proxy-from-env": "^1.1.0" + "proxy-from-env": "^2.1.0" } }, "node_modules/bail": { @@ -13791,10 +13791,13 @@ } }, "node_modules/proxy-from-env": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/proxy-from-env/-/proxy-from-env-1.1.0.tgz", - "integrity": "sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg==", - "license": "MIT" + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/proxy-from-env/-/proxy-from-env-2.1.0.tgz", + "integrity": "sha512-cJ+oHTW1VAEa8cJslgmUZrc+sjRKgAKl3Zyse6+PV38hZe/V6Z14TbCuXcan9F9ghlz4QrFr2c92TNF82UkYHA==", + "license": "MIT", + "engines": { + "node": ">=10" + } }, "node_modules/pump": { "version": "3.0.3", diff --git a/admin/package.json b/admin/package.json index ab229f42..206ef190 100644 --- a/admin/package.json +++ b/admin/package.json @@ -91,7 +91,7 @@ "@vinejs/vine": "^3.0.1", "@vitejs/plugin-react": "^4.6.0", "autoprefixer": "^10.4.21", - "axios": "^1.13.5", + "axios": "^1.15.0", "better-sqlite3": "^12.1.1", "bullmq": "^5.65.1", "cheerio": "^1.2.0", From 5ee4e1187cf69cc425e5fee4552a724c9d8e3523 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 17 Apr 2026 14:46:03 -0700 Subject: [PATCH 012/108] build(deps): bump protocol-buffers-schema from 3.6.0 to 3.6.1 in /admin (#736) Bumps [protocol-buffers-schema](https://github.com/mafintosh/protocol-buffers-schema) from 3.6.0 to 3.6.1. - [Commits](https://github.com/mafintosh/protocol-buffers-schema/compare/v3.6.0...v3.6.1) --- updated-dependencies: - dependency-name: protocol-buffers-schema dependency-version: 3.6.1 dependency-type: indirect ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- admin/package-lock.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/admin/package-lock.json b/admin/package-lock.json index 5c18331f..25648b80 100644 --- a/admin/package-lock.json +++ b/admin/package-lock.json @@ -13772,9 +13772,9 @@ } }, "node_modules/protocol-buffers-schema": { - "version": "3.6.0", - "resolved": "https://registry.npmjs.org/protocol-buffers-schema/-/protocol-buffers-schema-3.6.0.tgz", - "integrity": "sha512-TdDRD+/QNdrCGCE7v8340QyuXd4kIWIgapsE2+n/SaGiSSbomYl4TjHlvIoCWRpE7wFt02EpB35VVA2ImcBVqw==", + "version": "3.6.1", + "resolved": "https://registry.npmjs.org/protocol-buffers-schema/-/protocol-buffers-schema-3.6.1.tgz", + "integrity": "sha512-VG2K63Igkiv9p76tk1lilczEK1cT+kCjKtkdhw1dQZV3k3IXJbd3o6Ho8b9zJZaHSnT2hKe4I+ObmX9w6m5SmQ==", "license": "MIT" }, "node_modules/proxy-addr": { From 38dfb19f183373bffb85d59457598336c44649f5 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 17 Apr 2026 14:46:24 -0700 Subject: [PATCH 013/108] build(deps): bump protobufjs from 7.5.4 to 7.5.5 in /admin (#737) Bumps [protobufjs](https://github.com/protobufjs/protobuf.js) from 7.5.4 to 7.5.5. - [Release notes](https://github.com/protobufjs/protobuf.js/releases) - [Changelog](https://github.com/protobufjs/protobuf.js/blob/master/CHANGELOG.md) - [Commits](https://github.com/protobufjs/protobuf.js/compare/protobufjs-v7.5.4...protobufjs-v7.5.5) --- updated-dependencies: - dependency-name: protobufjs dependency-version: 7.5.5 dependency-type: indirect ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- admin/package-lock.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/admin/package-lock.json b/admin/package-lock.json index 25648b80..e80b7786 100644 --- a/admin/package-lock.json +++ b/admin/package-lock.json @@ -13748,9 +13748,9 @@ } }, "node_modules/protobufjs": { - "version": "7.5.4", - "resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-7.5.4.tgz", - "integrity": "sha512-CvexbZtbov6jW2eXAvLukXjXUW1TzFaivC46BpWc/3BpcCysb5Vffu+B3XHMm8lVEuy2Mm4XGex8hBSg1yapPg==", + "version": "7.5.5", + "resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-7.5.5.tgz", + "integrity": "sha512-3wY1AxV+VBNW8Yypfd1yQY9pXnqTAN+KwQxL8iYm3/BjKYMNg4i0owhEe26PWDOMaIrzeeF98Lqd5NGz4omiIg==", "hasInstallScript": true, "license": "BSD-3-Clause", "dependencies": { From 1aa26011b13c16547037435eb401987b77257f65 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 17 Apr 2026 14:47:06 -0700 Subject: [PATCH 014/108] build(deps): bump @adonisjs/http-server from 7.8.0 to 7.8.1 in /admin (#724) Bumps [@adonisjs/http-server](https://github.com/adonisjs/http-server) from 7.8.0 to 7.8.1. - [Release notes](https://github.com/adonisjs/http-server/releases) - [Commits](https://github.com/adonisjs/http-server/compare/v7.8.0...v7.8.1) --- updated-dependencies: - dependency-name: "@adonisjs/http-server" dependency-version: 7.8.1 dependency-type: indirect ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- admin/package-lock.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/admin/package-lock.json b/admin/package-lock.json index e80b7786..36b8d1b1 100644 --- a/admin/package-lock.json +++ b/admin/package-lock.json @@ -520,9 +520,9 @@ } }, "node_modules/@adonisjs/http-server": { - "version": "7.8.0", - "resolved": "https://registry.npmjs.org/@adonisjs/http-server/-/http-server-7.8.0.tgz", - "integrity": "sha512-aVMOpExPDNwxjnKGnc4g4sJTIQC3CfNwzWfPFWJm4WnAGXxdI3OxI2zU9FTopB50y0OVK3dWO4/c1Fu6U4vjWQ==", + "version": "7.8.1", + "resolved": "https://registry.npmjs.org/@adonisjs/http-server/-/http-server-7.8.1.tgz", + "integrity": "sha512-ScwKHJstXQbkQXSNqD6MOESowZ+WhRyDXxjSQV/T7IpyMEg/F8NxpR5jAvrpw1BaGzd3t50LrgTrb7ouD8DOpA==", "license": "MIT", "dependencies": { "@paralleldrive/cuid2": "^2.2.2", From 4497e361007b22d3694b8799925ffe7580df6e39 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 17 Apr 2026 14:47:31 -0700 Subject: [PATCH 015/108] build(deps-dev): bump vite from 6.4.1 to 6.4.2 in /admin (#677) Bumps [vite](https://github.com/vitejs/vite/tree/HEAD/packages/vite) from 6.4.1 to 6.4.2. - [Release notes](https://github.com/vitejs/vite/releases) - [Changelog](https://github.com/vitejs/vite/blob/v6.4.2/packages/vite/CHANGELOG.md) - [Commits](https://github.com/vitejs/vite/commits/v6.4.2/packages/vite) --- updated-dependencies: - dependency-name: vite dependency-version: 6.4.2 dependency-type: direct:development ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- admin/package-lock.json | 8 ++++---- admin/package.json | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/admin/package-lock.json b/admin/package-lock.json index 36b8d1b1..e3e9052c 100644 --- a/admin/package-lock.json +++ b/admin/package-lock.json @@ -96,7 +96,7 @@ "prettier": "^3.5.3", "ts-node-maintained": "^10.9.5", "typescript": "~5.8.3", - "vite": "^6.4.1" + "vite": "^6.4.2" } }, "node_modules/@adobe/css-tools": { @@ -16418,9 +16418,9 @@ } }, "node_modules/vite": { - "version": "6.4.1", - "resolved": "https://registry.npmjs.org/vite/-/vite-6.4.1.tgz", - "integrity": "sha512-+Oxm7q9hDoLMyJOYfUYBuHQo+dkAloi33apOPP56pzj+vsdJDzr+j1NISE5pyaAuKL4A3UD34qd0lx5+kfKp2g==", + "version": "6.4.2", + "resolved": "https://registry.npmjs.org/vite/-/vite-6.4.2.tgz", + "integrity": "sha512-2N/55r4JDJ4gdrCvGgINMy+HH3iRpNIz8K6SFwVsA+JbQScLiC+clmAxBgwiSPgcG9U15QmvqCGWzMbqda5zGQ==", "license": "MIT", "dependencies": { "esbuild": "^0.25.0", diff --git a/admin/package.json b/admin/package.json index 206ef190..d98e7e02 100644 --- a/admin/package.json +++ b/admin/package.json @@ -59,7 +59,7 @@ "prettier": "^3.5.3", "ts-node-maintained": "^10.9.5", "typescript": "~5.8.3", - "vite": "^6.4.1" + "vite": "^6.4.2" }, "dependencies": { "@adonisjs/auth": "^9.4.0", From dcd9f4b238b70bfdf1546e122aae340ffa55f8f9 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 17 Apr 2026 14:48:19 -0700 Subject: [PATCH 016/108] build(deps): bump lodash from 4.17.23 to 4.18.1 in /admin (#643) Bumps [lodash](https://github.com/lodash/lodash) from 4.17.23 to 4.18.1. - [Release notes](https://github.com/lodash/lodash/releases) - [Commits](https://github.com/lodash/lodash/compare/4.17.23...4.18.1) --- updated-dependencies: - dependency-name: lodash dependency-version: 4.18.1 dependency-type: indirect ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- admin/package-lock.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/admin/package-lock.json b/admin/package-lock.json index e3e9052c..ae7ef36b 100644 --- a/admin/package-lock.json +++ b/admin/package-lock.json @@ -11019,9 +11019,9 @@ } }, "node_modules/lodash": { - "version": "4.17.23", - "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.23.tgz", - "integrity": "sha512-LgVTMpQtIopCi79SJeDiP0TfWi5CNEc/L/aRdTh3yIvmZXTnheWpKjSZhnvMl8iXbC1tFg9gdHHDMLoV7CnG+w==", + "version": "4.18.1", + "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.18.1.tgz", + "integrity": "sha512-dMInicTPVE8d1e5otfwmmjlxkZoUpiVLwyeTdUsi/Caj/gfzzblBcCE5sRHV/AsjuCmxWrte2TNGSYuCeCq+0Q==", "license": "MIT" }, "node_modules/lodash-es": { From 6e4795f0d82371b15d44485c7929b707020d2a0d Mon Sep 17 00:00:00 2001 From: Jake Turner Date: Fri, 17 Apr 2026 22:02:45 +0000 Subject: [PATCH 017/108] docs: update release notes --- admin/docs/release-notes.md | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/admin/docs/release-notes.md b/admin/docs/release-notes.md index 08715c83..c8c27662 100644 --- a/admin/docs/release-notes.md +++ b/admin/docs/release-notes.md @@ -1,5 +1,23 @@ # Release Notes +## Unreleased + +### Features + +### Bug Fixes +- **AI Assistant**: In-progress model downloads can now be cancelled properly and the progress UI now matches that of file downloads. Thanks @chriscrosstalk for the contribution! +- **AI Assistant**: Fixed an issue where the AI Assistant settings page could crash if a model object did not have a details property. Thanks @hestela for the fix! +- **Disk Collector**: Improved reporting for NFS mount stats and display in the UI. Thanks @bgauger and @bravosierra99 for the contribution! +- **Downloads**: Downloads are now staged to .tmp files and atomically renamed upon completion to prevent issues with incomplete/corrupt files. Thanks @artbird309 for the contribution! +- **Downloads**: Removed a duplicate error listener and improved stability when handling Range requests for file downloads. Thanks @jakeaturner for the contribution! +- **Downloads**: Added improved handling for corrupt ZIM file downloads and removed duplicate Ollama download logs. Thanks @aegisman for the contribution! +- **Security**: Closed a potential SSRF vulnerability in the map file download functionality by implementing stricter URL validation and blocking private IP ranges. Thanks @LuisMIguelFurlanettoSousa for the fix! +- **Security**: Sanitized error messages from the backend to prevent potential information disclosure. Thanks @LuisMIguelFurlanettoSousa for the fix! + +### Improvements +- **AI Assistant**: Now uses the currently loaded model for query rewriting and chat title generation for improved performance and consistency. Thanks @hestela for the contribution! +- **Dependencies**: Updated various dependencies to close security vulnerabilities and improve stability + ## Version 1.31.0 - April 3, 2026 ### Features From c4aa23a9b6d2380eb75f2ce154aeb6bbfd77df8a Mon Sep 17 00:00:00 2001 From: chriscrosstalk <49691103+chriscrosstalk@users.noreply.github.com> Date: Mon, 20 Apr 2026 14:57:53 -0700 Subject: [PATCH 018/108] docs: add Community Add-Ons page with field manuals + W3Schools packs (#753) Introduces a dedicated page listing third-party ZIM content packs built by the community. Launches with the two current add-ons (jrsphoto field manuals, kennethbrewer W3Schools) and explains how to install a ZIM pack and where to submit a new one for inclusion. - New doc at admin/docs/community-add-ons.md - Wired into DocsService DOC_ORDER (slot 4) and TITLE_OVERRIDES so the hyphen in "Add-Ons" is preserved in the sidebar - README gets a link under Community & Resources Co-authored-by: Claude Opus 4.7 (1M context) --- README.md | 1 + admin/app/services/docs_service.ts | 8 +++-- admin/docs/community-add-ons.md | 48 ++++++++++++++++++++++++++++++ 3 files changed, 54 insertions(+), 3 deletions(-) create mode 100644 admin/docs/community-add-ons.md diff --git a/README.md b/README.md index 99890c6f..0edd05a3 100644 --- a/README.md +++ b/README.md @@ -124,6 +124,7 @@ Contributions are welcome and appreciated! Please see [CONTRIBUTING.md](CONTRIBU - **Benchmark Leaderboard:** [benchmark.projectnomad.us](https://benchmark.projectnomad.us) - See how your hardware stacks up against other NOMAD builds - **Troubleshooting Guide:** [TROUBLESHOOTING.md](TROUBLESHOOTING.md) - Find solutions to common issues - **FAQ:** [FAQ.md](FAQ.md) - Find answers to frequently asked questions +- **Community Add-Ons:** [admin/docs/community-add-ons.md](admin/docs/community-add-ons.md) - Third-party content packs built by the community ## License diff --git a/admin/app/services/docs_service.ts b/admin/app/services/docs_service.ts index ef861a3b..17fee292 100644 --- a/admin/app/services/docs_service.ts +++ b/admin/app/services/docs_service.ts @@ -12,9 +12,10 @@ export class DocsService { 'home': 1, 'getting-started': 2, 'use-cases': 3, - 'faq': 4, - 'about': 5, - 'release-notes': 6, + 'community-add-ons': 4, + 'faq': 5, + 'about': 6, + 'release-notes': 7, } async getDocs() { @@ -91,6 +92,7 @@ export class DocsService { private static readonly TITLE_OVERRIDES: Record = { 'faq': 'FAQ', + 'community-add-ons': 'Community Add-Ons', } private prettify(filename: string) { diff --git a/admin/docs/community-add-ons.md b/admin/docs/community-add-ons.md new file mode 100644 index 00000000..525c1369 --- /dev/null +++ b/admin/docs/community-add-ons.md @@ -0,0 +1,48 @@ +# Community Add-Ons + +Project N.O.M.A.D. ships with a curated set of built-in tools and content, but the community has started building add-ons that extend the platform with specialized offline content packs. These are third-party projects, not maintained by the N.O.M.A.D. team. Install them at your own discretion, and please direct any bugs or feature requests to the add-on's own repository. + +Have you built a NOMAD add-on? Open an issue on the [Project N.O.M.A.D. GitHub repository](https://github.com/Crosstalk-Solutions/project-nomad/issues/new) or send us a note through the [contact form on projectnomad.us](https://www.projectnomad.us/contact), and we'll review it for inclusion on this page. + +--- + +## ZIM Content Packs + +ZIM content packs drop additional offline reference material into your existing Kiwix library. They typically ship with an `install.sh` script that downloads source material, builds a ZIM file with `zimwriterfs`, and registers it with your running Kiwix container. + +### U.S. Military Field Manuals + +**Repository:** [github.com/jrsphoto/ZIM-military-field-manuals](https://github.com/jrsphoto/ZIM-military-field-manuals) + +Roughly 180 public-domain U.S. military field manuals covering field medicine, survival, combat first aid, map reading, and more. Built into a searchable ZIM that drops into your Kiwix library. + +Final ZIM size is around 2 GB. The builder downloads about 2 GB of source PDFs from archive.org during the build. + +### W3Schools Programming Archive + +**Repository:** [github.com/kennethbrewer3/ZIM-w3schools-offline](https://github.com/kennethbrewer3/ZIM-w3schools-offline) + +A full offline copy of the W3Schools programming tutorials, covering HTML, CSS, JavaScript, Python, SQL, and more. Good for learning to code, looking up syntax, or teaching programming in an environment without internet. + +Final ZIM size is around 700 MB. The builder downloads about 6 GB of source files from a GitHub mirror during the build. + +--- + +## Installing a Community Add-On + +Each add-on has its own install instructions, but most ZIM packs follow the same shape: + +1. Clone the add-on's repository onto your NOMAD host over SSH. +2. Check the README for required build dependencies. Most need `git`, `python3`, `unzip`, and `zim-tools`. +3. Run the included `install.sh` with a `--deploy` flag, pointing it at your Kiwix library path (`/opt/project-nomad/storage/zim`) and your Kiwix container name (`nomad_kiwix_server`). +4. The script builds the ZIM, copies it into your Kiwix library, registers it with Kiwix, and restarts the Kiwix container. + +Once the script finishes, the new content will appear in your Information Library the next time you load it. + +Expect the initial build to take anywhere from a few minutes to an hour or more depending on the add-on's size and your host's CPU. + +--- + +## A Note on Support + +These add-ons are community-built and community-maintained. If something goes wrong with an install script or the content inside a ZIM, please open an issue on the add-on's own repository rather than Project N.O.M.A.D.'s. We're happy to help if the issue is with NOMAD itself, for example if Kiwix isn't picking up a new ZIM after an install, but we can't maintain or support third-party content. From 776d099c4ad3938a3e748ce4a8b927c7afa03492 Mon Sep 17 00:00:00 2001 From: chriscrosstalk <49691103+chriscrosstalk@users.noreply.github.com> Date: Mon, 20 Apr 2026 15:25:58 -0700 Subject: [PATCH 019/108] fix(qdrant): disable anonymous telemetry by default (#747) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Qdrant's upstream default enables anonymous telemetry to telemetry.qdrant.io, which doesn't match NOMAD's offline-first "zero telemetry" posture. Adding QDRANT__TELEMETRY_DISABLED=true to the container environment turns it off for fresh installs and reinstalls. Existing installs keep their current telemetry-enabled container until the Qdrant service is force-reinstalled via the Knowledge Base panel or the next container recreation — Docker bakes Env into containers at create time, so env changes require a new container. Closes #742 Co-authored-by: Claude Opus 4.7 (1M context) --- admin/database/seeders/service_seeder.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/admin/database/seeders/service_seeder.ts b/admin/database/seeders/service_seeder.ts index 4bf10870..fd0fcec7 100644 --- a/admin/database/seeders/service_seeder.ts +++ b/admin/database/seeders/service_seeder.ts @@ -57,6 +57,10 @@ export default class ServiceSeeder extends BaseSeeder { PortBindings: { '6333/tcp': [{ HostPort: '6333' }], '6334/tcp': [{ HostPort: '6334' }] }, }, ExposedPorts: { '6333/tcp': {}, '6334/tcp': {} }, + // Disable Qdrant's anonymous telemetry to telemetry.qdrant.io. NOMAD is offline-first + // and ships with zero telemetry by default — Qdrant's upstream default of enabled + // telemetry doesn't match that posture. + Env: ['QDRANT__TELEMETRY_DISABLED=true'], }), ui_location: '6333', installed: false, From 644170ed6bb5281aa64bf770633c31df81994bd7 Mon Sep 17 00:00:00 2001 From: 0xGlitch <92540908+bgauger@users.noreply.github.com> Date: Mon, 20 Apr 2026 16:46:46 -0600 Subject: [PATCH 020/108] fix(UI): gate NAS Storage label on network filesystem type (#749) Closes #743 --- admin/inertia/hooks/useDiskDisplayData.ts | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/admin/inertia/hooks/useDiskDisplayData.ts b/admin/inertia/hooks/useDiskDisplayData.ts index 331b323b..415fae98 100644 --- a/admin/inertia/hooks/useDiskDisplayData.ts +++ b/admin/inertia/hooks/useDiskDisplayData.ts @@ -19,9 +19,15 @@ export function getAllDiskDisplayItems( ): DiskDisplayItem[] { const validDisks = disks?.filter((d) => d.totalSize > 0) || [] - // If /app/storage is on a dedicated filesystem (e.g. NFS), it won't appear - // in the block-device list. Prepend it so NAS and OS disk are both shown. - const storageMount = fsSize?.find((fs) => fs.mount === '/app/storage' && fs.size > 0) + // If /app/storage is backed by a network filesystem (NFS/CIFS), it won't + // appear in the block-device list. Prepend it so NAS and OS disk are both + // shown. Local-disk-backed /app/storage is already reported in disk[] and + // fsSize[], so skip it here to avoid a phantom "NAS Storage" entry. + const NETWORK_FS_TYPES = new Set(['nfs', 'nfs4', 'cifs', 'smbfs', 'smb2', 'smb3']) + const storageMount = fsSize?.find( + (fs) => + fs.mount === '/app/storage' && fs.size > 0 && NETWORK_FS_TYPES.has(fs.type?.toLowerCase()) + ) const storageMountItem: DiskDisplayItem[] = storageMount ? [ { From 36b7613f8511a097b10eb6462053af08c2893b49 Mon Sep 17 00:00:00 2001 From: chriscrosstalk <49691103+chriscrosstalk@users.noreply.github.com> Date: Mon, 20 Apr 2026 16:02:49 -0700 Subject: [PATCH 021/108] fix(AI): stop local nomad_ollama container when remote Ollama is configured (#744) When users set a remote Ollama URL via AI Settings, the local nomad_ollama container continued running and competed with the remote host for port 11434 and GPU access. Now configureRemote stops the local container on set and restores it on clear (if still present). Container and its models volume are preserved so the local install can be re-enabled later. Closes #662 Co-authored-by: Claude Opus 4.7 (1M context) --- admin/app/controllers/ollama_controller.ts | 62 ++++++++++++++++++++-- 1 file changed, 59 insertions(+), 3 deletions(-) diff --git a/admin/app/controllers/ollama_controller.ts b/admin/app/controllers/ollama_controller.ts index edd6f362..68b6b785 100644 --- a/admin/app/controllers/ollama_controller.ts +++ b/admin/app/controllers/ollama_controller.ts @@ -212,13 +212,21 @@ export default class OllamaController { return response.status(404).send({ success: false, message: 'Ollama service record not found.' }) } - // Clear path: null or empty URL removes remote config and marks service as not installed + // Clear path: null or empty URL removes remote config. If a local nomad_ollama container + // still exists (user had previously installed AI Assistant locally), restart it and keep + // the service marked installed. Otherwise fall back to uninstalled. if (!remoteUrl || remoteUrl.trim() === '') { await KVStore.clearValue('ai.remoteOllamaUrl') - ollamaService.installed = false + const hasLocalContainer = await this._startLocalOllamaContainerIfExists() + ollamaService.installed = hasLocalContainer ollamaService.installation_status = 'idle' await ollamaService.save() - return { success: true, message: 'Remote Ollama configuration cleared.' } + return { + success: true, + message: hasLocalContainer + ? 'Remote Ollama cleared. Local Ollama container restored.' + : 'Remote Ollama configuration cleared.', + } } // Validate URL format @@ -253,6 +261,10 @@ export default class OllamaController { ollamaService.installation_status = 'idle' await ollamaService.save() + // Stop the local nomad_ollama container (if running) so it doesn't compete with the + // remote host for GPU / port 11434. Preserves the container and its models volume. + await this._stopLocalOllamaContainer() + // Install Qdrant if not already installed (fire-and-forget) const qdrantService = await Service.query().where('service_name', SERVICE_NAMES.QDRANT).first() if (qdrantService && !qdrantService.installed) { @@ -270,6 +282,50 @@ export default class OllamaController { return { success: true, message: 'Remote Ollama configured.' } } + private async _stopLocalOllamaContainer(): Promise { + try { + const containers = await this.dockerService.docker.listContainers({ all: true }) + const ollamaContainer = containers.find((c) => + c.Names.includes(`/${SERVICE_NAMES.OLLAMA}`) + ) + if (!ollamaContainer || ollamaContainer.State !== 'running') { + return + } + await this.dockerService.docker.getContainer(ollamaContainer.Id).stop() + this.dockerService.invalidateServicesStatusCache() + logger.info('[OllamaController] Stopped local nomad_ollama (remote Ollama configured)') + } catch (error: any) { + logger.error( + { err: error }, + '[OllamaController] Failed to stop local nomad_ollama; remote Ollama is still active' + ) + } + } + + private async _startLocalOllamaContainerIfExists(): Promise { + try { + const containers = await this.dockerService.docker.listContainers({ all: true }) + const ollamaContainer = containers.find((c) => + c.Names.includes(`/${SERVICE_NAMES.OLLAMA}`) + ) + if (!ollamaContainer) { + return false + } + if (ollamaContainer.State !== 'running') { + await this.dockerService.docker.getContainer(ollamaContainer.Id).start() + this.dockerService.invalidateServicesStatusCache() + logger.info('[OllamaController] Started local nomad_ollama (remote Ollama cleared)') + } + return true + } catch (error: any) { + logger.error( + { err: error }, + '[OllamaController] Failed to start local nomad_ollama on remote clear' + ) + return false + } + } + async deleteModel({ request }: HttpContext) { const reqData = await request.validateUsing(modelNameSchema) await this.ollamaService.deleteModel(reqData.model) From d22c0b202c3cd95c6485c0651daf8bc3d6bb77da Mon Sep 17 00:00:00 2001 From: chriscrosstalk <49691103+chriscrosstalk@users.noreply.github.com> Date: Mon, 20 Apr 2026 16:05:27 -0700 Subject: [PATCH 022/108] fix(ZIM): accumulate across Kiwix pages to prevent empty Content Explorer (#746) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When many ZIMs are already installed locally, a single Kiwix catalog page (12 items) could return 12 already-installed items, which zim_service would fully filter out client-side. The endpoint returned items: [] with has_more: true, and the frontend's infinite-scroll guard (flatData.length > 0) blocked fetchNextPage — leaving the user with "No records found" despite plenty of uninstalled ZIMs available. Backend now accumulates across up to 5 Kiwix fetches (60 items each) until it has enough post-filter results to return, dedupes by entry id, advances currentStart by actual entries returned (not requested), and returns a next_start cursor. The frontend consumes that cursor instead of computing Kiwix offsets locally, and the flatData.length > 0 guard is removed so the existing on-mount effect drives bounded auto-fetch when a short page lands. The pre-existing has_more off-by-one (compared totalResults against the input start rather than the post-fetch position) is fixed implicitly. Diagnosis credit: @johno10661. Closes #731 Co-authored-by: Claude Opus 4.7 (1M context) --- admin/app/services/zim_service.ts | 147 ++++++++++-------- .../pages/settings/zim/remote-explorer.tsx | 27 ++-- admin/types/zim.ts | 1 + 3 files changed, 96 insertions(+), 79 deletions(-) diff --git a/admin/app/services/zim_service.ts b/admin/app/services/zim_service.ts index bee43095..c6f427c6 100644 --- a/admin/app/services/zim_service.ts +++ b/admin/app/services/zim_service.ts @@ -57,84 +57,105 @@ export class ZimService { query?: string }): Promise { const LIBRARY_BASE_URL = 'https://browse.library.kiwix.org/catalog/v2/entries' + // Kiwix returns pages of content unaware of what the user has installed locally. When + // the installed set is large, a single 12-item Kiwix page can come back with everything + // already installed → 0 post-filter items → frontend deadlock (#731). Accumulate across + // upstream pages so we return a useful batch. Bounded by MAX_KIWIX_FETCHES so a heavily + // saturated install doesn't hang a single request; the frontend scroll loop + auto-fetch + // effect handle continuation. + const KIWIX_PAGE_SIZE = 60 + const MAX_KIWIX_FETCHES = 5 - const res = await axios.get(LIBRARY_BASE_URL, { - params: { - start: start, - count: count, - lang: 'eng', - ...(query ? { q: query } : {}), - }, - responseType: 'text', - }) - - const data = res.data const parser = new XMLParser({ ignoreAttributes: false, attributeNamePrefix: '', textNodeName: '#text', }) - const result = parser.parse(data) - - if (!isRawListRemoteZimFilesResponse(result)) { - throw new Error('Invalid response format from remote library') - } - - const entries = result.feed.entry - ? Array.isArray(result.feed.entry) - ? result.feed.entry - : [result.feed.entry] - : [] - const filtered = entries.filter((entry: any) => { - return isRawRemoteZimFileEntry(entry) - }) + // Snapshot locally-installed files once — the filesystem won't change mid-request. + const existing = await this.list() + const existingKeys = new Set(existing.files.map((file) => file.name)) - const mapped: (RemoteZimFileEntry | null)[] = filtered.map((entry: RawRemoteZimFileEntry) => { - const downloadLink = entry.link.find((link: any) => { - return ( - typeof link === 'object' && - 'rel' in link && - 'length' in link && - 'href' in link && - 'type' in link && - link.type === 'application/x-zim' - ) + const accumulated: RemoteZimFileEntry[] = [] + const seenIds = new Set() + let currentStart = start + let totalResults = 0 + + for (let i = 0; i < MAX_KIWIX_FETCHES; i++) { + const res = await axios.get(LIBRARY_BASE_URL, { + params: { + start: currentStart, + count: KIWIX_PAGE_SIZE, + lang: 'eng', + ...(query ? { q: query } : {}), + }, + responseType: 'text', }) - if (!downloadLink) { - return null + const parsed = parser.parse(res.data) + if (!isRawListRemoteZimFilesResponse(parsed)) { + throw new Error('Invalid response format from remote library') } - - // downloadLink['href'] will end with .meta4, we need to remove that to get the actual download URL - const download_url = downloadLink['href'].substring(0, downloadLink['href'].length - 6) - const file_name = download_url.split('/').pop() || `${entry.title}.zim` - const sizeBytes = parseInt(downloadLink['length'], 10) - - return { - id: entry.id, - title: entry.title, - updated: entry.updated, - summary: entry.summary, - size_bytes: sizeBytes || 0, - download_url: download_url, - author: entry.author.name, - file_name: file_name, + totalResults = parsed.feed.totalResults + + const rawEntries = parsed.feed.entry + ? Array.isArray(parsed.feed.entry) + ? parsed.feed.entry + : [parsed.feed.entry] + : [] + + // Empty upstream response — bail even if totalResults suggests more (transient Kiwix + // hiccup or totalResults drift between pages). Prevents a pointless spin. + if (rawEntries.length === 0) break + + // Advance by actual returned count, not requested count. Short pages at the tail + // would otherwise cause us to skip entries on the next fetch. + currentStart += rawEntries.length + + for (const raw of rawEntries) { + if (!isRawRemoteZimFileEntry(raw)) continue + const entry = raw as RawRemoteZimFileEntry + + const downloadLink = entry.link.find( + (link: any) => + typeof link === 'object' && + 'rel' in link && + 'length' in link && + 'href' in link && + 'type' in link && + link.type === 'application/x-zim' + ) + if (!downloadLink) continue + + // downloadLink['href'] ends with .meta4; strip that to get the actual .zim URL. + const download_url = downloadLink['href'].substring(0, downloadLink['href'].length - 6) + const file_name = download_url.split('/').pop() || `${entry.title}.zim` + if (existingKeys.has(file_name)) continue + if (seenIds.has(entry.id)) continue + seenIds.add(entry.id) + + const sizeBytes = parseInt(downloadLink['length'], 10) + accumulated.push({ + id: entry.id, + title: entry.title, + updated: entry.updated, + summary: entry.summary, + size_bytes: sizeBytes || 0, + download_url, + author: entry.author.name, + file_name, + }) } - }) - // Filter out any null entries (those without a valid download link) - // or files that already exist in the local storage - const existing = await this.list() - const existingKeys = new Set(existing.files.map((file) => file.name)) - const withoutExisting = mapped.filter( - (entry): entry is RemoteZimFileEntry => entry !== null && !existingKeys.has(entry.file_name) - ) + if (accumulated.length >= count) break + if (currentStart >= totalResults) break + } return { - items: withoutExisting, - has_more: result.feed.totalResults > start, - total_count: result.feed.totalResults, + items: accumulated, + has_more: currentStart < totalResults, + total_count: totalResults, + next_start: currentStart, } } diff --git a/admin/inertia/pages/settings/zim/remote-explorer.tsx b/admin/inertia/pages/settings/zim/remote-explorer.tsx index b515a509..9d1ca3eb 100644 --- a/admin/inertia/pages/settings/zim/remote-explorer.tsx +++ b/admin/inertia/pages/settings/zim/remote-explorer.tsx @@ -83,8 +83,10 @@ export default function ZimRemoteExplorer() { useInfiniteQuery({ queryKey: ['remote-zim-files', query], queryFn: async ({ pageParam = 0 }) => { - const pageParsed = parseInt((pageParam as number).toString(), 10) - const start = isNaN(pageParsed) ? 0 : pageParsed * 12 + // pageParam is an opaque Kiwix offset returned by the backend as `next_start`. + // The backend accumulates across multiple upstream pages when needed (#731), so the + // frontend can't derive the next offset from a 12-item page assumption. + const start = typeof pageParam === 'number' ? pageParam : 0 const res = await api.listRemoteZimFiles({ start, count: 12, query: query || undefined }) if (!res) { throw new Error('Failed to fetch remote ZIM files.') @@ -92,12 +94,7 @@ export default function ZimRemoteExplorer() { return res.data }, initialPageParam: 0, - getNextPageParam: (_lastPage, pages) => { - if (!_lastPage.has_more) { - return undefined // No more pages to fetch - } - return pages.length - }, + getNextPageParam: (lastPage) => (lastPage.has_more ? lastPage.next_start : undefined), refetchOnWindowFocus: false, placeholderData: keepPreviousData, }) @@ -119,18 +116,16 @@ export default function ZimRemoteExplorer() { (parentRef?: HTMLDivElement | null) => { if (parentRef) { const { scrollHeight, scrollTop, clientHeight } = parentRef - //once the user has scrolled within 200px of the bottom of the table, fetch more data if we can - if ( - scrollHeight - scrollTop - clientHeight < 200 && - !isFetching && - hasMore && - flatData.length > 0 - ) { + // Fetch more when near the bottom. The `flatData.length > 0` guard that used to be + // here caused the #731 deadlock when a heavily-saturated install returned an empty + // page with has_more=true — removing it lets the existing on-mount/on-data effect + // below drive bounded auto-fetch until hasMore flips false. + if (scrollHeight - scrollTop - clientHeight < 200 && !isFetching && hasMore) { fetchNextPage() } } }, - [fetchNextPage, isFetching, hasMore, flatData.length] + [fetchNextPage, isFetching, hasMore] ) const virtualizer = useVirtualizer({ diff --git a/admin/types/zim.ts b/admin/types/zim.ts index cfd040a3..96ed081c 100644 --- a/admin/types/zim.ts +++ b/admin/types/zim.ts @@ -16,6 +16,7 @@ export type ListRemoteZimFilesResponse = { items: RemoteZimFileEntry[] has_more: boolean total_count: number + next_start: number } export type RawRemoteZimFileEntry = { From 9c98d8225b20aa3cb6b464a3f50b49dc1e8af935 Mon Sep 17 00:00:00 2001 From: chriscrosstalk <49691103+chriscrosstalk@users.noreply.github.com> Date: Mon, 20 Apr 2026 16:23:25 -0700 Subject: [PATCH 023/108] fix(rag): repair ZIM embedding pipeline (sync filter, batch gate, DOM walk) (#745) Three bugs in the RAG embedding pipeline, diagnosed and patched by @sbruschke against v1.31.0 with working before/after chunk counts. All three are root-cause contributors to #388. 1. scanAndSyncStorage queued every file under /storage/zim/ for embedding, including Kiwix's generated kiwix-library.xml. EmbedFileJob rejected it with "Unsupported file type" and the default 30-attempt retry policy kept it looping on every sync, flooding nomad_admin logs. Now gated on determineFileType(filePath) !== 'unknown'. 2. hasMoreBatches compared zimChunks.length (section-level chunk count under the 'structured' strategy) against ZIM_BATCH_SIZE (an article limit). Because articles emit multiple sections, the two are never equal for real archives and processing silently stopped after the first 50 articles. Now gated on articlesInBatch >= ZIM_BATCH_SIZE. 3. extractStructuredContent walked only direct children of , so any ZIM that wraps content in a container div (Devdocs, Wikipedia, FreeCodeCamp, React docs, etc.) produced zero sections and silently embedded zero chunks while reporting success. Now walks the full DOM via $('body').find('h2, h3, h4, p, ul, ol, dl, table'), with a whole-body text fallback when the selector walk yields nothing. Before/after chunk counts confirmed by @sbruschke on v1.31.0: devdocs_en_git 0 -> 916 devdocs_en_react 0 -> 481 devdocs_en_node 0 -> 423 libretexts_en_eng 1 -> 35 (climbing) Wikipedia resumed progressing normally through its 6M articles. Closes #718 Closes #719 Closes #720 Closes #388 Co-authored-by: Claude Opus 4.7 (1M context) --- admin/app/services/rag_service.ts | 15 +++++++++++---- admin/app/services/zim_extraction_service.ts | 19 ++++++++++++++++++- 2 files changed, 29 insertions(+), 5 deletions(-) diff --git a/admin/app/services/rag_service.ts b/admin/app/services/rag_service.ts index 67e8627a..81145f89 100644 --- a/admin/app/services/rag_service.ts +++ b/admin/app/services/rag_service.ts @@ -532,9 +532,12 @@ export class RagService { } } - // Count unique articles processed in this batch + // Count unique articles processed in this batch. hasMoreBatches gates on the article + // count — zimChunks.length counts section-level chunks (multiple per article under the + // 'structured' strategy), so comparing it to ZIM_BATCH_SIZE (an article limit) caps + // processing at the first batch for any real archive. const articlesInBatch = new Set(zimChunks.map((c) => c.documentId)).size - const hasMoreBatches = zimChunks.length === ZIM_BATCH_SIZE + const hasMoreBatches = articlesInBatch >= ZIM_BATCH_SIZE logger.info( `[RAG] Successfully embedded ${totalChunks} total chunks from ${articlesInBatch} articles (hasMore: ${hasMoreBatches})` @@ -1252,8 +1255,12 @@ export class RagService { logger.info(`[RAG] Found ${sourcesInQdrant.size} unique sources in Qdrant`) - // Find files that are in storage but not in Qdrant - const filesToEmbed = filesInStorage.filter((filePath) => !sourcesInQdrant.has(filePath)) + // Find files that are in storage, not already in Qdrant, and have an embeddable type. + // Non-embeddable files (e.g. kiwix-library.xml in /storage/zim) would otherwise be + // dispatched to EmbedFileJob, fail with "Unsupported file type", and retry on every sync. + const filesToEmbed = filesInStorage.filter( + (filePath) => !sourcesInQdrant.has(filePath) && determineFileType(filePath) !== 'unknown' + ) logger.info(`[RAG] Found ${filesToEmbed.length} files that need embedding`) diff --git a/admin/app/services/zim_extraction_service.ts b/admin/app/services/zim_extraction_service.ts index b3594b64..c48b72d0 100644 --- a/admin/app/services/zim_extraction_service.ts +++ b/admin/app/services/zim_extraction_service.ts @@ -216,7 +216,10 @@ export class ZIMExtractionService { const sections: Array<{ heading: string; text: string; level: number }> = []; let currentSection = { heading: 'Introduction', content: [] as string[], level: 2 }; - $('body').children().each((_, element) => { + // Walk the full DOM rather than only direct children of . Modern ZIMs (Devdocs, + // Wikipedia, FreeCodeCamp, etc.) wrap article content in a container div, which under + // .children() would be a single non-heading/non-paragraph element and yield zero sections. + $('body').find('h2, h3, h4, p, ul, ol, dl, table').each((_, element) => { const $el = $(element); const tagName = element.tagName?.toLowerCase(); @@ -253,6 +256,20 @@ export class ZIMExtractionService { }); } + // Fallback: if the selector walk produced no sections but the body has meaningful + // text (unusual structure, minimal markup), emit one section with the full body text + // so the article still contributes to the knowledge base. + if (sections.length === 0) { + const bodyText = $('body').text().replace(/\s+/g, ' ').trim(); + if (bodyText.length > 0) { + sections.push({ + heading: title || 'Content', + text: bodyText, + level: 2, + }); + } + } + return { title, sections, From 08d14473d214b662b23cceff73fccffee98593bf Mon Sep 17 00:00:00 2001 From: chriscrosstalk <49691103+chriscrosstalk@users.noreply.github.com> Date: Mon, 20 Apr 2026 20:12:36 -0700 Subject: [PATCH 024/108] build: write version.json from VERSION build-arg (#754) The Dockerfile copied root package.json to /app/version.json, which SystemService.getAppVersion() reads on every render of the app version in the UI. semantic-release only reliably commits that bump back on the main branch; on the rc branch it does not, so v1.31.1-rc.1 and v1.31.1-rc.2 both shipped with a version.json still reading 1.31.0. Result: a user who upgrades to rc.2 sees "1.31.0" in the UI and a persistent "update to v1.31.1-rc.2 available" prompt. The build workflow already passes VERSION as a build-arg (used today only for the OCI image label). Generating version.json from that arg at build time makes the image tag the single source of truth and eliminates the drift, regardless of what the committed-back package.json says. Dev builds (no VERSION override) write "dev", which matches the existing NODE_ENV=development short-circuit in getAppVersion(). Co-authored-by: Claude Opus 4.7 (1M context) --- Dockerfile | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index c91f9acb..03acaa9d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -43,8 +43,10 @@ ENV NODE_ENV=production WORKDIR /app COPY --from=production-deps /app/node_modules /app/node_modules COPY --from=build /app/build /app -# Copy root package.json for version info -COPY package.json /app/version.json +# Generate version.json from the VERSION build-arg so the image tag is the +# single source of truth (previously copied root package.json, which drifted +# from the tag when semantic-release did not commit the bump back). +RUN echo "{\"version\":\"${VERSION}\"}" > /app/version.json # Copy docs and README for access within the container COPY admin/docs /app/docs From 90946ecf5a9bbfcf6bd5b3d22c6661045c8583c0 Mon Sep 17 00:00:00 2001 From: chriscrosstalk <49691103+chriscrosstalk@users.noreply.github.com> Date: Mon, 27 Apr 2026 18:37:57 -0700 Subject: [PATCH 025/108] docs: require linked issue for non-trivial PRs (#799) Tightens the existing "open an issue first" guidance: non-trivial PRs must reference a corresponding issue and may be closed without one. Adds an explicit carveout for trivial fixes (typos, doc clarifications, small one-line bugs) so drive-by improvements still flow through. --- CONTRIBUTING.md | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 1fda7fbe..46e0558b 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -30,7 +30,14 @@ We are committed to providing a welcoming environment for everyone. Disrespectfu ## Before You Start -**Open an issue first.** Before writing any code, please [open an issue](../../issues/new) to discuss your proposed change. This helps avoid duplicate work and ensures your contribution aligns with the project's direction. +**Open an issue first.** Before writing any code for a non-trivial change, you must [open an issue](../../issues/new) to discuss your proposed change. This helps avoid duplicate work and ensures your contribution aligns with the project's direction. **Pull requests submitted without a corresponding issue may be closed at the maintainers' discretion.** + +**Trivial fixes are exempt** and may be submitted directly as a PR. Examples: +- Typo and grammar corrections +- Documentation clarifications +- Small one-line bug fixes with an obvious cause + +If you're not sure whether your change qualifies as trivial, open an issue first. When opening an issue: - Use a clear, descriptive title @@ -149,7 +156,7 @@ This project uses [Semantic Versioning](https://semver.org/). Versions are manag 2. Open a pull request against the `dev` branch of this repository 3. In the PR description: - Summarize what your changes do and why - - Reference the related issue (e.g., `Closes #123`) + - Reference the related issue (e.g., `Closes #123`) — required for non-trivial changes - Note any relevant testing steps or environment details 4. Be responsive to feedback — maintainers may request changes. Pull requests with no activity for an extended period may be closed. From b168001450e96ea52f53036416f9afd423d304c4 Mon Sep 17 00:00:00 2001 From: chriscrosstalk <49691103+chriscrosstalk@users.noreply.github.com> Date: Mon, 27 Apr 2026 18:39:28 -0700 Subject: [PATCH 026/108] fix(install): warn loudly on non-x86_64 architectures before pulling images (#797) Detects the host architecture early in the preflight sequence. On any architecture other than x86_64/amd64, prints a 5-line warning that NOMAD officially supports x86_64 only, points at PR #419, and sleeps 10 seconds before continuing. Ctrl+C aborts cleanly before any Docker work happens. Preserves the community/hacker path: ARM64 users running with QEMU binfmt_misc emulation can still let the install proceed. The change just stops the silent 2.7GB amd64 pull on architectures where it will not work, which leaves partial images and /opt/project-nomad/ debris that confuse first-time users. Reported in #782. --- install/install_nomad.sh | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/install/install_nomad.sh b/install/install_nomad.sh index 76ca8075..ced178fe 100644 --- a/install/install_nomad.sh +++ b/install/install_nomad.sh @@ -86,6 +86,21 @@ check_is_debian_based() { echo -e "${GREEN}#${RESET} This script is running on a Debian-based system.\\n" } +check_is_x86_64() { + local arch + arch="$(uname -m)" + if [[ "${arch}" != "x86_64" && "${arch}" != "amd64" ]]; then + echo -e "${YELLOW}#${RESET} WARNING: Detected architecture '${arch}'. NOMAD officially supports x86_64 only.\\n" + echo -e "${YELLOW}#${RESET} ARM64/aarch64 support is tracked in PR #419 and is not yet ready.\\n" + echo -e "${YELLOW}#${RESET} Continuing on an unsupported architecture will likely fail and may leave\\n" + echo -e "${YELLOW}#${RESET} partial Docker images and files behind that you'll need to clean up manually.\\n" + echo -e "${YELLOW}#${RESET} Continuing in 10 seconds... press Ctrl+C now to abort.\\n" + sleep 10 + return + fi + echo -e "${GREEN}#${RESET} Architecture check passed (${arch}).\\n" +} + ensure_dependencies_installed() { local missing_deps=() @@ -539,6 +554,7 @@ success_message() { # Pre-flight checks check_is_debian_based +check_is_x86_64 check_is_bash check_has_sudo ensure_dependencies_installed From 3bacd14dbd3e2fb1b41a04c7e0f9e42ae6de7696 Mon Sep 17 00:00:00 2001 From: chriscrosstalk <49691103+chriscrosstalk@users.noreply.github.com> Date: Mon, 27 Apr 2026 18:49:51 -0700 Subject: [PATCH 027/108] feat(content-manager): add sortable file size column (#698) Closes #685 Content Manager now surfaces the on-disk size of each ZIM file alongside title/summary, and lets users sort the list by Size or Title. Defaults to Size descending so the largest files are visible first. - ZimService.list() now stats each file and returns size_bytes - Content Manager table adds a formatted Size column (via formatBytes) - Sortable headers for Title and Size with asc/desc toggle --- admin/app/services/zim_service.ts | 16 +++++- admin/inertia/pages/settings/zim/index.tsx | 64 +++++++++++++++++++++- 2 files changed, 77 insertions(+), 3 deletions(-) diff --git a/admin/app/services/zim_service.ts b/admin/app/services/zim_service.ts index c6f427c6..db6b5b77 100644 --- a/admin/app/services/zim_service.ts +++ b/admin/app/services/zim_service.ts @@ -40,7 +40,21 @@ export class ZimService { await ensureDirectoryExists(dirPath) const all = await listDirectoryContents(dirPath) - const files = all.filter((item) => item.name.endsWith('.zim')) + const zimEntries = all.filter((item) => item.name.endsWith('.zim')) + + const files = await Promise.all( + zimEntries.map(async (entry) => { + const filePath = entry.type === 'file' ? entry.key : join(dirPath, entry.name) + const stats = await getFileStatsIfExists(filePath) + return { + ...entry, + title: null, + summary: null, + author: null, + size_bytes: stats ? Number(stats.size) : null, + } + }) + ) return { files, diff --git a/admin/inertia/pages/settings/zim/index.tsx b/admin/inertia/pages/settings/zim/index.tsx index 9d22937f..68ae2b16 100644 --- a/admin/inertia/pages/settings/zim/index.tsx +++ b/admin/inertia/pages/settings/zim/index.tsx @@ -1,5 +1,6 @@ import { Head } from '@inertiajs/react' import { useMutation, useQuery, useQueryClient } from '@tanstack/react-query' +import { useMemo, useState } from 'react' import StyledTable from '~/components/StyledTable' import SettingsLayout from '~/layouts/SettingsLayout' import api from '~/lib/api' @@ -10,11 +11,18 @@ import useServiceInstalledStatus from '~/hooks/useServiceInstalledStatus' import Alert from '~/components/Alert' import { ZimFileWithMetadata } from '../../../../types/zim' import { SERVICE_NAMES } from '../../../../constants/service_names' +import { formatBytes } from '~/lib/util' +import { IconArrowDown, IconArrowUp, IconArrowsSort } from '@tabler/icons-react' + +type SortKey = 'name' | 'size' +type SortDirection = 'asc' | 'desc' export default function ZimPage() { const queryClient = useQueryClient() const { openModal, closeAllModals } = useModals() const { isInstalled } = useServiceInstalledStatus(SERVICE_NAMES.KIWIX) + const [sortKey, setSortKey] = useState('size') + const [sortDirection, setSortDirection] = useState('desc') const { data, isLoading } = useQuery({ queryKey: ['zim-files'], queryFn: getFiles, @@ -25,6 +33,49 @@ export default function ZimPage() { return res.data.files } + const sortedData = useMemo(() => { + if (!data) return [] + const copy = [...data] + copy.sort((a, b) => { + let cmp = 0 + if (sortKey === 'size') { + const aSize = a.size_bytes ?? 0 + const bSize = b.size_bytes ?? 0 + cmp = aSize - bSize + } else { + const aName = (a.title || a.name).toLowerCase() + const bName = (b.title || b.name).toLowerCase() + cmp = aName.localeCompare(bName) + } + return sortDirection === 'asc' ? cmp : -cmp + }) + return copy + }, [data, sortKey, sortDirection]) + + function toggleSort(key: SortKey) { + if (sortKey === key) { + setSortDirection((d) => (d === 'asc' ? 'desc' : 'asc')) + } else { + setSortKey(key) + setSortDirection(key === 'size' ? 'desc' : 'asc') + } + } + + function renderSortHeader(label: string, key: SortKey) { + const active = sortKey === key + const Icon = !active ? IconArrowsSort : sortDirection === 'asc' ? IconArrowUp : IconArrowDown + return ( + + ) + } + async function confirmDeleteFile(file: ZimFileWithMetadata) { openModal( ( {record.title || record.name} @@ -99,6 +150,15 @@ export default function ZimPage() { ), }, + { + accessor: 'size_bytes', + title: renderSortHeader('Size', 'size'), + render: (record) => ( + + {record.size_bytes ? formatBytes(record.size_bytes, 1) : '—'} + + ), + }, { accessor: 'actions', title: 'Actions', @@ -117,7 +177,7 @@ export default function ZimPage() { ), }, ]} - data={data || []} + data={sortedData} />
From 00b4b26224b0bb47cc850d16d10af87f5e3f3d3c Mon Sep 17 00:00:00 2001 From: chriscrosstalk <49691103+chriscrosstalk@users.noreply.github.com> Date: Mon, 27 Apr 2026 19:00:31 -0700 Subject: [PATCH 028/108] fix(API): skip compression for Server-Sent Events (#798) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(stream): skip compression for Server-Sent Events The global compression middleware (added in v1.31.0-rc.2) buffers response writes to determine encoding, which collapses per-token streaming into a single block delivered after generation completes. This broke the AI chat streaming UX from v1.31.0-rc.2 onward — text no longer appears progressively as the model generates it, only at the end. Adds a filter to compression() that returns false when the response Content-Type is text/event-stream. Other responses still go through the default compression filter (compressible types are still compressed; e.g. text/html via Brotli). Reproduced on NOMAD3 v1.31.1: before fix, all SSE chunks for a 1B model arrive within 10ms of each other after the model finishes. After fix, tokens arrive at ~150ms intervals as they're generated on a 12B model, with no Content-Encoding header on the SSE response. Verified on the same host that /home still returns Content-Encoding: br for HTML responses. Closes #781. Reported and bisected by @toasterking (works in v1.31.0-rc.1, broken from v1.31.0-rc.2 onward). * fix(stream): use any for filter params to match existing as-any pattern The compression library types its filter as (req: Request, res: Response) expecting Express types, but AdonisJS passes raw IncomingMessage/ServerResponse which is why the surrounding middleware uses `as any` casts at the call site. The IncomingMessage/ServerResponse types I added are runtime-correct but fail tsc against the library's declared types. Drop the typed import in favor of `any` parameters, which matches how the existing `compress(request.request as any, response.response as any, ...)` call resolves the same mismatch. --- admin/app/middleware/compression_middleware.ts | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/admin/app/middleware/compression_middleware.ts b/admin/app/middleware/compression_middleware.ts index 0661ac7f..9c114116 100644 --- a/admin/app/middleware/compression_middleware.ts +++ b/admin/app/middleware/compression_middleware.ts @@ -3,7 +3,21 @@ import type { HttpContext } from '@adonisjs/core/http' import type { NextFn } from '@adonisjs/core/types/http' import compression from 'compression' -const compress = env.get('DISABLE_COMPRESSION') ? null : compression() +// Skip compression for Server-Sent Events. The compression library buffers +// response writes to determine encoding, which collapses per-token streaming +// into a single block delivered after generation completes (regression in +// v1.31.0-rc.2, reported in #781 by @toasterking). +const compress = env.get('DISABLE_COMPRESSION') + ? null + : compression({ + filter: (req: any, res: any) => { + const contentType = res.getHeader('Content-Type') + if (typeof contentType === 'string' && contentType.includes('text/event-stream')) { + return false + } + return compression.filter(req, res) + }, + }) export default class CompressionMiddleware { async handle({ request, response }: HttpContext, next: NextFn) { From b194dfa136aee0bfadcc4e5eb011de077de4c8ef Mon Sep 17 00:00:00 2001 From: chriscrosstalk <49691103+chriscrosstalk@users.noreply.github.com> Date: Mon, 27 Apr 2026 21:43:10 -0700 Subject: [PATCH 029/108] fix(RAG): pass num_ctx and truncate to Ollama embed call (#763) Some Ollama installs ship nomic-embed-text:v1.5 with the embedding model's default num_ctx=2048, which the RAG chunker (sized for ~1500 tokens of estimated content with ratio=2 chars/token) can exceed on dense PDFs. The result is `400 the input length exceeds the context length` from /api/embed, which then hits the OpenAI-compatible fallback (which also errors), and surfaces as a BadRequestError. Pass options.num_ctx=8192 (nomic-embed-text v1.5's RoPE-extrapolated max) and truncate=true (silent truncation safety net) on every embed call so we don't depend on the local modelfile defaults. Reported on #756 by @NC4WD; same root cause as #369 and #670 which were closed without an actual fix. Co-authored-by: Claude Opus 4.7 (1M context) --- admin/app/services/ollama_service.ts | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/admin/app/services/ollama_service.ts b/admin/app/services/ollama_service.ts index 27f5cac7..fe0cb1c8 100644 --- a/admin/app/services/ollama_service.ts +++ b/admin/app/services/ollama_service.ts @@ -480,10 +480,21 @@ export class OllamaService { } try { - // Prefer Ollama native endpoint (supports batch input natively) + // Prefer Ollama native endpoint (supports batch input natively). + // Pass num_ctx explicitly so we don't depend on the embedding model's + // modelfile defaults. Some installs ship nomic-embed-text:v1.5 with + // num_ctx=2048, which our chunker (sized for ~1500 tokens) can exceed + // on dense content, causing "input length exceeds context length" errors. + // truncate:true is a runtime safety net for any chunk that still overshoots. + // 8192 matches nomic-embed-text:v1.5's RoPE-extrapolated max. const response = await axios.post( `${this.baseUrl}/api/embed`, - { model, input }, + { + model, + input, + truncate: true, + options: { num_ctx: 8192 }, + }, { timeout: 60000 } ) // Some backends (e.g. LM Studio) return HTTP 200 for unknown endpoints with an incompatible From 269c7ce695c0d9b5d15a1ad95af82a790e45251b Mon Sep 17 00:00:00 2001 From: John Scherer Date: Tue, 28 Apr 2026 00:11:19 -0500 Subject: [PATCH 030/108] fix(API): accept notes, marker_type, and position on markers endpoints (#770) The VineJS validators in createMarker and updateMarker silently dropped fields not in their schema. The MapMarker model and DB include notes and marker_type, and GET responses return them, but POST and PATCH would not persist them. updateMarker additionally did not accept latitude/longitude, so markers could not be repositioned via the API after creation. - Add notes and marker_type to both validators and model assignments. - Add latitude/longitude to the update validator. - Add coordinate range validation on both endpoints. Closes #768 --- admin/app/controllers/maps_controller.ts | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/admin/app/controllers/maps_controller.ts b/admin/app/controllers/maps_controller.ts index f5fb0f3d..dd93a8b1 100644 --- a/admin/app/controllers/maps_controller.ts +++ b/admin/app/controllers/maps_controller.ts @@ -137,9 +137,11 @@ export default class MapsController { vine.compile( vine.object({ name: vine.string().trim().minLength(1).maxLength(255), - longitude: vine.number(), - latitude: vine.number(), + longitude: vine.number().min(-180).max(180), + latitude: vine.number().min(-90).max(90), color: vine.string().trim().maxLength(20).optional(), + notes: vine.string().trim().nullable().optional(), + marker_type: vine.string().trim().maxLength(20).optional(), }) ) ) @@ -148,6 +150,8 @@ export default class MapsController { longitude: payload.longitude, latitude: payload.latitude, color: payload.color ?? 'orange', + notes: payload.notes ?? null, + marker_type: payload.marker_type ?? 'pin', }) return marker } @@ -163,11 +167,19 @@ export default class MapsController { vine.object({ name: vine.string().trim().minLength(1).maxLength(255).optional(), color: vine.string().trim().maxLength(20).optional(), + longitude: vine.number().min(-180).max(180).optional(), + latitude: vine.number().min(-90).max(90).optional(), + notes: vine.string().trim().nullable().optional(), + marker_type: vine.string().trim().maxLength(20).optional(), }) ) ) if (payload.name !== undefined) marker.name = payload.name if (payload.color !== undefined) marker.color = payload.color + if (payload.longitude !== undefined) marker.longitude = payload.longitude + if (payload.latitude !== undefined) marker.latitude = payload.latitude + if (payload.notes !== undefined) marker.notes = payload.notes + if (payload.marker_type !== undefined) marker.marker_type = payload.marker_type await marker.save() return marker } From fe57d598682c39c50880278ed0b71434030fa35d Mon Sep 17 00:00:00 2001 From: Kenneth Brewer Date: Tue, 28 Apr 2026 01:21:06 -0400 Subject: [PATCH 031/108] docs: add map markers to API reference (#783) Co-authored-by: Kenneth Brewer --- admin/docs/api-reference.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/admin/docs/api-reference.md b/admin/docs/api-reference.md index 5cf126fc..928b1e38 100644 --- a/admin/docs/api-reference.md +++ b/admin/docs/api-reference.md @@ -148,6 +148,15 @@ ZIM files provide offline Wikipedia, books, and other content via Kiwix. | POST | `/api/maps/download-collection` | Download an entire collection by slug (async) | | DELETE | `/api/maps/:filename` | Delete a local map file | +### Map Markers + +| Method | Path | Description | +|--------|------|-------------| +| GET | `/api/maps/markers` | List map markers | +| POST | `/api/maps/markers` | Add map marker (body: {"name": "Test Marker", "longitude": 0.0, "latitude": 0.0, "color": "yellow", "marker_type": "pin"} ) | +| PATCH | `/api/maps/markers/{id}` | Update a map marker (body: {"name": "Test Marker", "longitude": 0.0, "latitude": 0.0, "color": "yellow", "marker_type": "pin"} ) fields that don't change can be omitted| +| DELETE | `/api/maps/markers/{id}` | Delete a map marker | + --- ## Downloads From cc789c1863d9d5bc23ebdb7e3af0d00a0ed5a736 Mon Sep 17 00:00:00 2001 From: Henry Estela Date: Tue, 28 Apr 2026 05:26:46 +0000 Subject: [PATCH 032/108] fix(RAG): add start button in kb modal and ensure restart policy exists (#700) Adds a check to RAG health to make sure nomad_qdrant is online, if not then the user will be blocked from clicking any buttons in the KB modal until they click the start qdrant button and let the container start There is a new file qdrant_restart_policy_provider.ts, which tries to ensure that the restart policy always exists for the nomad_qdrant container even though the policy should have been there when the container is created. --- admin/adonisrc.ts | 1 + admin/app/controllers/rag_controller.ts | 5 ++ admin/app/services/rag_service.ts | 23 ++++++- .../components/chat/KnowledgeBaseModal.tsx | 49 +++++++++++++-- admin/inertia/lib/api.ts | 7 +++ .../qdrant_restart_policy_provider.ts | 62 +++++++++++++++++++ admin/start/routes.ts | 1 + 7 files changed, 142 insertions(+), 6 deletions(-) create mode 100644 admin/providers/qdrant_restart_policy_provider.ts diff --git a/admin/adonisrc.ts b/admin/adonisrc.ts index 37046d27..a091ce2d 100644 --- a/admin/adonisrc.ts +++ b/admin/adonisrc.ts @@ -55,6 +55,7 @@ export default defineConfig({ () => import('@adonisjs/transmit/transmit_provider'), () => import('#providers/map_static_provider'), () => import('#providers/kiwix_migration_provider'), + () => import('#providers/qdrant_restart_policy_provider'), ], /* diff --git a/admin/app/controllers/rag_controller.ts b/admin/app/controllers/rag_controller.ts index 149ba7e4..c836393a 100644 --- a/admin/app/controllers/rag_controller.ts +++ b/admin/app/controllers/rag_controller.ts @@ -97,4 +97,9 @@ export default class RagController { return response.status(500).json({ error: 'Error scanning and syncing storage' }) } } + + public async health({ response }: HttpContext) { + const result = await this.ragService.checkQdrantHealth() + return response.status(200).json(result) + } } diff --git a/admin/app/services/rag_service.ts b/admin/app/services/rag_service.ts index 81145f89..bd5371d1 100644 --- a/admin/app/services/rag_service.ts +++ b/admin/app/services/rag_service.ts @@ -52,14 +52,33 @@ export class RagService { this.qdrantInitPromise = (async () => { const qdrantUrl = await this.dockerService.getServiceURL(SERVICE_NAMES.QDRANT) if (!qdrantUrl) { - throw new Error('Qdrant service is not installed or running.') + throw new Error('Qdrant vector database is offline. Restart the AI Assistant service in Settings to restore the Knowledge Base.') } this.qdrant = new QdrantClient({ url: qdrantUrl }) - })() + })().catch((err) => { + this.qdrantInitPromise = null + this.qdrant = null + throw err + }) } return this.qdrantInitPromise } + public async checkQdrantHealth(): Promise<{ online: boolean; message?: string }> { + try { + await this._ensureDependencies() + await this.qdrant!.getCollections() + return { online: true } + } catch { + this.qdrant = null + this.qdrantInitPromise = null + return { + online: false, + message: 'Qdrant vector database is offline. Restart the AI Assistant service in Settings to restore the Knowledge Base.', + } + } + } + private async _ensureDependencies() { if (!this.qdrant) { await this._initializeQdrantClient() diff --git a/admin/inertia/components/chat/KnowledgeBaseModal.tsx b/admin/inertia/components/chat/KnowledgeBaseModal.tsx index e77a0c93..62303984 100644 --- a/admin/inertia/components/chat/KnowledgeBaseModal.tsx +++ b/admin/inertia/components/chat/KnowledgeBaseModal.tsx @@ -1,5 +1,5 @@ import { useMutation, useQuery, useQueryClient } from '@tanstack/react-query' -import { useRef, useState } from 'react' +import { useEffect, useRef, useState } from 'react' import FileUploader from '~/components/file-uploader' import StyledButton from '~/components/StyledButton' import StyledSectionHeader from '~/components/StyledSectionHeader' @@ -10,6 +10,7 @@ import { IconX } from '@tabler/icons-react' import { useModals } from '~/context/ModalContext' import StyledModal from '../StyledModal' import ActiveEmbedJobs from '~/components/ActiveEmbedJobs' +import { SERVICE_NAMES } from '../../../constants/service_names' interface KnowledgeBaseModalProps { aiAssistantName?: string @@ -30,6 +31,19 @@ export default function KnowledgeBaseModal({ aiAssistantName = "AI Assistant", o const { openModal, closeModal } = useModals() const queryClient = useQueryClient() + const [isStartingQdrant, setIsStartingQdrant] = useState(false) + + const { data: healthStatus } = useQuery({ + queryKey: ['qdrantHealth'], + queryFn: () => api.checkRAGHealth(), + refetchInterval: isStartingQdrant ? 3_000 : 30_000, + }) + const qdrantOffline = healthStatus?.online === false + + useEffect(() => { + if (!qdrantOffline) setIsStartingQdrant(false) + }, [qdrantOffline]) + const { data: storedFiles = [], isLoading: isLoadingFiles } = useQuery({ queryKey: ['storedFiles'], queryFn: () => api.getStoredRAGFiles(), @@ -64,6 +78,17 @@ export default function KnowledgeBaseModal({ aiAssistantName = "AI Assistant", o }, }) + const startQdrantMutation = useMutation({ + mutationFn: () => api.affectService(SERVICE_NAMES.QDRANT, 'start'), + onSuccess: () => { + setIsStartingQdrant(true) + queryClient.invalidateQueries({ queryKey: ['qdrantHealth'] }) + }, + onError: (error: any) => { + addNotification({ type: 'error', message: error?.message || 'Failed to start Qdrant.' }) + }, + }) + const syncMutation = useMutation({ mutationFn: () => api.syncRAGStorage(), onSuccess: (data) => { @@ -149,6 +174,22 @@ export default function KnowledgeBaseModal({ aiAssistantName = "AI Assistant", o
+ {qdrantOffline && ( +
+ + Knowledge Base unavailable: The Qdrant vector database is offline. + + startQdrantMutation.mutate()} + loading={startQdrantMutation.isPending || isStartingQdrant} + disabled={startQdrantMutation.isPending || isStartingQdrant} + > + {isStartingQdrant ? 'Starting…' : 'Start Qdrant'} + +
+ )}
Upload @@ -236,7 +277,7 @@ export default function KnowledgeBaseModal({ aiAssistantName = "AI Assistant", o icon="IconTrash" onClick={() => cleanupFailedMutation.mutate()} loading={cleanupFailedMutation.isPending} - disabled={cleanupFailedMutation.isPending} + disabled={cleanupFailedMutation.isPending || qdrantOffline} > Clean Up Failed @@ -252,7 +293,7 @@ export default function KnowledgeBaseModal({ aiAssistantName = "AI Assistant", o size="md" icon='IconRefresh' onClick={handleConfirmSync} - disabled={syncMutation.isPending || isUploading} + disabled={syncMutation.isPending || isUploading || qdrantOffline} loading={syncMutation.isPending || isUploading} > Sync Storage diff --git a/admin/inertia/lib/api.ts b/admin/inertia/lib/api.ts index dc1c7ed3..0df95ae0 100644 --- a/admin/inertia/lib/api.ts +++ b/admin/inertia/lib/api.ts @@ -451,6 +451,13 @@ class API { })() } + async checkRAGHealth() { + return catchInternal(async () => { + const response = await this.client.get<{ online: boolean; message?: string }>('/rag/health') + return response.data + })() + } + async getStoredRAGFiles() { return catchInternal(async () => { const response = await this.client.get<{ files: string[] }>('/rag/files') diff --git a/admin/providers/qdrant_restart_policy_provider.ts b/admin/providers/qdrant_restart_policy_provider.ts new file mode 100644 index 00000000..b9092429 --- /dev/null +++ b/admin/providers/qdrant_restart_policy_provider.ts @@ -0,0 +1,62 @@ +import logger from '@adonisjs/core/services/logger' +import type { ApplicationService } from '@adonisjs/core/types' + +/** + * Ensures the nomad_qdrant container has the `unless-stopped` restart policy. + * + * Existing installations may have been created before this policy was enforced + * in the service seeder. Docker allows updating a container's restart policy + * without recreating it via the container.update() API. + * + * This provider runs once on every admin startup. If the policy is already + * correct, the check is a no-op. + */ +export default class QdrantRestartPolicyProvider { + constructor(protected app: ApplicationService) {} + + async boot() { + if (this.app.getEnvironment() !== 'web') return + + setImmediate(async () => { + try { + const Service = (await import('#models/service')).default + const { SERVICE_NAMES } = await import('../constants/service_names.js') + const Docker = (await import('dockerode')).default + + const qdrantService = await Service.query() + .where('service_name', SERVICE_NAMES.QDRANT) + .first() + + if (!qdrantService?.installed) { + logger.info('[QdrantRestartPolicyProvider] Qdrant not installed — skipping restart policy check.') + return + } + + const docker = new Docker({ socketPath: '/var/run/docker.sock' }) + const containers = await docker.listContainers({ all: true }) + const containerInfo = containers.find((c) => c.Names.includes(`/${SERVICE_NAMES.QDRANT}`)) + + if (!containerInfo) { + logger.warn('[QdrantRestartPolicyProvider] Qdrant container not found — skipping restart policy check.') + return + } + + const container = docker.getContainer(containerInfo.Id) + const inspected = await container.inspect() + const currentPolicy = inspected.HostConfig?.RestartPolicy?.Name + + if (currentPolicy === 'unless-stopped') { + logger.info('[QdrantRestartPolicyProvider] Qdrant already has unless-stopped restart policy — no update needed.') + return + } + + logger.info(`[QdrantRestartPolicyProvider] Qdrant restart policy is "${currentPolicy ?? 'none'}" — updating to unless-stopped.`) + await container.update({ RestartPolicy: { Name: 'unless-stopped', MaximumRetryCount: 0 } }) + logger.info('[QdrantRestartPolicyProvider] Qdrant restart policy updated successfully.') + } catch (err: any) { + logger.error(`[QdrantRestartPolicyProvider] Failed to update Qdrant restart policy: ${err.message}`) + // Non-fatal: the container will still run, just without auto-restart on crash. + } + }) + } +} diff --git a/admin/start/routes.ts b/admin/start/routes.ts index d2011742..07ee6b96 100644 --- a/admin/start/routes.ts +++ b/admin/start/routes.ts @@ -143,6 +143,7 @@ router router.delete('/failed-jobs', [RagController, 'cleanupFailedJobs']) router.get('/job-status', [RagController, 'getJobStatus']) router.post('/sync', [RagController, 'scanAndSync']) + router.get('/health', [RagController, 'health']) }) .prefix('/api/rag') From 322087c1b79317bb04b1d1a97f4e0b65467b395d Mon Sep 17 00:00:00 2001 From: Ryanba <92616678+Gujiassh@users.noreply.github.com> Date: Wed, 29 Apr 2026 11:55:25 +0800 Subject: [PATCH 033/108] fix(UI): improve global map banner display logic (#702) --- admin/inertia/lib/global_map_banner.ts | 10 ++++++ admin/inertia/pages/settings/maps.tsx | 20 +++++++++++- admin/tests/unit/global_map_banner.spec.ts | 37 ++++++++++++++++++++++ 3 files changed, 66 insertions(+), 1 deletion(-) create mode 100644 admin/inertia/lib/global_map_banner.ts create mode 100644 admin/tests/unit/global_map_banner.spec.ts diff --git a/admin/inertia/lib/global_map_banner.ts b/admin/inertia/lib/global_map_banner.ts new file mode 100644 index 00000000..1693dacf --- /dev/null +++ b/admin/inertia/lib/global_map_banner.ts @@ -0,0 +1,10 @@ +export function hasDownloadedGlobalMap( + globalMapKey: string | null | undefined, + storedMapFiles: Array<{ name: string }> +): boolean { + if (!globalMapKey) { + return false + } + + return storedMapFiles.some((file) => file.name === globalMapKey || /^\d{8}\.pmtiles$/.test(file.name)) +} diff --git a/admin/inertia/pages/settings/maps.tsx b/admin/inertia/pages/settings/maps.tsx index 0212931b..da97bb1f 100644 --- a/admin/inertia/pages/settings/maps.tsx +++ b/admin/inertia/pages/settings/maps.tsx @@ -17,6 +17,7 @@ import type { CollectionWithStatus } from '../../../types/collections' import ActiveDownloads from '~/components/ActiveDownloads' import Alert from '~/components/Alert' import { formatBytes } from '~/lib/util' +import { hasDownloadedGlobalMap } from '~/lib/global_map_banner' const CURATED_COLLECTIONS_KEY = 'curated-map-collections' const GLOBAL_MAP_INFO_KEY = 'global-map-info' @@ -45,6 +46,7 @@ export default function MapsManager(props: { queryFn: () => api.getGlobalMapInfo(), refetchOnWindowFocus: false, }) + const globalMapAlreadyDownloaded = hasDownloadedGlobalMap(globalMapInfo?.key, props.maps.regionFiles) const downloadGlobalMap = useMutation({ mutationFn: () => api.downloadGlobalMap(), @@ -251,7 +253,23 @@ export default function MapsManager(props: { }} /> )} - {globalMapInfo && ( + {globalMapInfo && globalMapAlreadyDownloaded && ( + confirmGlobalMapDownload(), + }} + /> + )} + {globalMapInfo && !globalMapAlreadyDownloaded && ( { + assert.equal( + hasDownloadedGlobalMap('20260402.pmtiles', [ + { name: '20260402.pmtiles' }, + { name: 'california.pmtiles' }, + ]), + true + ) +}) + +test('returns false when the global map key is missing', () => { + assert.equal( + hasDownloadedGlobalMap('20260402.pmtiles', [ + { name: 'california.pmtiles' }, + ]), + false + ) +}) + +test('returns true when an older global map build is already on disk', () => { + assert.equal( + hasDownloadedGlobalMap('20260402.pmtiles', [ + { name: '20260315.pmtiles' }, + { name: 'california.pmtiles' }, + ]), + true + ) +}) + +test('returns false when there is no global map info', () => { + assert.equal(hasDownloadedGlobalMap(undefined, [{ name: '20260402.pmtiles' }]), false) +}) From 5924056502f098a6c6271edd6970689e04a44377 Mon Sep 17 00:00:00 2001 From: chriscrosstalk <49691103+chriscrosstalk@users.noreply.github.com> Date: Tue, 28 Apr 2026 21:53:56 -0700 Subject: [PATCH 034/108] feat(AI): improved AMD GPU acceleration for Ollama via ROCm + HSA override (#804) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat(AI): re-enable AMD GPU acceleration for Ollama via ROCm + HSA override Re-enables AMD GPU support that was disabled in 77f1868 pending validation of the ROCm image and device discovery. Validation done 2026-04-28 on a Minisforum UM890 Pro (Ryzen 9 PRO 8945HS + Radeon 780M iGPU) — Ollama correctly offloaded all model layers to the iGPU when the container was started with /dev/kfd + /dev/dri passthrough and HSA_OVERRIDE_GFX_VERSION=11.0.0. On llama3.2:1b, GPU inference ran at 51.83 tok/s vs 33.16 tok/s on CPU (same hardware, same prompt) — a 1.56x speedup confirmed by Ollama logs showing "load_tensors: offloaded 17/17 layers to GPU". Changes ------- docker_service.ts - Restore _discoverAMDDevices() (simplified — pass /dev/dri as a directory entry, mirroring `docker run --device /dev/dri` behavior, instead of the prior brittle hardcoded card0/renderD128 fallback that broke on systems where the AMD GPU enumerates as card1+). - Restore the AMD branch in _createContainer(): - Switches Ollama image to ollama/ollama:rocm - Mounts /dev/kfd + /dev/dri via Devices - Sets HSA_OVERRIDE_GFX_VERSION=11.0.0 (required for unsupported-but-RDNA3 iGPUs like gfx1103; harmless on supported discrete cards) - KV opt-out via ai.amdGpuAcceleration (default on) - Mirror the AMD branch in updateContainer(): - Lifted GPU detection above docker.pull() so AMD updates pull :rocm rather than the standard :targetVersion tag (per-version ROCm tags aren't always published) - Replaces stale HSA_OVERRIDE in the inspect-captured env on update, so containers built before this PR pick up the current value system_service.ts - New getOllamaInferenceComputeFromLogs() — parses Ollama startup log line "msg=\"inference compute\" ... library=CUDA|ROCm ..." which Ollama emits for both NVIDIA and AMD. Catches silent CPU fallback (e.g. NVML death after update, or HSA_OVERRIDE failure) that the prior nvidia-smi exec probe couldn't detect. - gpuHealth refactored to use log parsing as the primary probe for both vendors, with nvidia-smi exec retained as the NVIDIA-only secondary path for hardware enrichment when log parsing has no startup line yet. - AMD path uses gpu.type KV value (persisted by DockerService._detectGPUType) + ai.amdGpuAcceleration opt-out to determine hasRocmRuntime. types/system.ts - GpuHealthStatus extended additively: hasRocmRuntime + optional gpuVendor. types/kv_store.ts - New ai.amdGpuAcceleration boolean (default-on). settings/models.tsx, settings/system.tsx - passthrough_failed banner copy now reads vendor from gpuHealth.gpuVendor ("an AMD GPU" vs "an NVIDIA GPU"). Same Fix button hits the same force-reinstall endpoint, which now configures AMD correctly. install_nomad.sh - AMD detection in verify_gpu_setup() upgraded from a strict-positive "ROCm not currently available" message to "ROCm acceleration will be configured automatically." Also tightens the lspci match to display controller classes (avoids false positives from AMD CPU host bridges, matching the same fix already in DockerService._detectGPUType). Auto-remediation ---------------- Issue #755 proposes auto-remediation when gpuHealth.status flips to passthrough_failed (today the user has to click "Fix: Reinstall AI Assistant"). When that PR lands, AMD coverage falls out for free since this PR uses the same passthrough_failed status code via the shared gpuHealth machinery — #755's guard will need to flip from hasNvidiaRuntime === true to (hasNvidiaRuntime || hasRocmRuntime). Closes #124 (AMD GPU support). Co-Authored-By: Claude Opus 4.7 (1M context) * fix(AI): detect AMD GPU presence inside admin container via marker file The admin container doesn't have lspci installed, and AMD GPUs don't register a Docker runtime the way NVIDIA does — so DockerService._detectGPUType() and SystemService.gpuHealth had no way to know an AMD GPU was present. The previous implementation fell through to lspci, which silently failed inside the admin container, leaving gpu.type unset and gpuHealth stuck at 'no_gpu' even on systems with an AMD GPU. (NVIDIA worked because Docker registers the nvidia runtime, which is reachable via dockerInfo.Runtimes from any container.) Discovered while testing the AMD acceleration patch on a Minisforum UM890 Pro: the AMD branch in _createContainer() never fired because _detectGPUType() returned 'none' even on a host with a working /dev/kfd. Fix --- install_nomad.sh writes the host-detected GPU type ('nvidia' | 'amd') to a marker file in the storage volume the admin container already bind-mounts: /opt/project-nomad/storage/.nomad-gpu-type → /app/storage/.nomad-gpu-type DockerService._detectGPUType() reads the marker as a secondary probe (after the Docker runtime check) — covers AMD detection from inside the container without requiring lspci or a /dev bind mount. SystemService falls back to the marker file when KV gpu.type is empty so the System page reflects AMD presence even before the user installs AI Assistant for the first time. (Without this, the page would say 'no_gpu' until Ollama was installed, even on hosts with an AMD GPU detected at install time.) Verified on NOMAD6 (UM890 Pro, Ubuntu 24.04, 780M iGPU): with the marker file in place and admin restarted, the patch's AMD branch fires correctly on Force Reinstall AI Assistant. Resulting nomad_ollama runs ollama/ollama:rocm with /dev/kfd + /dev/dri passthrough and HSA_OVERRIDE_GFX_VERSION=11.0.0; Ollama logs show 'library=ROCm compute=gfx1100 ... type=iGPU'. NOMAD's in-product benchmark on the same hardware climbed from 33.8 tok/s (CPU) to 57.3 tok/s (GPU) — a 1.69x speedup, with TTFT dropping from 148ms to 66ms. Migration for existing AMD installs ----------------------------------- Users on an existing NOMAD install with an AMD GPU have no marker file (the install script wrote it on a fresh install). Two paths get them on the GPU: 1. Re-run install_nomad.sh — writes the marker, no other side effects 2. Manually: echo amd | sudo tee /opt/project-nomad/storage/.nomad-gpu-type Either then triggers AMD detection on the next AI Assistant install/reinstall. Co-Authored-By: Claude Opus 4.7 (1M context) * fix(AI): pull ollama/ollama:rocm separately when AMD branch overrides image The pull-if-missing logic in _createContainer ran against service.container_image (the DB-pinned tag, e.g. ollama/ollama:0.18.2). The AMD branch then overrode finalImage to ollama/ollama:rocm — but if that image wasn't already local, the container creation step failed with "no such image: ollama/ollama:rocm". Caught while validating on NOMAD2 (Ryzen AI 9 HX 370 + Radeon 890M / RDNA 3.5): the prior end-to-end test on NOMAD6 had silently passed because the rocm image was already pulled there from an earlier sidecar test, masking the bug. Fix: inside the AMD branch, after setting finalImage to ollama/ollama:rocm, run a parallel _checkImageExists + docker.pull dance for the new tag. Also confirmed via this validation: the same HSA_OVERRIDE_GFX_VERSION=11.0.0 override works on the 890M (gfx1150 / RDNA 3.5) — Ollama logs report 'library=ROCm compute=gfx1100 description="AMD Radeon 890M Graphics"' and inference runs at 51.68 tok/s (matching the existing X1 Pro published tile of 51.7 tok/s on the same hardware class). RDNA 3 (780M, gfx1103) and RDNA 3.5 (890M, gfx1150) both use the same override successfully. Co-Authored-By: Claude Opus 4.7 (1M context) * build(Dockerfile): include pciutils for lspci gpu detection fallback --------- Co-authored-by: Claude Opus 4.7 (1M context) Co-authored-by: Jake Turner --- Dockerfile | 9 +- admin/app/services/docker_service.ts | 257 ++++++++++++++---------- admin/app/services/system_service.ts | 187 ++++++++++++++--- admin/inertia/pages/settings/models.tsx | 2 +- admin/inertia/pages/settings/system.tsx | 2 +- admin/types/kv_store.ts | 1 + admin/types/system.ts | 2 + install/install_nomad.sh | 27 ++- 8 files changed, 343 insertions(+), 144 deletions(-) diff --git a/Dockerfile b/Dockerfile index 03acaa9d..8850e237 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,7 +1,14 @@ FROM node:22-slim AS base # Install bash & curl for entrypoint script compatibility, graphicsmagick for pdf2pic, and vips-dev & build-base for sharp -RUN apt-get update && apt-get install -y bash curl graphicsmagick libvips-dev build-essential +RUN apt-get update && apt-get install -y \ + bash \ + curl \ + graphicsmagick \ + libvips-dev \ + build-essential \ + pciutils \ + && rm -rf /var/lib/apt/lists/* # All deps stage FROM base AS deps diff --git a/admin/app/services/docker_service.ts b/admin/app/services/docker_service.ts index 714cd112..7501e78c 100644 --- a/admin/app/services/docker_service.ts +++ b/admin/app/services/docker_service.ts @@ -10,7 +10,7 @@ import { KiwixLibraryService } from './kiwix_library_service.js' import { SERVICE_NAMES } from '../../constants/service_names.js' import { exec } from 'child_process' import { promisify } from 'util' -// import { readdir } from 'fs/promises' +import { readFile } from 'node:fs/promises' import KVStore from '#models/kv_store' import { BROADCAST_CHANNELS } from '../../constants/broadcast.js' import { KIWIX_LIBRARY_CMD } from '../../constants/kiwix.js' @@ -500,6 +500,7 @@ export class DockerService { // GPU-aware configuration for Ollama let finalImage = service.container_image let gpuHostConfig = containerConfig?.HostConfig || {} + let amdGpuConfigured = false if (service.service_name === SERVICE_NAMES.OLLAMA) { const gpuResult = await this._detectGPUType() @@ -523,16 +524,51 @@ export class DockerService { ], } } else if (gpuResult.type === 'amd') { - this._broadcast( - service.service_name, - 'gpu-config', - `AMD GPU detected. ROCm GPU acceleration is not yet supported in this version — proceeding with CPU-only configuration. GPU support for AMD will be available in a future update.` - ) - logger.warn('[DockerService] AMD GPU detected but ROCm support is not yet enabled. Using CPU-only configuration.') - // TODO: Re-enable AMD GPU support once ROCm image and device discovery are validated. - // When re-enabling: - // 1. Switch image to 'ollama/ollama:rocm' - // 2. Restore _discoverAMDDevices() to map /dev/kfd and /dev/dri/* into the container + // AMD acceleration is opt-out via the 'ai.amdGpuAcceleration' KV key (default-on). + // Per memory feedback: KV values can be string or boolean — coerce explicitly. + const amdEnabledRaw = await KVStore.getValue('ai.amdGpuAcceleration') + const amdAccelerationEnabled = String(amdEnabledRaw) !== 'false' + + if (amdAccelerationEnabled) { + this._broadcast( + service.service_name, + 'gpu-config', + `AMD GPU detected. Using ROCm image with /dev/kfd and /dev/dri passthrough...` + ) + + finalImage = 'ollama/ollama:rocm' + + // The pull-if-missing earlier in this function used service.container_image + // (the DB-pinned tag, e.g. ollama/ollama:0.18.2). The AMD branch overrides + // to a different tag — so we need to pull :rocm separately if it's not local. + const rocmImageExists = await this._checkImageExists(finalImage) + if (!rocmImageExists) { + this._broadcast( + service.service_name, + 'pulling', + `Pulling Docker image ${finalImage}...` + ) + const rocmPullStream = await this.docker.pull(finalImage) + await new Promise((res) => this.docker.modem.followProgress(rocmPullStream, res)) + } + + const amdDevices = await this._discoverAMDDevices() + gpuHostConfig = { + ...gpuHostConfig, + Devices: amdDevices, + } + amdGpuConfigured = true + logger.info( + `[DockerService] Configured ROCm image and ${amdDevices.length} AMD device entries for Ollama` + ) + } else { + this._broadcast( + service.service_name, + 'gpu-config', + `AMD GPU detected but acceleration is disabled via ai.amdGpuAcceleration. Using CPU-only configuration.` + ) + logger.info('[DockerService] AMD GPU acceleration disabled by KV opt-out; using CPU-only configuration.') + } } else if (gpuResult.toolkitMissing) { this._broadcast( service.service_name, @@ -555,6 +591,12 @@ export class DockerService { if (flashAttentionEnabled !== false) { ollamaEnv.push('OLLAMA_FLASH_ATTENTION=1') } + if (amdGpuConfigured) { + // RDNA3 iGPUs (gfx1103: 780M, 880M, 890M, ...) aren't on AMD's official ROCm + // allowlist but work when forced to identify as gfx1100 via HSA_OVERRIDE_GFX_VERSION. + // Harmless on supported discrete cards (gfx1030 RX 6800, etc.) — they ignore the override. + ollamaEnv.push('HSA_OVERRIDE_GFX_VERSION=11.0.0') + } } this._broadcast( @@ -857,7 +899,10 @@ export class DockerService { /** * Detect GPU type and toolkit availability. * Primary: Check Docker runtimes via docker.info() (works from inside containers). - * Fallback: lspci for host-based installs and AMD detection. + * Secondary: Read /app/storage/.nomad-gpu-type written by install_nomad.sh — needed + * for AMD detection because lspci isn't available inside the admin container and + * AMD has no Docker runtime registration to query. + * Fallback: lspci for host-based installs. */ private async _detectGPUType(): Promise<{ type: 'nvidia' | 'amd' | 'none'; toolkitMissing?: boolean }> { try { @@ -874,6 +919,24 @@ export class DockerService { logger.warn(`[DockerService] Could not query Docker info for GPU runtimes: ${error.message}`) } + // Secondary: install_nomad.sh writes the host-detected GPU type to a marker file in + // the storage volume so the admin container (which lacks lspci) can read it. + try { + const marker = (await readFile('/app/storage/.nomad-gpu-type', 'utf8')).trim() + if (marker === 'nvidia') { + // Hardware present but Docker doesn't have nvidia runtime → toolkit missing + logger.warn('[DockerService] NVIDIA GPU recorded in marker file but NVIDIA Container Toolkit is not installed') + return { type: 'none', toolkitMissing: true } + } + if (marker === 'amd') { + logger.info('[DockerService] AMD GPU detected via install-time marker file') + await this._persistGPUType('amd') + return { type: 'amd' } + } + } catch { + // No marker file — fall through to lspci attempt for host-based installs + } + // Fallback: lspci for host-based installs (not available inside Docker) const execAsync = promisify(exec) @@ -937,60 +1000,23 @@ export class DockerService { } /** - * Discover AMD GPU DRI devices dynamically. - * Returns an array of device configurations for Docker. + * Build the Docker Devices array for AMD GPU passthrough. + * + * Returns /dev/kfd (Kernel Fusion Driver, required by ROCm) and /dev/dri (the DRM + * device tree). Passing /dev/dri as a single directory entry mirrors Docker CLI + * --device behavior — the daemon expands it to all child devices (card*, renderD*) + * regardless of how the host enumerates them. This avoids the brittle hardcoded + * fallback (card0/renderD128) the prior implementation used, which was wrong on + * systems where the AMD GPU enumerates as card1+ (e.g. UM890 Pro 780M iGPU). */ - // private async _discoverAMDDevices(): Promise< - // Array<{ PathOnHost: string; PathInContainer: string; CgroupPermissions: string }> - // > { - // try { - // const devices: Array<{ - // PathOnHost: string - // PathInContainer: string - // CgroupPermissions: string - // }> = [] - - // // Always add /dev/kfd (Kernel Fusion Driver) - // devices.push({ - // PathOnHost: '/dev/kfd', - // PathInContainer: '/dev/kfd', - // CgroupPermissions: 'rwm', - // }) - - // // Discover DRI devices in /dev/dri/ - // try { - // const driDevices = await readdir('/dev/dri') - // for (const device of driDevices) { - // const devicePath = `/dev/dri/${device}` - // devices.push({ - // PathOnHost: devicePath, - // PathInContainer: devicePath, - // CgroupPermissions: 'rwm', - // }) - // } - // logger.info( - // `[DockerService] Discovered ${driDevices.length} DRI devices: ${driDevices.join(', ')}` - // ) - // } catch (error) { - // logger.warn(`[DockerService] Could not read /dev/dri directory: ${error.message}`) - // // Fallback to common device names if directory read fails - // const fallbackDevices = ['card0', 'renderD128'] - // for (const device of fallbackDevices) { - // devices.push({ - // PathOnHost: `/dev/dri/${device}`, - // PathInContainer: `/dev/dri/${device}`, - // CgroupPermissions: 'rwm', - // }) - // } - // logger.info(`[DockerService] Using fallback DRI devices: ${fallbackDevices.join(', ')}`) - // } - - // return devices - // } catch (error) { - // logger.error(`[DockerService] Error discovering AMD devices: ${error.message}`) - // return [] - // } - // } + private async _discoverAMDDevices(): Promise< + Array<{ PathOnHost: string; PathInContainer: string; CgroupPermissions: string }> + > { + return [ + { PathOnHost: '/dev/kfd', PathInContainer: '/dev/kfd', CgroupPermissions: 'rwm' }, + { PathOnHost: '/dev/dri', PathInContainer: '/dev/dri', CgroupPermissions: 'rwm' }, + ] + } /** * Update a service container to a new image version while preserving volumes and data. @@ -1014,12 +1040,60 @@ export class DockerService { this.activeInstallations.add(serviceName) - // Compute new image string + // Compute new image string. AMD-on-Ollama overrides this to the rolling :rocm tag + // (set during GPU detection below) since per-version ROCm tags aren't always published. const currentImage = service.container_image const imageBase = currentImage.includes(':') ? currentImage.substring(0, currentImage.lastIndexOf(':')) : currentImage - const newImage = `${imageBase}:${targetVersion}` + let newImage = `${imageBase}:${targetVersion}` + + // GPU detection runs before the pull so AMD updates pull ollama/ollama:rocm rather + // than the standard tag. Detection result is reused below when building the new + // container config (devices, env). Non-Ollama services skip this entirely. + let updatedDeviceRequests: any[] | undefined = undefined + let updatedAmdDevices: any[] | undefined = undefined + let updatedAmdGpuConfigured = false + if (serviceName === SERVICE_NAMES.OLLAMA) { + const gpuResult = await this._detectGPUType() + if (gpuResult.type === 'nvidia') { + this._broadcast( + serviceName, + 'update-gpu-config', + `NVIDIA container runtime detected. Configuring updated container with GPU support...` + ) + updatedDeviceRequests = [ + { Driver: 'nvidia', Count: -1, Capabilities: [['gpu']] }, + ] + } else if (gpuResult.type === 'amd') { + const amdEnabledRaw = await KVStore.getValue('ai.amdGpuAcceleration') + const amdAccelerationEnabled = String(amdEnabledRaw) !== 'false' + if (amdAccelerationEnabled) { + this._broadcast( + serviceName, + 'update-gpu-config', + `AMD GPU detected. Using ROCm image with /dev/kfd and /dev/dri passthrough...` + ) + newImage = 'ollama/ollama:rocm' + updatedAmdDevices = await this._discoverAMDDevices() + updatedAmdGpuConfigured = true + } else { + this._broadcast( + serviceName, + 'update-gpu-config', + `AMD GPU detected but acceleration is disabled via ai.amdGpuAcceleration. Using CPU-only configuration.` + ) + } + } else if (gpuResult.toolkitMissing) { + this._broadcast( + serviceName, + 'update-gpu-config', + `NVIDIA GPU detected but NVIDIA Container Toolkit is not installed. Using CPU-only configuration. Install the toolkit and reinstall AI Assistant for GPU acceleration: https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/latest/install-guide.html` + ) + } else { + this._broadcast(serviceName, 'update-gpu-config', `No GPU detected. Using CPU-only configuration.`) + } + } // Step 1: Pull new image this._broadcast(serviceName, 'update-pulling', `Pulling image ${newImage}...`) @@ -1054,48 +1128,21 @@ export class DockerService { const hostConfig = inspectData.HostConfig || {} - // Re-run GPU detection for Ollama so updates always reflect the current GPU environment. - // This handles cases where the NVIDIA Container Toolkit was installed after the initial - // Ollama setup, and ensures DeviceRequests are always built fresh rather than relying on - // round-tripping the Docker inspect format back into the create API. - let updatedDeviceRequests: any[] | undefined = undefined - if (serviceName === SERVICE_NAMES.OLLAMA) { - const gpuResult = await this._detectGPUType() - - if (gpuResult.type === 'nvidia') { - this._broadcast( - serviceName, - 'update-gpu-config', - `NVIDIA container runtime detected. Configuring updated container with GPU support...` - ) - updatedDeviceRequests = [ - { - Driver: 'nvidia', - Count: -1, - Capabilities: [['gpu']], - }, + // GPU detection already ran above (before the pull) so we know the right image, devices, + // and whether HSA_OVERRIDE needs injection. For AMD, replace any prior HSA_OVERRIDE in + // the inspect-captured env so updates from older containers pick up the current value. + const baseEnv = inspectData.Config?.Env || [] + const finalEnv = updatedAmdGpuConfigured + ? [ + ...baseEnv.filter((e: string) => !e.startsWith('HSA_OVERRIDE_GFX_VERSION=')), + 'HSA_OVERRIDE_GFX_VERSION=11.0.0', ] - } else if (gpuResult.type === 'amd') { - this._broadcast( - serviceName, - 'update-gpu-config', - `AMD GPU detected. ROCm GPU acceleration is not yet supported — using CPU-only configuration.` - ) - } else if (gpuResult.toolkitMissing) { - this._broadcast( - serviceName, - 'update-gpu-config', - `NVIDIA GPU detected but NVIDIA Container Toolkit is not installed. Using CPU-only configuration. Install the toolkit and reinstall AI Assistant for GPU acceleration: https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/latest/install-guide.html` - ) - } else { - this._broadcast(serviceName, 'update-gpu-config', `No GPU detected. Using CPU-only configuration.`) - } - } + : baseEnv const newContainerConfig: any = { Image: newImage, name: serviceName, - Env: inspectData.Config?.Env || undefined, + Env: finalEnv.length > 0 ? finalEnv : undefined, Cmd: inspectData.Config?.Cmd || undefined, ExposedPorts: inspectData.Config?.ExposedPorts || undefined, WorkingDir: inspectData.Config?.WorkingDir || undefined, @@ -1105,7 +1152,7 @@ export class DockerService { PortBindings: hostConfig.PortBindings || undefined, RestartPolicy: hostConfig.RestartPolicy || undefined, DeviceRequests: serviceName === SERVICE_NAMES.OLLAMA ? updatedDeviceRequests : (hostConfig.DeviceRequests || undefined), - Devices: hostConfig.Devices || undefined, + Devices: serviceName === SERVICE_NAMES.OLLAMA && updatedAmdDevices ? updatedAmdDevices : (hostConfig.Devices || undefined), }, NetworkingConfig: inspectData.NetworkSettings?.Networks ? { diff --git a/admin/app/services/system_service.ts b/admin/app/services/system_service.ts index 5701de33..1a55cfb2 100644 --- a/admin/app/services/system_service.ts +++ b/admin/app/services/system_service.ts @@ -12,6 +12,7 @@ import { } from '../../types/system.js' import { SERVICE_NAMES } from '../../constants/service_names.js' import { readFileSync } from 'node:fs' +import { readFile } from 'node:fs/promises' import path, { join } from 'node:path' import { getAllFilesystems, getFile } from '../utils/fs.js' import axios from 'axios' @@ -72,6 +73,61 @@ export class SystemService { return false } + /** + * Probe Ollama startup logs for the canonical "inference compute" line that records + * which compute backend was selected. This catches silent CPU fallback (e.g. when + * /dev/kfd is mounted but ROCm initialization fails, or NVML dies after an update) + * which the older nvidia-smi exec probe could not detect. + * + * Returns the parsed library, GPU model name, and VRAM in MiB, or null when: + * - the Ollama container is not running + * - the line has not been emitted (Ollama still starting up) + * - logs show CPU-only operation (no GPU detected) + */ + async getOllamaInferenceComputeFromLogs(): Promise<{ + library: 'CUDA' | 'ROCm' + name: string + vramMiB: number + } | null> { + try { + const containers = await this.dockerService.docker.listContainers({ all: false }) + const ollamaContainer = containers.find((c) => c.Names.includes(`/${SERVICE_NAMES.OLLAMA}`)) + if (!ollamaContainer) return null + + const container = this.dockerService.docker.getContainer(ollamaContainer.Id) + const buf = (await container.logs({ + stdout: true, + stderr: true, + tail: 500, + follow: false, + })) as unknown as Buffer + const logs = buf.toString('utf8') + + const lines = logs.split('\n').filter((l) => l.includes('msg="inference compute"')) + if (lines.length === 0) return null + + const lastLine = lines[lines.length - 1] + const libraryMatch = lastLine.match(/library=(CUDA|ROCm)/) + if (!libraryMatch) return null + + const descMatch = lastLine.match(/description="([^"]+)"/) + const totalMatch = lastLine.match(/total="([0-9.]+)\s*GiB"/) + + return { + library: libraryMatch[1] as 'CUDA' | 'ROCm', + name: + descMatch?.[1] || + (libraryMatch[1] === 'CUDA' ? 'NVIDIA GPU' : 'AMD GPU'), + vramMiB: totalMatch ? Math.round(Number.parseFloat(totalMatch[1]) * 1024) : 0, + } + } catch (error) { + logger.warn( + `[SystemService] Failed to probe Ollama logs for inference compute line: ${error instanceof Error ? error.message : error}` + ) + return null + } + } + async getNvidiaSmiInfo(): Promise< | Array<{ vendor: string; model: string; vram: number }> | { error: string } @@ -317,10 +373,14 @@ export class SystemService { logger.error('Error reading disk info file:', error) } - // GPU health tracking — detect when host has NVIDIA GPU but Ollama can't access it + // GPU health tracking — detect when host has a GPU runtime but Ollama can't access it. + // Primary probe: parse Ollama's "inference compute" startup log line for both NVIDIA + // and AMD. Secondary probe (NVIDIA only): nvidia-smi exec, retained as a fallback for + // hardware enrichment when log parsing has not yet captured a startup line. let gpuHealth: GpuHealthStatus = { status: 'no_gpu', hasNvidiaRuntime: false, + hasRocmRuntime: false, ollamaGpuAccessible: false, } @@ -340,27 +400,51 @@ export class SystemService { } // If si.graphics() returned no controllers (common inside Docker), - // fall back to nvidia runtime + nvidia-smi detection + // fall back to runtime + Ollama log probe to figure out what's accessible. if (!graphics.controllers || graphics.controllers.length === 0) { const runtimes = dockerInfo.Runtimes || {} - if ('nvidia' in runtimes) { - gpuHealth.hasNvidiaRuntime = true - const nvidiaInfo = await this.getNvidiaSmiInfo() - if (Array.isArray(nvidiaInfo)) { - graphics.controllers = nvidiaInfo.map((gpu) => ({ - model: gpu.model, - vendor: gpu.vendor, - bus: '', - vram: gpu.vram, - vramDynamic: false, // assume false here, we don't actually use this field for our purposes. - })) + gpuHealth.hasNvidiaRuntime = 'nvidia' in runtimes + + // AMD doesn't register a Docker runtime. Detection sources, in priority order: + // 1. KV 'gpu.type' (set by DockerService._detectGPUType after first Ollama install) + // 2. Marker file at /app/storage/.nomad-gpu-type (written by install_nomad.sh) + // The marker file matters because the System page should reflect AMD presence + // even before AI Assistant has been installed for the first time. + let savedGpuType: string | null | undefined = await KVStore.getValue('gpu.type') as string | undefined + if (!savedGpuType) { + try { + savedGpuType = (await readFile('/app/storage/.nomad-gpu-type', 'utf8')).trim() + } catch {} + } + const amdEnabledRaw = await KVStore.getValue('ai.amdGpuAcceleration') + const amdAccelerationEnabled = String(amdEnabledRaw) !== 'false' + gpuHealth.hasRocmRuntime = savedGpuType === 'amd' && amdAccelerationEnabled + + if (gpuHealth.hasNvidiaRuntime || gpuHealth.hasRocmRuntime) { + gpuHealth.gpuVendor = gpuHealth.hasNvidiaRuntime ? 'nvidia' : 'amd' + + // Primary probe: Ollama log parsing — works for both vendors and catches silent fallback + const logInfo = await this.getOllamaInferenceComputeFromLogs() + if (logInfo) { + graphics.controllers = [ + { + model: logInfo.name, + vendor: logInfo.library === 'CUDA' ? 'NVIDIA' : 'AMD', + bus: '', + vram: logInfo.vramMiB, + vramDynamic: false, + }, + ] gpuHealth.status = 'ok' gpuHealth.ollamaGpuAccessible = true - } else if (nvidiaInfo === 'OLLAMA_NOT_FOUND') { - // No local Ollama container — check if a remote Ollama URL is configured - const externalOllamaGpu = await this.getExternalOllamaGpuInfo() - if (externalOllamaGpu) { - graphics.controllers = externalOllamaGpu.map((gpu) => ({ + } else if (gpuHealth.hasNvidiaRuntime) { + // NVIDIA secondary path: nvidia-smi exec preserves prior behavior when + // the log parser hasn't seen a startup line yet (e.g. log rotation, + // very fresh container). Distinguishes "no Ollama container" from + // "container exists but GPU broken". + const nvidiaInfo = await this.getNvidiaSmiInfo() + if (Array.isArray(nvidiaInfo)) { + graphics.controllers = nvidiaInfo.map((gpu) => ({ model: gpu.model, vendor: gpu.vendor, bus: '', @@ -369,25 +453,66 @@ export class SystemService { })) gpuHealth.status = 'ok' gpuHealth.ollamaGpuAccessible = true + } else if (nvidiaInfo === 'OLLAMA_NOT_FOUND') { + const externalOllamaGpu = await this.getExternalOllamaGpuInfo() + if (externalOllamaGpu) { + graphics.controllers = externalOllamaGpu.map((gpu) => ({ + model: gpu.model, + vendor: gpu.vendor, + bus: '', + vram: gpu.vram, + vramDynamic: false, + })) + gpuHealth.status = 'ok' + gpuHealth.ollamaGpuAccessible = true + } else { + gpuHealth.status = 'ollama_not_installed' + } } else { - gpuHealth.status = 'ollama_not_installed' + const externalOllamaGpu = await this.getExternalOllamaGpuInfo() + if (externalOllamaGpu) { + graphics.controllers = externalOllamaGpu.map((gpu) => ({ + model: gpu.model, + vendor: gpu.vendor, + bus: '', + vram: gpu.vram, + vramDynamic: false, + })) + gpuHealth.status = 'ok' + gpuHealth.ollamaGpuAccessible = true + } else { + gpuHealth.status = 'passthrough_failed' + logger.warn( + `NVIDIA runtime detected but GPU passthrough failed: ${typeof nvidiaInfo === 'string' ? nvidiaInfo : JSON.stringify(nvidiaInfo)}` + ) + } } } else { - const externalOllamaGpu = await this.getExternalOllamaGpuInfo() - if (externalOllamaGpu) { - graphics.controllers = externalOllamaGpu.map((gpu) => ({ - model: gpu.model, - vendor: gpu.vendor, - bus: '', - vram: gpu.vram, - vramDynamic: false, - })) - gpuHealth.status = 'ok' - gpuHealth.ollamaGpuAccessible = true + // AMD path: no nvidia-smi equivalent worth running — log parser is authoritative. + // Distinguish "Ollama not running" from "Ollama running but no GPU log line". + const containers = await this.dockerService.docker.listContainers({ all: false }) + const ollamaRunning = containers.some((c) => + c.Names.includes(`/${SERVICE_NAMES.OLLAMA}`) + ) + if (!ollamaRunning) { + const externalOllamaGpu = await this.getExternalOllamaGpuInfo() + if (externalOllamaGpu) { + graphics.controllers = externalOllamaGpu.map((gpu) => ({ + model: gpu.model, + vendor: gpu.vendor, + bus: '', + vram: gpu.vram, + vramDynamic: false, + })) + gpuHealth.status = 'ok' + gpuHealth.ollamaGpuAccessible = true + } else { + gpuHealth.status = 'ollama_not_installed' + } } else { gpuHealth.status = 'passthrough_failed' logger.warn( - `NVIDIA runtime detected but GPU passthrough failed: ${typeof nvidiaInfo === 'string' ? nvidiaInfo : JSON.stringify(nvidiaInfo)}` + 'AMD GPU detected but Ollama logs show no ROCm initialization — passthrough or HSA override may have failed' ) } } diff --git a/admin/inertia/pages/settings/models.tsx b/admin/inertia/pages/settings/models.tsx index fc2b1dc1..fe119d89 100644 --- a/admin/inertia/pages/settings/models.tsx +++ b/admin/inertia/pages/settings/models.tsx @@ -283,7 +283,7 @@ export default function ModelsPage(props: { type="warning" variant="bordered" title="GPU Not Accessible" - message={`Your system has an NVIDIA GPU, but ${aiAssistantName} can't access it. AI is running on CPU only, which is significantly slower.`} + message={`Your system has ${systemInfo?.gpuHealth?.gpuVendor === 'amd' ? 'an AMD' : 'an NVIDIA'} GPU, but ${aiAssistantName} can't access it. AI is running on CPU only, which is significantly slower.`} className="!mt-6" dismissible={true} onDismiss={handleDismissGpuBanner} diff --git a/admin/inertia/pages/settings/system.tsx b/admin/inertia/pages/settings/system.tsx index 7b40088d..1a13b52f 100644 --- a/admin/inertia/pages/settings/system.tsx +++ b/admin/inertia/pages/settings/system.tsx @@ -209,7 +209,7 @@ export default function SettingsPage(props: { type="warning" variant="bordered" title="GPU Not Accessible to AI Assistant" - message="Your system has an NVIDIA GPU, but the AI Assistant can't access it. AI is running on CPU only, which is significantly slower." + message={`Your system has ${info?.gpuHealth?.gpuVendor === 'amd' ? 'an AMD' : 'an NVIDIA'} GPU, but the AI Assistant can't access it. AI is running on CPU only, which is significantly slower.`} dismissible={true} onDismiss={handleDismissGpuBanner} buttonProps={{ diff --git a/admin/types/kv_store.ts b/admin/types/kv_store.ts index 8fb26866..a3632ab0 100644 --- a/admin/types/kv_store.ts +++ b/admin/types/kv_store.ts @@ -12,6 +12,7 @@ export const KV_STORE_SCHEMA = { 'gpu.type': 'string', 'ai.remoteOllamaUrl': 'string', 'ai.ollamaFlashAttention': 'boolean', + 'ai.amdGpuAcceleration': 'boolean', } as const type KVTagToType = T extends 'boolean' ? boolean : string diff --git a/admin/types/system.ts b/admin/types/system.ts index 7c4e6d7f..c8ed4abc 100644 --- a/admin/types/system.ts +++ b/admin/types/system.ts @@ -3,7 +3,9 @@ import { Systeminformation } from 'systeminformation' export type GpuHealthStatus = { status: 'ok' | 'passthrough_failed' | 'no_gpu' | 'ollama_not_installed' hasNvidiaRuntime: boolean + hasRocmRuntime: boolean ollamaGpuAccessible: boolean + gpuVendor?: 'nvidia' | 'amd' } export type SystemInformationResponse = { diff --git a/install/install_nomad.sh b/install/install_nomad.sh index ced178fe..ef501a0d 100644 --- a/install/install_nomad.sh +++ b/install/install_nomad.sh @@ -517,18 +517,35 @@ verify_gpu_setup() { echo -e "${YELLOW}○${RESET} Docker NVIDIA runtime not detected\\n" fi - # Check for AMD GPU + # Check for AMD GPU — restrict to display controller classes to avoid false positives + # from AMD CPU host bridges, PCI bridges, and chipset devices. + local has_amd_gpu='false' if command -v lspci &> /dev/null; then - if lspci 2>/dev/null | grep -iE "amd|radeon" &> /dev/null; then - echo -e "${YELLOW}○${RESET} AMD GPU detected (ROCm support not currently available)\\n" + if lspci 2>/dev/null | grep -iE "VGA|3D controller|Display" | grep -iE "amd|radeon" &> /dev/null; then + has_amd_gpu='true' + echo -e "${GREEN}✓${RESET} AMD GPU detected — ROCm acceleration will be configured automatically when AI Assistant is installed.\\n" fi fi - + + # Write detected GPU type to a marker file the admin container can read. The admin + # container lacks lspci and AMD GPUs don't register a Docker runtime, so this is the + # only reliable way for the admin to know an AMD GPU is present at install time. + local gpu_marker_path="${NOMAD_DIR}/storage/.nomad-gpu-type" + if command -v nvidia-smi &> /dev/null; then + echo 'nvidia' | sudo tee "${gpu_marker_path}" > /dev/null 2>&1 || true + elif [[ "${has_amd_gpu}" == 'true' ]]; then + echo 'amd' | sudo tee "${gpu_marker_path}" > /dev/null 2>&1 || true + else + sudo rm -f "${gpu_marker_path}" 2>/dev/null || true + fi + echo -e "${YELLOW}===========================================${RESET}\\n" - + # Summary if command -v nvidia-smi &> /dev/null && docker info 2>/dev/null | grep -q "nvidia"; then echo -e "${GREEN}#${RESET} GPU acceleration is properly configured! The AI Assistant will use your GPU.\\n" + elif [[ "${has_amd_gpu}" == 'true' ]]; then + echo -e "${GREEN}#${RESET} GPU acceleration will be enabled (AMD/ROCm) when AI Assistant is installed from the dashboard.\\n" else echo -e "${YELLOW}#${RESET} GPU acceleration not detected. The AI Assistant will run in CPU-only mode.\\n" if command -v nvidia-smi &> /dev/null && ! docker info 2>/dev/null | grep -q "nvidia"; then From 0836d84bb21843456e83ab8a59003d132ea77d0e Mon Sep 17 00:00:00 2001 From: Kenneth Brewer Date: Wed, 29 Apr 2026 00:55:11 -0400 Subject: [PATCH 035/108] docs: added notes field info to the map pin API reference (#803) --- admin/docs/api-reference.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/admin/docs/api-reference.md b/admin/docs/api-reference.md index 928b1e38..c14348cb 100644 --- a/admin/docs/api-reference.md +++ b/admin/docs/api-reference.md @@ -153,8 +153,8 @@ ZIM files provide offline Wikipedia, books, and other content via Kiwix. | Method | Path | Description | |--------|------|-------------| | GET | `/api/maps/markers` | List map markers | -| POST | `/api/maps/markers` | Add map marker (body: {"name": "Test Marker", "longitude": 0.0, "latitude": 0.0, "color": "yellow", "marker_type": "pin"} ) | -| PATCH | `/api/maps/markers/{id}` | Update a map marker (body: {"name": "Test Marker", "longitude": 0.0, "latitude": 0.0, "color": "yellow", "marker_type": "pin"} ) fields that don't change can be omitted| +| POST | `/api/maps/markers` | Add map marker (body: {"name": "Test Marker", "notes": "Example note", "longitude": 0.0, "latitude": 0.0, "color": "yellow", "marker_type": "pin"} ) | +| PATCH | `/api/maps/markers/{id}` | Update a map marker (body: {"name": "Test Marker", "notes": "Example note", "longitude": 0.0, "latitude": 0.0, "color": "yellow", "marker_type": "pin"} ) fields that don't change can be omitted| | DELETE | `/api/maps/markers/{id}` | Delete a map marker | --- From bb1834a36438e545f522d965cf64855c8cb1e1ce Mon Sep 17 00:00:00 2001 From: cuyua9 <2114364329@qq.com> Date: Mon, 4 May 2026 03:49:06 +0800 Subject: [PATCH 036/108] fix(UI): wire map file delete confirmation to API (#732) Co-authored-by: cuyua9 --- admin/inertia/lib/api.ts | 9 +++++++++ admin/inertia/pages/settings/maps.tsx | 29 ++++++++++++++++++++++++--- 2 files changed, 35 insertions(+), 3 deletions(-) diff --git a/admin/inertia/lib/api.ts b/admin/inertia/lib/api.ts index 0df95ae0..2449259d 100644 --- a/admin/inertia/lib/api.ts +++ b/admin/inertia/lib/api.ts @@ -130,6 +130,15 @@ class API { })() } + async deleteMapRegionFile(filename: string): Promise<{ message: string }> { + return catchInternal(async () => { + const response = await this.client.delete<{ message: string }>( + `/maps/${encodeURIComponent(filename)}` + ) + return response.data + })() + } + async downloadRemoteZimFile( url: string, metadata?: { title: string; summary?: string; author?: string; size_bytes?: number } diff --git a/admin/inertia/pages/settings/maps.tsx b/admin/inertia/pages/settings/maps.tsx index da97bb1f..47055fa7 100644 --- a/admin/inertia/pages/settings/maps.tsx +++ b/admin/inertia/pages/settings/maps.tsx @@ -29,6 +29,7 @@ export default function MapsManager(props: { const { openModal, closeAllModals } = useModals() const { addNotification } = useNotifications() const [downloading, setDownloading] = useState(false) + const [deletingFileKey, setDeletingFileKey] = useState(null) const { data: curatedCollections } = useQuery({ queryKey: [CURATED_COLLECTIONS_KEY], @@ -120,18 +121,40 @@ export default function MapsManager(props: { } } + async function deleteFile(file: FileEntry) { + if (file.type !== 'file') return + + try { + setDeletingFileKey(file.key) + await api.deleteMapRegionFile(file.key) + addNotification({ + type: 'success', + message: `${file.name} has been deleted.`, + }) + closeAllModals() + router.reload({ only: ['maps'] }) + } catch (error) { + console.error('Error deleting map file:', error) + addNotification({ + type: 'error', + message: `Failed to delete ${file.name}. Please try again.`, + }) + } finally { + setDeletingFileKey(null) + } + } + async function confirmDeleteFile(file: FileEntry) { openModal( { - closeAllModals() - }} + onConfirm={() => deleteFile(file)} onCancel={closeAllModals} open={true} confirmText="Delete" cancelText="Cancel" confirmVariant="danger" + confirmLoading={file.type === 'file' && deletingFileKey === file.key} >

Are you sure you want to delete {file.name}? This action cannot be undone. From 360e7a0af48714db06792896ce302b85c0543213 Mon Sep 17 00:00:00 2001 From: Chris Sherwood Date: Wed, 22 Apr 2026 14:36:05 -0700 Subject: [PATCH 037/108] feat(content-updates): show size, surface downloads in Active Downloads MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Content Updates had three UX problems that compounded: 1. No size column, so users had to guess how big an update would be before clicking Update All. Upstream /api/v1/resources/check-updates doesn't return size, so CollectionUpdateService now enriches each update with a Content-Length HEAD request in parallel (5s timeout, non-fatal on failure — the row just renders an em-dash). 2. Small ZIM updates (1-8 MB) never appeared in Active Downloads. Two causes, both fixed: handleApply / handleApplyAll didn't invalidate the download-jobs query after dispatching, and useDownloads idled at 30s between polls — enough for a fast job to dispatch, download, and get cleaned up by removeOnComplete before the next refetch. 3. applyUpdate didn't forward title / totalBytes to RunDownloadJob, so any update that did briefly surface in Active Downloads had no label and no byte-count progress, just a filename and a percentage. It now passes both (matching zim_service's dispatch pattern). Also parallelized applyAllUpdates so dispatching five updates doesn't serialize five sequential BullMQ round-trips. Co-Authored-By: Claude Opus 4.7 (1M context) --- .../app/services/collection_update_service.ts | 49 ++++++++++++++----- admin/app/validators/common.ts | 1 + admin/inertia/hooks/useDownloads.ts | 5 +- admin/inertia/pages/settings/update.tsx | 19 ++++++- admin/types/collections.ts | 1 + 5 files changed, 60 insertions(+), 15 deletions(-) diff --git a/admin/app/services/collection_update_service.ts b/admin/app/services/collection_update_service.ts index fee6c148..2e19cac8 100644 --- a/admin/app/services/collection_update_service.ts +++ b/admin/app/services/collection_update_service.ts @@ -53,8 +53,10 @@ export class CollectionUpdateService { `[CollectionUpdateService] Update check complete: ${response.data.length} update(s) available` ) + const updates = await this.enrichWithSizes(response.data) + return { - updates: response.data, + updates, checked_at: new Date().toISOString(), } } catch (error) { @@ -105,6 +107,8 @@ export class CollectionUpdateService { update.resource_type === 'zim' ? ZIM_MIME_TYPES : PMTILES_MIME_TYPES, forceNew: true, filetype: update.resource_type, + title: update.resource_id, + totalBytes: update.size_bytes, resourceMetadata: { resource_id: update.resource_id, version: update.latest_version, @@ -126,21 +130,42 @@ export class CollectionUpdateService { async applyAllUpdates( updates: ResourceUpdateInfo[] ): Promise<{ results: Array<{ resource_id: string; success: boolean; jobId?: string; error?: string }> }> { - const results: Array<{ - resource_id: string - success: boolean - jobId?: string - error?: string - }> = [] - - for (const update of updates) { - const result = await this.applyUpdate(update) - results.push({ resource_id: update.resource_id, ...result }) - } + const results = await Promise.all( + updates.map(async (update) => { + const result = await this.applyUpdate(update) + return { resource_id: update.resource_id, ...result } + }) + ) return { results } } + /** + * Fetch Content-Length for each update URL in parallel. HEAD failures are non-fatal — + * the update row just renders without a size. Bounded to HEAD_TIMEOUT_MS so a slow + * mirror doesn't block the whole check. + */ + private async enrichWithSizes(updates: ResourceUpdateInfo[]): Promise { + const HEAD_TIMEOUT_MS = 5000 + + return await Promise.all( + updates.map(async (update) => { + if (update.size_bytes) return update // Trust upstream if it already gave us one + try { + const head = await axios.head(update.download_url, { + timeout: HEAD_TIMEOUT_MS, + maxRedirects: 5, + validateStatus: (s) => s >= 200 && s < 400, + }) + const len = Number(head.headers['content-length']) + return Number.isFinite(len) && len > 0 ? { ...update, size_bytes: len } : update + } catch { + return update + } + }) + ) + } + private buildFilename(update: ResourceUpdateInfo): string { if (update.resource_type === 'zim') { return `${update.resource_id}_${update.latest_version}.zim` diff --git a/admin/app/validators/common.ts b/admin/app/validators/common.ts index 8fe78bdd..7065d4ca 100644 --- a/admin/app/validators/common.ts +++ b/admin/app/validators/common.ts @@ -100,6 +100,7 @@ const resourceUpdateInfoBase = vine.object({ installed_version: vine.string().trim(), latest_version: vine.string().trim().minLength(1), download_url: vine.string().url({ require_tld: false }).trim(), + size_bytes: vine.number().positive().optional(), }) export const applyContentUpdateValidator = vine.compile(resourceUpdateInfoBase) diff --git a/admin/inertia/hooks/useDownloads.ts b/admin/inertia/hooks/useDownloads.ts index 3cdb859c..b03399d5 100644 --- a/admin/inertia/hooks/useDownloads.ts +++ b/admin/inertia/hooks/useDownloads.ts @@ -19,8 +19,9 @@ const useDownloads = (props: useDownloadsProps) => { queryFn: () => api.listDownloadJobs(props.filetype), refetchInterval: (query) => { const data = query.state.data - // Only poll when there are active downloads; otherwise use a slower interval - return data && data.length > 0 ? 2000 : 30000 + // Idle poll is kept tight so newly-dispatched jobs surface quickly — small ZIM + // updates can complete in ~2s, so a 30s idle interval almost always missed them. + return data && data.length > 0 ? 2000 : 3000 }, enabled: props.enabled ?? true, }) diff --git a/admin/inertia/pages/settings/update.tsx b/admin/inertia/pages/settings/update.tsx index 23527fb6..348040da 100644 --- a/admin/inertia/pages/settings/update.tsx +++ b/admin/inertia/pages/settings/update.tsx @@ -12,9 +12,10 @@ import type { ContentUpdateCheckResult, ResourceUpdateInfo } from '../../../type import api from '~/lib/api' import Input from '~/components/inputs/Input' import Switch from '~/components/inputs/Switch' -import { useMutation } from '@tanstack/react-query' +import { useMutation, useQueryClient } from '@tanstack/react-query' import { useNotifications } from '~/context/NotificationContext' import { useSystemSetting } from '~/hooks/useSystemSetting' +import { formatBytes } from '~/lib/util' type Props = { updateAvailable: boolean @@ -25,6 +26,7 @@ type Props = { function ContentUpdatesSection() { const { addNotification } = useNotifications() + const queryClient = useQueryClient() const [checkResult, setCheckResult] = useState(null) const [isChecking, setIsChecking] = useState(false) const [applyingIds, setApplyingIds] = useState>(new Set()) @@ -60,6 +62,9 @@ function ContentUpdatesSection() { ? { ...prev, updates: prev.updates.filter((u) => u.resource_id !== update.resource_id) } : prev ) + // Force Active Downloads to refetch now — small updates finish before the next + // idle poll fires, so without this the user wouldn't see them. + queryClient.invalidateQueries({ queryKey: ['download-jobs'] }) } else { addNotification({ type: 'error', message: result?.error || 'Failed to start update' }) } @@ -95,6 +100,9 @@ function ContentUpdatesSection() { ? { ...prev, updates: prev.updates.filter((u) => !successIds.has(u.resource_id)) } : prev ) + if (successIds.size > 0) { + queryClient.invalidateQueries({ queryKey: ['download-jobs'] }) + } } } catch { addNotification({ type: 'error', message: 'Failed to apply updates' }) @@ -182,6 +190,15 @@ function ContentUpdatesSection() { ), }, + { + accessor: 'size_bytes', + title: 'Size', + render: (record) => ( + + {record.size_bytes ? formatBytes(record.size_bytes, 1) : '—'} + + ), + }, { accessor: 'installed_version', title: 'Version', diff --git a/admin/types/collections.ts b/admin/types/collections.ts index 1ec6d5c2..abd47fc0 100644 --- a/admin/types/collections.ts +++ b/admin/types/collections.ts @@ -86,6 +86,7 @@ export type ResourceUpdateInfo = { installed_version: string latest_version: string download_url: string + size_bytes?: number } export type ContentUpdateCheckResult = { From 27cd80309025dc63ab5ec4fdc86ef828b49272e1 Mon Sep 17 00:00:00 2001 From: 0xGlitch <92540908+bgauger@users.noreply.github.com> Date: Sun, 3 May 2026 14:47:53 -0600 Subject: [PATCH 038/108] feat(Maps): regional map downloads via go-pmtiles extract (#780) * feat(maps): add regional map downloads via go-pmtiles extract * address Copilot review feedback on PR #780 - auto-refresh preflight on selection/maxzoom change with 400ms debounce and requestId stale-safety so the confirm button no longer requires a two-step "Estimate Size" -> "Start Download" dance - safeUpdateProgress helper replaces fire-and-forget updateProgress().catch() pattern so cancelled-job errors (code -1) can't surface as unhandled rejections - gate world basemap source on worldBasemapReady - when ensureWorldBasemap() fails we already delete world.pmtiles, so emitting the source was producing 404s on every tile request - verify go-pmtiles binary SHA256 at image build time; upstream doesn't ship a checksums file so per-arch hashes are pinned as build args with a regenerate note when bumping PMTILES_VERSION --- Dockerfile | 25 ++ admin/adonisrc.ts | 4 + admin/app/controllers/maps_controller.ts | 24 ++ admin/app/jobs/run_extract_pmtiles_job.ts | 294 ++++++++++++++ admin/app/services/countries_service.ts | 308 ++++++++++++++ admin/app/services/download_service.ts | 112 +++-- admin/app/services/map_service.ts | 322 ++++++++++++++- admin/app/validators/common.ts | 28 ++ admin/commands/queue/work.ts | 6 + admin/constants/map_regions.ts | 32 ++ .../inertia/components/CountryPickerModal.tsx | 384 ++++++++++++++++++ admin/inertia/lib/api.ts | 41 ++ admin/inertia/pages/settings/maps.tsx | 33 ++ .../geodata/ne_50m_admin_0_countries.geojson | 1 + admin/start/routes.ts | 4 + admin/types/maps.ts | 34 ++ 16 files changed, 1620 insertions(+), 32 deletions(-) create mode 100644 admin/app/jobs/run_extract_pmtiles_job.ts create mode 100644 admin/app/services/countries_service.ts create mode 100644 admin/constants/map_regions.ts create mode 100644 admin/inertia/components/CountryPickerModal.tsx create mode 100644 admin/resources/geodata/ne_50m_admin_0_countries.geojson diff --git a/Dockerfile b/Dockerfile index 8850e237..0dcc1845 100644 --- a/Dockerfile +++ b/Dockerfile @@ -34,6 +34,31 @@ FROM base ARG VERSION=dev ARG BUILD_DATE ARG VCS_REF +ARG TARGETARCH + +# go-pmtiles (regional map extracts). Pinned so the CLI's stdout format stays +# in sync with parseDryRunOutput(). +ARG PMTILES_VERSION=1.30.2 +# Upstream releases don't ship a checksums file, so pin per-arch SHA256 here. +# When bumping PMTILES_VERSION, regenerate these with: +# curl -fsSL | sha256sum +ARG PMTILES_SHA256_AMD64=2cd3aa18868297fc88425038f794efdc0995e0275f4ca16fa496dd79e245a40c +ARG PMTILES_SHA256_ARM64=804cdf071834e1156af554c1a26cc42b56b9cde5a2db9c6e3653d16fb846d5fa +RUN set -eux; \ + case "${TARGETARCH:-amd64}" in \ + amd64) PMTILES_ARCH=x86_64; PMTILES_SHA256="${PMTILES_SHA256_AMD64}" ;; \ + arm64) PMTILES_ARCH=arm64; PMTILES_SHA256="${PMTILES_SHA256_ARM64}" ;; \ + *) echo "Unsupported TARGETARCH: ${TARGETARCH}" >&2; exit 1 ;; \ + esac; \ + TARBALL="go-pmtiles_${PMTILES_VERSION}_Linux_${PMTILES_ARCH}.tar.gz"; \ + cd /tmp; \ + curl -fsSL -o "$TARBALL" \ + "https://github.com/protomaps/go-pmtiles/releases/download/v${PMTILES_VERSION}/${TARBALL}"; \ + echo "${PMTILES_SHA256} ${TARBALL}" | sha256sum -c -; \ + tar -xzf "$TARBALL" -C /usr/local/bin pmtiles; \ + rm -f "$TARBALL"; \ + chmod +x /usr/local/bin/pmtiles; \ + /usr/local/bin/pmtiles version # Labels LABEL org.opencontainers.image.title="Project N.O.M.A.D" \ diff --git a/admin/adonisrc.ts b/admin/adonisrc.ts index a091ce2d..34586ca4 100644 --- a/admin/adonisrc.ts +++ b/admin/adonisrc.ts @@ -107,6 +107,10 @@ export default defineConfig({ pattern: 'resources/views/**/*.edge', reloadServer: false, }, + { + pattern: 'resources/geodata/**/*.geojson', + reloadServer: false, + }, { pattern: 'public/**', reloadServer: false, diff --git a/admin/app/controllers/maps_controller.ts b/admin/app/controllers/maps_controller.ts index dd93a8b1..097d9fb8 100644 --- a/admin/app/controllers/maps_controller.ts +++ b/admin/app/controllers/maps_controller.ts @@ -4,6 +4,8 @@ import { assertNotPrivateUrl, downloadCollectionValidator, filenameParamValidator, + mapExtractPreflightValidator, + mapExtractValidator, remoteDownloadValidator, remoteDownloadValidatorOptional, } from '#validators/common' @@ -87,6 +89,28 @@ export default class MapsController { } } + async listCountries({}: HttpContext) { + return { countries: await this.mapService.listCountries() } + } + + async listCountryGroups({}: HttpContext) { + return { groups: await this.mapService.listCountryGroups() } + } + + async extractPreflight({ request }: HttpContext) { + const payload = await request.validateUsing(mapExtractPreflightValidator) + return await this.mapService.extractPreflight(payload) + } + + async extractRegion({ request }: HttpContext) { + const payload = await request.validateUsing(mapExtractValidator) + const result = await this.mapService.extractRegion(payload) + return { + message: 'Extract started successfully', + ...result, + } + } + async styles({ request, response }: HttpContext) { // Automatically ensure base assets are present before generating styles const baseAssetsExist = await this.mapService.ensureBaseAssets() diff --git a/admin/app/jobs/run_extract_pmtiles_job.ts b/admin/app/jobs/run_extract_pmtiles_job.ts new file mode 100644 index 00000000..73c4eed3 --- /dev/null +++ b/admin/app/jobs/run_extract_pmtiles_job.ts @@ -0,0 +1,294 @@ +import { Job, UnrecoverableError } from 'bullmq' +import { spawn, ChildProcess } from 'child_process' +import { createHash } from 'crypto' +import { readdir, stat } from 'fs/promises' +import { basename, dirname, join } from 'path' +import { QueueService } from '#services/queue_service' +import logger from '@adonisjs/core/services/logger' +import { DownloadProgressData } from '../../types/downloads.js' +import { PMTILES_BINARY_PATH, buildPmtilesExtractArgs } from '../../constants/map_regions.js' +import { deleteFileIfExists } from '../utils/fs.js' + +export interface RunExtractPmtilesJobParams { + sourceUrl: string + outputFilepath: string + /** Path to a GeoJSON FeatureCollection file passed to `pmtiles extract --region`. */ + regionFilepath: string + maxzoom?: number + /** Hint for progress reporting; obtained from `pmtiles extract --dry-run` preflight */ + estimatedBytes?: number + filetype: 'map' + title?: string + resourceMetadata?: { + resource_id: string + version: string + collection_ref: string | null + } +} + +export class RunExtractPmtilesJob { + static get queue() { + return 'pmtiles-extract' + } + + static get key() { + return 'run-pmtiles-extract' + } + + /** In-memory registry of active child processes so in-process cancels can SIGTERM them */ + static childProcesses: Map = new Map() + + static getJobId(sourceUrl: string, regionFilepath: string, maxzoom?: number): string { + const payload = JSON.stringify({ sourceUrl, regionFilepath, maxzoom: maxzoom ?? null }) + return createHash('sha256').update(payload).digest('hex').slice(0, 16) + } + + /** Redis key used to signal cancellation across processes */ + static cancelKey(jobId: string): string { + return `nomad:download:pmtiles-cancel:${jobId}` + } + + static async signalCancel(jobId: string): Promise { + const queueService = new QueueService() + const queue = queueService.getQueue(this.queue) + const client = await queue.client + await client.set(this.cancelKey(jobId), '1', 'EX', 300) + } + + /** Awaits job.updateProgress and swallows BullMQ stale-job errors (code -1), + * which occur when the job was removed from Redis (e.g. cancelled) between + * the await being issued and the Redis write completing. Anything else + * re-throws so it's caught by the surrounding try rather than becoming an + * unhandled rejection. */ + private async safeUpdateProgress(job: Job, progress: DownloadProgressData): Promise { + try { + await job.updateProgress(progress) + } catch (err: any) { + if (err?.code !== -1) throw err + } + } + + async handle(job: Job) { + const params = job.data as RunExtractPmtilesJobParams + const { sourceUrl, outputFilepath, regionFilepath, maxzoom, estimatedBytes } = params + + logger.info( + `[RunExtractPmtilesJob] Starting extract: source=${sourceUrl} region=${regionFilepath} ` + + `maxzoom=${maxzoom ?? 'source-max'} out=${outputFilepath}` + ) + + const queueService = new QueueService() + const cancelRedis = await queueService.getQueue(RunExtractPmtilesJob.queue).client + + let userCancelled = false + let proc: ChildProcess | null = null + let lastReportedBytes = -1 + + // One 2s tick polls the Redis cancel signal and reads file-size for progress. pmtiles + // writes incrementally but rewrites directories near the end so progress isn't strictly + // monotonic — we cap at 99% and skip emit when bytes are unchanged to avoid Redis chatter. + const tick = setInterval(async () => { + try { + const val = await cancelRedis.get(RunExtractPmtilesJob.cancelKey(job.id!)) + if (val) { + await cancelRedis.del(RunExtractPmtilesJob.cancelKey(job.id!)) + userCancelled = true + proc?.kill('SIGTERM') + } + } catch { + // Redis errors non-fatal — in-memory handle also covers same-process cancels + } + + try { + const fileStat = await stat(outputFilepath) + const downloadedBytes = Number(fileStat.size) + if (downloadedBytes === lastReportedBytes) return + lastReportedBytes = downloadedBytes + + const totalBytes = estimatedBytes ?? 0 + const percent = + totalBytes > 0 ? Math.min(99, Math.floor((downloadedBytes / totalBytes) * 100)) : 0 + + await this.safeUpdateProgress(job, { + percent, + downloadedBytes, + totalBytes, + lastProgressTime: Date.now(), + } as DownloadProgressData) + } catch { + // File doesn't exist yet (subprocess still setting up) + } + }, 2000) + + try { + const args = buildPmtilesExtractArgs({ + sourceUrl, + outputFilepath, + regionFilepath, + maxzoom, + downloadThreads: 8, + overfetch: 0.2, + }) + proc = spawn(PMTILES_BINARY_PATH, args, { stdio: ['ignore', 'pipe', 'pipe'] }) + RunExtractPmtilesJob.childProcesses.set(job.id!, proc) + + proc.stdout?.on('data', (chunk) => { + logger.debug(`[RunExtractPmtilesJob:${job.id}] ${chunk.toString().trimEnd()}`) + }) + proc.stderr?.on('data', (chunk) => { + logger.debug(`[RunExtractPmtilesJob:${job.id}] ${chunk.toString().trimEnd()}`) + }) + + const exitCode: number = await new Promise((resolve, reject) => { + proc!.on('close', (code) => resolve(code ?? -1)) + proc!.on('error', (err) => reject(err)) + }) + + if (exitCode !== 0) { + await deleteFileIfExists(outputFilepath) + if (userCancelled) { + throw new UnrecoverableError(`Extract cancelled by user (exit ${exitCode})`) + } + throw new Error(`pmtiles extract exited with code ${exitCode}`) + } + + // Final progress bump — tick caps at 99 so the UI doesn't flicker to 100 mid-extract + const finalStat = await stat(outputFilepath) + await this.safeUpdateProgress(job, { + percent: 100, + downloadedBytes: Number(finalStat.size), + totalBytes: estimatedBytes ?? Number(finalStat.size), + lastProgressTime: Date.now(), + } as DownloadProgressData) + + // Reuse the HTTP download path's post-download hook so the file is registered and + // the previous version (if any) is deleted + await this.onComplete(params) + + logger.info( + `[RunExtractPmtilesJob] Completed extract: out=${outputFilepath} size=${finalStat.size} bytes` + ) + + return { sourceUrl, outputFilepath } + } catch (error: any) { + if (userCancelled && !(error instanceof UnrecoverableError)) { + throw new UnrecoverableError(`Extract cancelled: ${error.message ?? error}`) + } + throw error + } finally { + clearInterval(tick) + RunExtractPmtilesJob.childProcesses.delete(job.id!) + } + } + + private async onComplete(params: RunExtractPmtilesJobParams) { + if (!params.resourceMetadata) return + + const [{ default: InstalledResource }, { DateTime }, fsUtils] = await Promise.all([ + import('#models/installed_resource'), + import('luxon'), + import('../utils/fs.js'), + ]) + + const fileStat = await fsUtils.getFileStatsIfExists(params.outputFilepath) + + const existing = await InstalledResource.query() + .where('resource_id', params.resourceMetadata.resource_id) + .where('resource_type', 'map') + .first() + const oldFilePath = existing?.file_path ?? null + + await InstalledResource.updateOrCreate( + { + resource_id: params.resourceMetadata.resource_id, + resource_type: 'map', + }, + { + version: params.resourceMetadata.version, + collection_ref: params.resourceMetadata.collection_ref, + url: params.sourceUrl, + file_path: params.outputFilepath, + file_size_bytes: fileStat ? Number(fileStat.size) : null, + installed_at: DateTime.now(), + } + ) + + if (oldFilePath && oldFilePath !== params.outputFilepath) { + try { + await fsUtils.deleteFileIfExists(oldFilePath) + } catch (err) { + logger.warn(`[RunExtractPmtilesJob] Failed to delete old file ${oldFilePath}: ${err}`) + } + } + + // Fallback: scan the pmtiles dir for orphans with the same resource_id that the DB + // lookup above didn't catch — e.g. a prior extract crashed before writing its + // InstalledResource row, or an earlier bug wrote a file without registering it. + // Matches both curated (`_YYYY-MM.pmtiles`) and regional (`_YYYYMMDD_zN.pmtiles`) + // naming — prefix-only so new filename formats don't silently miss. + const dir = dirname(params.outputFilepath) + const keepName = basename(params.outputFilepath) + const prefix = `${params.resourceMetadata.resource_id}_` + try { + const entries = await readdir(dir) + for (const entry of entries) { + if (entry === keepName || !entry.endsWith('.pmtiles')) continue + if (!entry.startsWith(prefix)) continue + const orphanPath = join(dir, entry) + if (orphanPath === oldFilePath) continue + try { + await fsUtils.deleteFileIfExists(orphanPath) + logger.info(`[RunExtractPmtilesJob] Pruned orphan pmtiles ${orphanPath}`) + } catch (err) { + logger.warn(`[RunExtractPmtilesJob] Failed to prune orphan ${orphanPath}: ${err}`) + } + } + } catch (err) { + logger.warn(`[RunExtractPmtilesJob] Directory scan for orphans failed: ${err}`) + } + } + + static async getById(jobId: string): Promise { + const queueService = new QueueService() + const queue = queueService.getQueue(this.queue) + return await queue.getJob(jobId) + } + + static async dispatch(params: RunExtractPmtilesJobParams) { + const queueService = new QueueService() + const queue = queueService.getQueue(this.queue) + const jobId = this.getJobId(params.sourceUrl, params.regionFilepath, params.maxzoom) + + const existing = await queue.getJob(jobId) + if (existing) { + const state = await existing.getState() + if (state === 'active' || state === 'waiting' || state === 'delayed') { + return { + job: existing, + created: false, + message: `Extract job already exists for these params`, + } + } + // Stale (completed/failed) — remove so we can re-dispatch under the same deterministic id + try { + await existing.remove() + } catch { + // Already gone or locked — add() below will still report a meaningful error + } + } + + // Fewer attempts than HTTP downloads — a failed extract usually means the source URL + // rotated or the CDN is throttling, and resuming mid-extract isn't supported by the CLI + const job = await queue.add(this.key, params, { + jobId, + attempts: 3, + backoff: { type: 'exponential', delay: 60000 }, + removeOnComplete: true, + }) + return { + job, + created: true, + message: `Dispatched pmtiles extract job`, + } + } +} diff --git a/admin/app/services/countries_service.ts b/admin/app/services/countries_service.ts new file mode 100644 index 00000000..b6ebf0fc --- /dev/null +++ b/admin/app/services/countries_service.ts @@ -0,0 +1,308 @@ +import { access, readFile, writeFile, mkdir } from 'fs/promises' +import { join, resolve } from 'path' +import { createHash } from 'crypto' +import { tmpdir } from 'os' +import logger from '@adonisjs/core/services/logger' +import type { Country, CountryCode, CountryGroup } from '../../types/maps.js' + +interface NEFeature { + type: 'Feature' + properties: Record + geometry: unknown +} + +interface NEFeatureCollection { + type: 'FeatureCollection' + features: NEFeature[] +} + +const COUNTRY_GEOJSON_PATH = join( + process.cwd(), + 'resources', + 'geodata', + 'ne_50m_admin_0_countries.geojson' +) + +// Natural Earth country polygons are land-only (no territorial waters), so a +// strict intersect leaves tiles fully over the ocean out of the extract — +// coastal cities render as grey off their coast. Inflate each polygon outward +// by ~11 km to pull in adjacent tiles without ballooning the extract size. +const REGION_BUFFER_DEGREES = 0.1 + +const GROUP_ORDER = [ + 'north-america', + 'south-america', + 'europe', + 'africa', + 'asia', + 'oceania', +] + +const GROUP_META: Record = { + 'North America': { + id: 'north-america', + name: 'North America', + description: 'All countries in North America and the Caribbean.', + }, + 'South America': { + id: 'south-america', + name: 'South America', + description: 'All countries in South America.', + }, + Europe: { + id: 'europe', + name: 'Europe', + description: 'All countries in Europe.', + }, + Africa: { + id: 'africa', + name: 'Africa', + description: 'All countries in Africa.', + }, + Asia: { + id: 'asia', + name: 'Asia', + description: 'All countries in Asia.', + }, + Oceania: { + id: 'oceania', + name: 'Oceania', + description: 'Australia, New Zealand, and Pacific island nations.', + }, +} + +export class CountriesService { + private static instance: CountriesService | null = null + private loadPromise: Promise | null = null + private countries: Country[] = [] + private byCode: Map = new Map() + private groups: CountryGroup[] = [] + + static getInstance(): CountriesService { + if (!this.instance) { + this.instance = new CountriesService() + } + return this.instance + } + + private async ensureLoaded(): Promise { + if (this.byCode.size > 0) return + if (!this.loadPromise) { + this.loadPromise = this.load() + } + await this.loadPromise + } + + private async load(): Promise { + const raw = await readFile(COUNTRY_GEOJSON_PATH, 'utf8') + const fc = JSON.parse(raw) as NEFeatureCollection + + // Natural Earth reuses a sovereign state's ISO_A2 for its dependencies + // (e.g. AU covers both Australia and Australian territories). Sort so the + // sovereign mainland wins the ISO-code slot, and skip any subsequent + // same-code dependency — otherwise the "AU" entry ends up being some tiny + // island territory. + const sortedFeatures = [...fc.features].sort((a, b) => typeRank(a) - typeRank(b)) + + const countries: Country[] = [] + const byCode = new Map() + const groupCodes: Record = {} + + for (const feature of sortedFeatures) { + const p = feature.properties + const code = resolveIso2(p) + if (!code) continue + if (byCode.has(code)) continue + + const continent = typeof p.CONTINENT === 'string' ? p.CONTINENT : 'Other' + if (continent === 'Antarctica' || continent === 'Seven seas (open ocean)') continue + + const country: Country = { + code, + code3: resolveIso3(p) ?? code, + name: typeof p.NAME === 'string' ? p.NAME : code, + continent, + subregion: typeof p.SUBREGION === 'string' ? p.SUBREGION : continent, + population: typeof p.POP_EST === 'number' ? p.POP_EST : 0, + } + + countries.push(country) + byCode.set(code, { country, feature }) + + if (GROUP_META[continent]) { + const groupId = GROUP_META[continent].id + if (!groupCodes[groupId]) groupCodes[groupId] = [] + groupCodes[groupId].push(code) + } + } + + countries.sort((a, b) => a.name.localeCompare(b.name)) + + const groups: CountryGroup[] = GROUP_ORDER.flatMap((groupId) => { + const meta = Object.values(GROUP_META).find((m) => m.id === groupId) + if (!meta) return [] + const codes = (groupCodes[groupId] ?? []).slice().sort() + if (codes.length === 0) return [] + return [{ id: meta.id, name: meta.name, description: meta.description, countries: codes }] + }) + + this.countries = countries + this.byCode = byCode + this.groups = groups + + logger.info( + `[CountriesService] Loaded ${countries.length} countries across ${groups.length} groups` + ) + } + + async list(): Promise { + await this.ensureLoaded() + return this.countries + } + + async listGroups(): Promise { + await this.ensureLoaded() + return this.groups + } + + /** Throws when a supplied code does not map to a known country. */ + async resolveCodes(codes: CountryCode[]): Promise { + await this.ensureLoaded() + const normalized = [...new Set(codes.map((c) => c.toUpperCase()))].sort() + const unknown = normalized.filter((c) => !this.byCode.has(c)) + if (unknown.length > 0) { + throw new Error(`Unknown country code(s): ${unknown.join(', ')}`) + } + return normalized + } + + /** + * Filename is keyed on a hash of the sorted ISO codes + buffer size so + * repeated calls with the same selection reuse the same path, and bumping + * the buffer auto-invalidates stale files. + */ + async writeRegionFile(codes: CountryCode[]): Promise { + await this.ensureLoaded() + const resolved = await this.resolveCodes(codes) + const key = `b${REGION_BUFFER_DEGREES}:${resolved.join(',')}` + const hash = createHash('sha1').update(key).digest('hex').slice(0, 12) + + const dir = resolve(tmpdir(), 'nomad-pmtiles-regions') + await mkdir(dir, { recursive: true }) + const filepath = join(dir, `region-${hash}.geojson`) + + try { + await access(filepath) + return filepath + } catch {} + + const fc = { + type: 'FeatureCollection', + features: resolved.map((code) => { + const entry = this.byCode.get(code)! + return { + type: 'Feature', + properties: { iso: code, name: entry.country.name }, + geometry: bufferGeometry(entry.feature.geometry, REGION_BUFFER_DEGREES), + } + }), + } + + await writeFile(filepath, JSON.stringify(fc)) + return filepath + } +} + +function typeRank(f: NEFeature): number { + const t = typeof f.properties.TYPE === 'string' ? f.properties.TYPE : '' + if (t === 'Sovereign country') return 0 + if (t === 'Country') return 1 + if (t === 'Sovereignty') return 2 + if (t === 'Disputed') return 3 + if (t === 'Dependency') return 4 + return 5 +} + +function resolveIso2(p: Record): CountryCode | null { + // Natural Earth's ISO_A2 sometimes holds political escapes like "CN-TW" for + // Taiwan or "-99" for countries involved in disputes. Only accept clean + // 2-letter codes; fall back to ISO_A2_EH (which reliably has the real code). + const primary = typeof p.ISO_A2 === 'string' ? p.ISO_A2 : null + if (primary && /^[A-Z]{2}$/i.test(primary)) return primary.toUpperCase() + const fallback = typeof p.ISO_A2_EH === 'string' ? p.ISO_A2_EH : null + if (fallback && /^[A-Z]{2}$/i.test(fallback)) return fallback.toUpperCase() + return null +} + +/** + * Inflate each polygon ring outward by `buffer` degrees via per-vertex + * averaged-normal offset. Not geodesically accurate — but at small buffers + * (<= 0.2°) it's within a few percent of a proper geodesic buffer at + * country scale, which is plenty for tile-inclusion purposes. + */ +function bufferGeometry(geometry: unknown, buffer: number): unknown { + const geom = geometry as { type: string; coordinates: any } + if (geom?.type === 'Polygon') { + return { type: 'Polygon', coordinates: bufferPolygonRings(geom.coordinates, buffer) } + } + if (geom?.type === 'MultiPolygon') { + return { + type: 'MultiPolygon', + coordinates: geom.coordinates.map((poly: number[][][]) => + bufferPolygonRings(poly, buffer) + ), + } + } + return geometry +} + +function bufferPolygonRings(rings: number[][][], buffer: number): number[][][] { + return rings.map((ring) => bufferRing(ring, buffer)) +} + +function bufferRing(ring: number[][], buffer: number): number[][] { + if (ring.length < 4) return ring + const sign = signedArea(ring) > 0 ? 1 : -1 + const n = ring.length - 1 + const out: number[][] = [] + for (let i = 0; i < n; i++) { + const prev = ring[(i - 1 + n) % n] + const curr = ring[i] + const next = ring[(i + 1) % n] + const e1x = curr[0] - prev[0] + const e1y = curr[1] - prev[1] + const e2x = next[0] - curr[0] + const e2y = next[1] - curr[1] + const l1 = Math.hypot(e1x, e1y) || 1 + const l2 = Math.hypot(e2x, e2y) || 1 + const n1x = (e1y / l1) * sign + const n1y = (-e1x / l1) * sign + const n2x = (e2y / l2) * sign + const n2y = (-e2x / l2) * sign + const sumX = n1x + n2x + const sumY = n1y + n2y + const sl = Math.hypot(sumX, sumY) || 1 + out.push([curr[0] + (sumX / sl) * buffer, curr[1] + (sumY / sl) * buffer]) + } + out.push(out[0]) + return out +} + +function signedArea(ring: number[][]): number { + let a = 0 + for (let i = 0; i < ring.length - 1; i++) { + a += ring[i][0] * ring[i + 1][1] - ring[i + 1][0] * ring[i][1] + } + return a / 2 +} + +function resolveIso3(p: Record): string | null { + const primary = typeof p.ISO_A3 === 'string' ? p.ISO_A3 : null + if (primary && primary !== '-99') return primary.toUpperCase() + const fallback = typeof p.ISO_A3_EH === 'string' ? p.ISO_A3_EH : null + if (fallback && fallback !== '-99') return fallback.toUpperCase() + const adm = typeof p.ADM0_A3 === 'string' ? p.ADM0_A3 : null + if (adm && adm !== '-99') return adm.toUpperCase() + return null +} + diff --git a/admin/app/services/download_service.ts b/admin/app/services/download_service.ts index bd9076c6..91b8288b 100644 --- a/admin/app/services/download_service.ts +++ b/admin/app/services/download_service.ts @@ -1,13 +1,19 @@ import { inject } from '@adonisjs/core' import { QueueService } from './queue_service.js' import { RunDownloadJob } from '#jobs/run_download_job' +import { RunExtractPmtilesJob } from '#jobs/run_extract_pmtiles_job' +import type { RunExtractPmtilesJobParams } from '#jobs/run_extract_pmtiles_job' import { DownloadModelJob } from '#jobs/download_model_job' import { DownloadJobWithProgress, DownloadProgressData } from '../../types/downloads.js' +import type { Job, Queue } from 'bullmq' import { normalize } from 'path' import { deleteFileIfExists } from '../utils/fs.js' import transmit from '@adonisjs/transmit/services/main' import { BROADCAST_CHANNELS } from '../../constants/broadcast.js' +type FileJobState = 'waiting' | 'active' | 'delayed' | 'failed' +type TaggedJob = { job: Job; state: FileJobState } + @inject() export class DownloadService { constructor(private queueService: QueueService) {} @@ -26,27 +32,32 @@ export class DownloadService { return { percent: parseInt(String(progress), 10) || 0 } } - async listDownloadJobs(filetype?: string): Promise { - // Get regular file download jobs (zim, map, etc.) — query each state separately so we can - // tag each job with its actual BullMQ state rather than guessing from progress data. - const queue = this.queueService.getQueue(RunDownloadJob.queue) - type FileJobState = 'waiting' | 'active' | 'delayed' | 'failed' - - const [waitingJobs, activeJobs, delayedJobs, failedJobs] = await Promise.all([ + /** Fetch all non-completed jobs from a queue, tagged with their current BullMQ state */ + private async fetchJobsWithStates(queueName: string): Promise { + const queue = this.queueService.getQueue(queueName) + const [waiting, active, delayed, failed] = await Promise.all([ queue.getJobs(['waiting']), queue.getJobs(['active']), queue.getJobs(['delayed']), queue.getJobs(['failed']), ]) - - const taggedFileJobs: Array<{ job: (typeof waitingJobs)[0]; state: FileJobState }> = [ - ...waitingJobs.map((j) => ({ job: j, state: 'waiting' as const })), - ...activeJobs.map((j) => ({ job: j, state: 'active' as const })), - ...delayedJobs.map((j) => ({ job: j, state: 'delayed' as const })), - ...failedJobs.map((j) => ({ job: j, state: 'failed' as const })), + return [ + ...waiting.map((j) => ({ job: j, state: 'waiting' as const })), + ...active.map((j) => ({ job: j, state: 'active' as const })), + ...delayed.map((j) => ({ job: j, state: 'delayed' as const })), + ...failed.map((j) => ({ job: j, state: 'failed' as const })), ] + } + + async listDownloadJobs(filetype?: string): Promise { + const modelQueue = this.queueService.getQueue(DownloadModelJob.queue) + const [fileTagged, extractTagged, modelJobs] = await Promise.all([ + this.fetchJobsWithStates(RunDownloadJob.queue), + this.fetchJobsWithStates(RunExtractPmtilesJob.queue), + modelQueue.getJobs(['waiting', 'active', 'delayed', 'failed']), + ]) - const fileDownloads = taggedFileJobs.map(({ job, state }) => { + const fileDownloads = fileTagged.map(({ job, state }) => { const parsed = this.parseProgress(job.progress) return { jobId: job.id!.toString(), @@ -63,26 +74,36 @@ export class DownloadService { } }) - // Get Ollama model download jobs - const modelQueue = this.queueService.getQueue(DownloadModelJob.queue) - const modelJobs = await modelQueue.getJobs(['waiting', 'active', 'delayed', 'failed']) + const extractDownloads = extractTagged.map(({ job, state }) => { + const parsed = this.parseProgress(job.progress) + return { + jobId: job.id!.toString(), + url: job.data.sourceUrl, + progress: parsed.percent, + filepath: normalize(job.data.outputFilepath), + filetype: job.data.filetype || 'map', + title: job.data.title || undefined, + downloadedBytes: parsed.downloadedBytes, + totalBytes: parsed.totalBytes || job.data.estimatedBytes || undefined, + lastProgressTime: parsed.lastProgressTime, + status: state, + failedReason: job.failedReason || undefined, + } + }) const modelDownloads = modelJobs.map((job) => ({ jobId: job.id!.toString(), - url: job.data.modelName || 'Unknown Model', // Use model name as url + url: job.data.modelName || 'Unknown Model', progress: parseInt(job.progress.toString(), 10), - filepath: job.data.modelName || 'Unknown Model', // Use model name as filepath + filepath: job.data.modelName || 'Unknown Model', filetype: 'model', status: (job.failedReason ? 'failed' : 'active') as 'active' | 'failed', failedReason: job.failedReason || undefined, })) - const allDownloads = [...fileDownloads, ...modelDownloads] - - // Filter by filetype if specified + const allDownloads = [...fileDownloads, ...extractDownloads, ...modelDownloads] const filtered = allDownloads.filter((job) => !filetype || job.filetype === filetype) - // Sort: active downloads first (by progress desc), then failed at the bottom return filtered.sort((a, b) => { if (a.status === 'failed' && b.status !== 'failed') return 1 if (a.status !== 'failed' && b.status === 'failed') return -1 @@ -91,7 +112,11 @@ export class DownloadService { } async removeFailedJob(jobId: string): Promise { - for (const queueName of [RunDownloadJob.queue, DownloadModelJob.queue]) { + for (const queueName of [ + RunDownloadJob.queue, + RunExtractPmtilesJob.queue, + DownloadModelJob.queue, + ]) { const queue = this.queueService.getQueue(queueName) const job = await queue.getJob(jobId) if (job) { @@ -113,7 +138,6 @@ export class DownloadService { } async cancelJob(jobId: string): Promise<{ success: boolean; message: string }> { - // Try the file download queue first (the original PR #554 path) const queue = this.queueService.getQueue(RunDownloadJob.queue) const job = await queue.getJob(jobId) @@ -121,7 +145,13 @@ export class DownloadService { return await this._cancelFileDownloadJob(jobId, job, queue) } - // Fall through to the model download queue + const extractQueue = this.queueService.getQueue(RunExtractPmtilesJob.queue) + const extractJob = await extractQueue.getJob(jobId) + + if (extractJob) { + return await this._cancelExtractJob(jobId, extractJob, extractQueue) + } + const modelQueue = this.queueService.getQueue(DownloadModelJob.queue) const modelJob = await modelQueue.getJob(jobId) @@ -129,11 +159,37 @@ export class DownloadService { return await this._cancelModelDownloadJob(jobId, modelJob, modelQueue) } - // Not found in either queue return { success: true, message: 'Job not found (may have already completed)' } } - /** Cancel a content download (zim, map, pmtiles, etc.) — original PR #554 logic */ + private async _cancelExtractJob( + jobId: string, + job: Job, + queue: Queue + ): Promise<{ success: boolean; message: string }> { + const outputFilepath = job.data.outputFilepath + + await RunExtractPmtilesJob.signalCancel(jobId) + + // Same-process fallback when worker and API share a process + RunExtractPmtilesJob.childProcesses.get(jobId)?.kill('SIGTERM') + RunExtractPmtilesJob.childProcesses.delete(jobId) + + await this._pollForTerminalState(job, jobId) + await this._removeJobWithLockFallback(job, queue, RunExtractPmtilesJob.queue, jobId) + + if (outputFilepath) { + try { + await deleteFileIfExists(outputFilepath) + } catch { + // File may not exist yet (subprocess may not have opened it) + } + } + + return { success: true, message: 'Extract cancelled and partial file deleted' } + } + + /** Cancel a content download (zim, map, pmtiles, etc.) */ private async _cancelFileDownloadJob( jobId: string, job: any, diff --git a/admin/app/services/map_service.ts b/admin/app/services/map_service.ts index c9902a3c..c1157b0e 100644 --- a/admin/app/services/map_service.ts +++ b/admin/app/services/map_service.ts @@ -16,11 +16,35 @@ import { import { join, resolve, sep } from 'path' import urlJoin from 'url-join' import { RunDownloadJob } from '#jobs/run_download_job' +import { RunExtractPmtilesJob } from '#jobs/run_extract_pmtiles_job' import logger from '@adonisjs/core/services/logger' import { assertNotPrivateUrl } from '#validators/common' import InstalledResource from '#models/installed_resource' import { CollectionManifestService } from './collection_manifest_service.js' import type { CollectionWithStatus, MapsSpec } from '../../types/collections.js' +import type { Country, CountryCode, CountryGroup, MapExtractPreflight } from '../../types/maps.js' +import { + EXTRACT_DEFAULT_MAX_ZOOM, + EXTRACT_MAX_ZOOM, + EXTRACT_MIN_ZOOM, + PMTILES_BINARY_PATH, + WORLD_BASEMAP_FILENAME, + WORLD_BASEMAP_MAX_ZOOM, + WORLD_BASEMAP_SOURCE_NAME, + buildPmtilesExtractArgs, +} from '../../constants/map_regions.js' +import { CountriesService } from './countries_service.js' +import { execFile } from 'child_process' +import { createHash, randomBytes } from 'crypto' +import { tmpdir } from 'os' +import { promisify } from 'util' + +const execFileAsync = promisify(execFile) +const DRY_RUN_TIMEOUT_MS = 60_000 +const DRY_RUN_MAX_BUFFER = 256 * 1024 +// Real extract of z0-5 world tiles; generous to tolerate slow/metered links +// since a failure leaves the map grey for uncovered regions. +const WORLD_BASEMAP_EXTRACT_TIMEOUT_MS = 5 * 60_000 const PROTOMAPS_BUILDS_METADATA_URL = 'https://build-metadata.protomaps.dev/builds.json' const PROTOMAPS_BUILD_BASE_URL = 'https://build.protomaps.com' @@ -53,10 +77,15 @@ export class MapService implements IMapService { private readonly baseAssetsTarFile = 'base-assets.tar.gz' private readonly baseDirPath = join(process.cwd(), this.mapStoragePath) private baseAssetsExistCache: boolean | null = null + private worldBasemapReady = false + private worldBasemapInFlight: Promise | null = null async listRegions() { const files = (await this.listAllMapStorageItems()).filter( - (item) => item.type === 'file' && item.name.endsWith('.pmtiles') + (item) => + item.type === 'file' && + item.name.endsWith('.pmtiles') && + item.name !== WORLD_BASEMAP_FILENAME ) return { @@ -327,11 +356,76 @@ export class MapService implements IMapService { async ensureBaseAssets(): Promise { const exists = await this.checkBaseAssetsExist() - if (exists) { - return true + if (!exists) { + const downloaded = await this.downloadBaseAssets() + if (!downloaded) return false + } + + try { + await this.ensureWorldBasemap() + } catch (err) { + logger.warn(`[MapService] World basemap setup failed, continuing without it: ${err}`) + } + + return true + } + + /** + * Extract a low-zoom global basemap once so the map isn't grey outside a + * regional extract's polygon. Cheap (~15 MB, a handful of HTTP range + * requests) and layered underneath regional sources at render time. + * + * Memoizes success in-process, and de-duplicates concurrent callers via a + * shared in-flight promise so two simultaneous `/maps` requests on a cold + * start don't both launch `pmtiles extract` against the same output path. + */ + private async ensureWorldBasemap(): Promise { + if (this.worldBasemapReady) return + if (this.worldBasemapInFlight) return this.worldBasemapInFlight + this.worldBasemapInFlight = this._setupWorldBasemap().finally(() => { + this.worldBasemapInFlight = null + }) + return this.worldBasemapInFlight + } + + private async _setupWorldBasemap(): Promise { + const basePath = resolve(join(this.baseDirPath, 'pmtiles')) + const filepath = resolve(join(basePath, WORLD_BASEMAP_FILENAME)) + if (!filepath.startsWith(basePath + sep)) { + throw new Error('Invalid world basemap path') + } + + await ensureDirectoryExists(basePath) + + const existing = await getFileStatsIfExists(filepath) + if (existing && Number(existing.size) > 0) { + this.worldBasemapReady = true + return } - return await this.downloadBaseAssets() + const info = await this.getGlobalMapInfo() + const args = buildPmtilesExtractArgs({ + sourceUrl: info.url, + outputFilepath: filepath, + maxzoom: WORLD_BASEMAP_MAX_ZOOM, + downloadThreads: 4, + }) + + logger.info( + `[MapService] Extracting world basemap (z0-${WORLD_BASEMAP_MAX_ZOOM}) from ${info.url}` + ) + try { + await execFileAsync(PMTILES_BINARY_PATH, args, { + timeout: WORLD_BASEMAP_EXTRACT_TIMEOUT_MS, + maxBuffer: DRY_RUN_MAX_BUFFER, + }) + this.worldBasemapReady = true + } catch (err: any) { + await deleteFileIfExists(filepath) + throw new Error( + `pmtiles extract for world basemap failed: ${err.message}. stderr: ${err.stderr ?? ''}` + ) + } } private async checkBaseAssetsExist(useCache: boolean = true): Promise { @@ -367,6 +461,19 @@ export class MapService implements IMapService { const sources: BaseStylesFile['sources'][] = [] const baseUrl = this.getPublicFileBaseUrl(host, 'pmtiles', protocol) + // World basemap goes first so its layers render underneath regional extracts. + // Only emitted when ensureWorldBasemap() succeeded — otherwise the style would + // reference a file that doesn't exist and produce 404s on every tile request. + if (this.worldBasemapReady) { + const worldSource: BaseStylesFile['sources'] = {} + worldSource[WORLD_BASEMAP_SOURCE_NAME] = { + type: 'vector', + attribution: PMTILES_ATTRIBUTION, + url: `pmtiles://${urlJoin(baseUrl, WORLD_BASEMAP_FILENAME)}`, + } + sources.push(worldSource) + } + for (const region of regions) { if (region.type === 'file' && region.name.endsWith('.pmtiles')) { // Strip .pmtiles and date suffix (e.g. "alaska_2025-12" -> "alaska") for stable source names @@ -489,12 +596,206 @@ export class MapService implements IMapService { } } + async listCountries(): Promise { + return CountriesService.getInstance().list() + } + + async listCountryGroups(): Promise { + return CountriesService.getInstance().listGroups() + } + + async extractPreflight(params: { + countries: CountryCode[] + maxzoom?: number + }): Promise { + this.validateMaxzoom(params.maxzoom) + const countries = await CountriesService.getInstance().resolveCodes(params.countries) + const regionFilepath = await CountriesService.getInstance().writeRegionFile(countries) + const info = await this.getGlobalMapInfo() + return this.runDryRun(info, regionFilepath, params.maxzoom) + } + + private async runDryRun( + info: { url: string; date: string; key: string }, + regionFilepath: string, + maxzoom?: number + ): Promise { + const dryRunOutput = join(tmpdir(), `pmtiles-dry-run-${randomBytes(6).toString('hex')}.pmtiles`) + const args = buildPmtilesExtractArgs({ + sourceUrl: info.url, + outputFilepath: dryRunOutput, + regionFilepath, + maxzoom, + dryRun: true, + }) + + let stdout = '' + let stderr = '' + try { + const result = await execFileAsync(PMTILES_BINARY_PATH, args, { + timeout: DRY_RUN_TIMEOUT_MS, + maxBuffer: DRY_RUN_MAX_BUFFER, + }) + stdout = result.stdout + stderr = result.stderr + } catch (err: any) { + throw new Error( + `pmtiles extract --dry-run failed: ${err.message}. stderr: ${err.stderr ?? ''}` + ) + } + + const parsed = this.parseDryRunOutput(stdout + '\n' + stderr) + + return { + tiles: parsed.tiles, + bytes: parsed.bytes, + source: { url: info.url, date: info.date, key: info.key }, + } + } + + async extractRegion(params: { + countries: CountryCode[] + maxzoom?: number + label?: string + estimatedBytes?: number + }): Promise<{ filename: string; jobId?: string }> { + this.validateMaxzoom(params.maxzoom) + const countriesService = CountriesService.getInstance() + const countries = await countriesService.resolveCodes(params.countries) + const regionFilepath = await countriesService.writeRegionFile(countries) + const maxzoom = params.maxzoom ?? EXTRACT_DEFAULT_MAX_ZOOM + + const [baseAssetsExist, info, groups] = await Promise.all([ + this.ensureBaseAssets(), + this.getGlobalMapInfo(), + countriesService.listGroups(), + ]) + if (!baseAssetsExist) { + throw new Error( + 'Base map assets are missing and could not be downloaded. Please check your connection and try again.' + ) + } + + const groupMatch = findExactGroupMatch(countries, groups) + const slug = this.buildRegionSlug(countries, groupMatch) + const dateSlug = info.key.replace('.pmtiles', '') + const filename = `${slug}_${dateSlug}_z${maxzoom}.pmtiles` + const basePath = resolve(join(this.baseDirPath, 'pmtiles')) + const filepath = resolve(join(basePath, filename)) + + if (!filepath.startsWith(basePath + sep)) { + throw new Error('Invalid filename') + } + + let estimatedBytes = params.estimatedBytes ?? 0 + if (estimatedBytes === 0) { + try { + const preflight = await this.runDryRun(info, regionFilepath, maxzoom) + estimatedBytes = preflight.bytes + } catch (err) { + logger.warn(`[MapService] extractRegion preflight failed, proceeding without estimate: ${err}`) + } + } + + const title = params.label ?? this.buildRegionTitle(countries, groupMatch) + + const result = await RunExtractPmtilesJob.dispatch({ + sourceUrl: info.url, + outputFilepath: filepath, + regionFilepath, + maxzoom, + estimatedBytes, + filetype: 'map', + title, + resourceMetadata: { + resource_id: slug, + version: dateSlug, + collection_ref: null, + }, + }) + + if (!result.job) { + throw new Error('Failed to dispatch extract job') + } + + logger.info( + `[MapService] Dispatched extract job ${result.job.id} for ${filename} ` + + `(countries=[${countries.join(',')}] maxzoom=${maxzoom} est=${estimatedBytes} bytes)` + ) + + return { + filename, + jobId: result.job.id, + } + } + + private buildRegionSlug(countries: CountryCode[], groupMatch: CountryGroup | null): string { + if (groupMatch) return groupMatch.id + if (countries.length === 1) return countries[0].toLowerCase() + const hash = createHash('sha1').update(countries.join(',')).digest('hex').slice(0, 8) + return `custom-${hash}` + } + + private buildRegionTitle(countries: CountryCode[], groupMatch: CountryGroup | null): string { + if (groupMatch) return groupMatch.name + if (countries.length === 1) return countries[0] + if (countries.length <= 3) return countries.join(', ') + return `${countries.slice(0, 2).join(', ')} +${countries.length - 2} more` + } + + private validateMaxzoom(maxzoom: number | undefined): void { + if (typeof maxzoom !== 'number') return + if ( + !Number.isInteger(maxzoom) || + maxzoom < EXTRACT_MIN_ZOOM || + maxzoom > EXTRACT_MAX_ZOOM + ) { + throw new Error( + `maxzoom must be an integer in [${EXTRACT_MIN_ZOOM}, ${EXTRACT_MAX_ZOOM}]` + ) + } + } + + // go-pmtiles output format isn't stable across versions — parse loosely and + // fall back to zeros. The extract can still proceed without an estimate. + private parseDryRunOutput(output: string): { tiles: number; bytes: number } { + let bytes = 0 + let tiles = 0 + + const byteLine = output.match(/archive\s+size\s+of\s+([\d,.]+)\s*(B|KB|MB|GB|TB|bytes?)?/i) + if (byteLine) { + const raw = parseFloat(byteLine[1].replace(/,/g, '')) + const unit = (byteLine[2] ?? 'B').toUpperCase() + const multipliers: Record = { + B: 1, + BYTE: 1, + BYTES: 1, + KB: 1_000, + MB: 1_000_000, + GB: 1_000_000_000, + TB: 1_000_000_000_000, + } + bytes = Math.round(raw * (multipliers[unit] ?? 1)) + } + + const tileLine = output.match(/(?:tiles\s+to\s+extract|tiles)[^\d]*([\d,]+)/i) + if (tileLine) { + tiles = parseInt(tileLine[1].replace(/,/g, ''), 10) || 0 + } + + return { tiles, bytes } + } + async delete(file: string): Promise { let fileName = file if (!fileName.endsWith('.pmtiles')) { fileName += '.pmtiles' } + if (fileName === WORLD_BASEMAP_FILENAME) { + throw new Error('The world basemap cannot be deleted') + } + const basePath = resolve(join(this.baseDirPath, 'pmtiles')) const fullPath = resolve(join(basePath, fileName)) @@ -573,3 +874,16 @@ export class MapService implements IMapService { return baseUrl } } + +function findExactGroupMatch( + countries: CountryCode[], + groups: CountryGroup[] +): CountryGroup | null { + return ( + groups.find( + (g) => + g.countries.length === countries.length && + g.countries.every((c, i) => c === countries[i]) + ) ?? null + ) +} diff --git a/admin/app/validators/common.ts b/admin/app/validators/common.ts index 7065d4ca..ba9f1079 100644 --- a/admin/app/validators/common.ts +++ b/admin/app/validators/common.ts @@ -112,3 +112,31 @@ export const applyAllContentUpdatesValidator = vine.compile( .minLength(1), }) ) + +// --- Map extract (regional pmtiles download) --- + +// ISO 3166-1 alpha-2, 2 letters. Loose regex; CountriesService.resolveCodes +// does the authoritative check against the polygon dataset. +const countryCodeSchema = vine + .string() + .trim() + .toUpperCase() + .regex(/^[A-Z]{2}$/) + +const countriesArraySchema = vine.array(countryCodeSchema).minLength(1).maxLength(300) + +export const mapExtractPreflightValidator = vine.compile( + vine.object({ + countries: countriesArraySchema.clone(), + maxzoom: vine.number().min(0).max(15).optional(), + }) +) + +export const mapExtractValidator = vine.compile( + vine.object({ + countries: countriesArraySchema.clone(), + maxzoom: vine.number().min(0).max(15).optional(), + label: vine.string().trim().minLength(1).maxLength(64).optional(), + estimatedBytes: vine.number().min(0).optional(), + }) +) diff --git a/admin/commands/queue/work.ts b/admin/commands/queue/work.ts index 31bb1cc0..49890e68 100644 --- a/admin/commands/queue/work.ts +++ b/admin/commands/queue/work.ts @@ -3,6 +3,7 @@ import type { CommandOptions } from '@adonisjs/core/types/ace' import { Worker } from 'bullmq' import queueConfig from '#config/queue' import { RunDownloadJob } from '#jobs/run_download_job' +import { RunExtractPmtilesJob } from '#jobs/run_extract_pmtiles_job' import { DownloadModelJob } from '#jobs/download_model_job' import { RunBenchmarkJob } from '#jobs/run_benchmark_job' import { EmbedFileJob } from '#jobs/embed_file_job' @@ -126,6 +127,7 @@ export default class QueueWork extends BaseCommand { const queues = new Map() handlers.set(RunDownloadJob.key, new RunDownloadJob()) + handlers.set(RunExtractPmtilesJob.key, new RunExtractPmtilesJob()) handlers.set(DownloadModelJob.key, new DownloadModelJob()) handlers.set(RunBenchmarkJob.key, new RunBenchmarkJob()) handlers.set(EmbedFileJob.key, new EmbedFileJob()) @@ -133,6 +135,7 @@ export default class QueueWork extends BaseCommand { handlers.set(CheckServiceUpdatesJob.key, new CheckServiceUpdatesJob()) queues.set(RunDownloadJob.key, RunDownloadJob.queue) + queues.set(RunExtractPmtilesJob.key, RunExtractPmtilesJob.queue) queues.set(DownloadModelJob.key, DownloadModelJob.queue) queues.set(RunBenchmarkJob.key, RunBenchmarkJob.queue) queues.set(EmbedFileJob.key, EmbedFileJob.queue) @@ -149,6 +152,9 @@ export default class QueueWork extends BaseCommand { private getConcurrencyForQueue(queueName: string): number { const concurrencyMap: Record = { [RunDownloadJob.queue]: 3, + // pmtiles extract hits the Protomaps CDN with many parallel range reads per job; + // cap concurrency at 2 so a second extract doesn't starve the first. + [RunExtractPmtilesJob.queue]: 2, [DownloadModelJob.queue]: 2, // Lower concurrency for resource-intensive model downloads [RunBenchmarkJob.queue]: 1, // Run benchmarks one at a time for accurate results [EmbedFileJob.queue]: 2, // Lower concurrency for embedding jobs, can be resource intensive diff --git a/admin/constants/map_regions.ts b/admin/constants/map_regions.ts new file mode 100644 index 00000000..75f2922b --- /dev/null +++ b/admin/constants/map_regions.ts @@ -0,0 +1,32 @@ +export const PMTILES_BINARY_PATH = '/usr/local/bin/pmtiles' + +// Clamp these so a user can't ask for nonsense that never extracts +export const EXTRACT_MIN_ZOOM = 0 +export const EXTRACT_MAX_ZOOM = 15 +export const EXTRACT_DEFAULT_MAX_ZOOM = 15 + +// Low-zoom global fallback extracted once during base-asset setup (~15 MB). Layered +// underneath regional extracts so the map isn't grey outside a region's polygon. +export const WORLD_BASEMAP_FILENAME = 'world.pmtiles' +export const WORLD_BASEMAP_MAX_ZOOM = 5 +export const WORLD_BASEMAP_SOURCE_NAME = 'world' + +export interface PmtilesExtractArgOptions { + sourceUrl: string + outputFilepath: string + regionFilepath?: string + maxzoom?: number + dryRun?: boolean + downloadThreads?: number + overfetch?: number +} + +export function buildPmtilesExtractArgs(opts: PmtilesExtractArgOptions): string[] { + const args = ['extract', opts.sourceUrl, opts.outputFilepath] + if (opts.regionFilepath) args.push(`--region=${opts.regionFilepath}`) + if (typeof opts.maxzoom === 'number') args.push(`--maxzoom=${opts.maxzoom}`) + if (opts.dryRun) args.push('--dry-run') + if (typeof opts.downloadThreads === 'number') args.push(`--download-threads=${opts.downloadThreads}`) + if (typeof opts.overfetch === 'number') args.push(`--overfetch=${opts.overfetch}`) + return args +} diff --git a/admin/inertia/components/CountryPickerModal.tsx b/admin/inertia/components/CountryPickerModal.tsx new file mode 100644 index 00000000..b490360c --- /dev/null +++ b/admin/inertia/components/CountryPickerModal.tsx @@ -0,0 +1,384 @@ +import { useEffect, useMemo, useRef, useState } from 'react' +import { useQuery } from '@tanstack/react-query' +import { IconCheck, IconSearch, IconX } from '@tabler/icons-react' +import StyledModal, { StyledModalProps } from './StyledModal' +import LoadingSpinner from './LoadingSpinner' +import api from '~/lib/api' +import { formatBytes } from '~/lib/util' +import classNames from '~/lib/classNames' +import { + EXTRACT_DEFAULT_MAX_ZOOM, + EXTRACT_MAX_ZOOM, + EXTRACT_MIN_ZOOM, +} from '../../constants/map_regions' +import type { + Country, + CountryCode, + CountryGroup, + MapExtractPreflight, +} from '../../types/maps' + +export type CountryPickerModalProps = Omit< + StyledModalProps, + | 'onConfirm' + | 'open' + | 'confirmText' + | 'cancelText' + | 'confirmVariant' + | 'children' + | 'title' + | 'large' +> & { + onDownloadStart?: () => void +} + +const CountryPickerModal: React.FC = ({ + onDownloadStart, + ...modalProps +}) => { + const [selected, setSelected] = useState>(new Set()) + const [search, setSearch] = useState('') + const [maxzoom, setMaxzoom] = useState(EXTRACT_DEFAULT_MAX_ZOOM) + const [preflight, setPreflight] = useState(null) + const [loading, setLoading] = useState(false) + const [downloading, setDownloading] = useState(false) + const [errorMessage, setErrorMessage] = useState(null) + const preflightRequestIdRef = useRef(0) + + const { data: countries = [], isLoading: countriesLoading } = useQuery({ + queryKey: ['maps-countries'], + queryFn: () => api.listCountries(), + staleTime: Infinity, + }) + + const { data: groups = [] } = useQuery({ + queryKey: ['maps-country-groups'], + queryFn: () => api.listCountryGroups(), + staleTime: Infinity, + }) + + const grouped = useMemo(() => { + const q = search.trim().toLowerCase() + const filtered = q + ? countries.filter( + (c) => c.name.toLowerCase().includes(q) || c.code.toLowerCase().includes(q) + ) + : countries + + const buckets: Record = {} + for (const country of filtered) { + if (!buckets[country.continent]) buckets[country.continent] = [] + buckets[country.continent].push(country) + } + return Object.entries(buckets).sort(([a], [b]) => a.localeCompare(b)) + }, [countries, search]) + + const selectedCountries = useMemo( + () => countries.filter((c) => selected.has(c.code)), + [countries, selected] + ) + + function toggleCountry(code: CountryCode) { + setSelected((prev) => { + const next = new Set(prev) + if (next.has(code)) next.delete(code) + else next.add(code) + return next + }) + } + + function toggleGroup(group: CountryGroup) { + setSelected((prev) => { + const next = new Set(prev) + const allIn = group.countries.every((c) => next.has(c)) + if (allIn) { + group.countries.forEach((c) => next.delete(c)) + } else { + group.countries.forEach((c) => next.add(c)) + } + return next + }) + } + + function clearAll() { + setSelected(new Set()) + } + + // Auto-refresh the preflight whenever selection or maxzoom changes. Debounced + // so rapid multi-select clicks collapse into a single CDN round-trip, and + // stale-safe via requestId so an earlier slow response can't clobber a later one. + useEffect(() => { + if (selected.size === 0) { + setPreflight(null) + setErrorMessage(null) + setLoading(false) + preflightRequestIdRef.current++ + return + } + + const requestId = ++preflightRequestIdRef.current + setLoading(true) + setErrorMessage(null) + const timer = setTimeout(async () => { + try { + const res = await api.extractMapPreflight({ + countries: [...selected], + maxzoom, + }) + if (requestId !== preflightRequestIdRef.current) return + if (!res) throw new Error('Preflight returned no data') + setPreflight(res) + } catch (err: any) { + if (requestId !== preflightRequestIdRef.current) return + console.error('Preflight failed:', err) + setErrorMessage(err?.message ?? 'Estimate failed') + } finally { + if (requestId === preflightRequestIdRef.current) setLoading(false) + } + }, 400) + + return () => clearTimeout(timer) + }, [selected, maxzoom]) + + async function startDownload() { + if (selected.size === 0) { + setErrorMessage('Pick at least one country before downloading.') + return + } + if (loading || !preflight) { + setErrorMessage('Still estimating size — hold on a moment.') + return + } + try { + setDownloading(true) + setErrorMessage(null) + await api.extractMapRegion({ + countries: [...selected], + maxzoom, + estimatedBytes: preflight?.bytes, + }) + onDownloadStart?.() + } catch (err: any) { + console.error('Extract dispatch failed:', err) + setErrorMessage(err?.message ?? 'Download failed') + } finally { + setDownloading(false) + } + } + + return ( + +

+
+
+ + setSearch(e.target.value)} + placeholder={`Search ${countries.length} countries...`} + className="w-full pl-9 pr-3 py-2 rounded-md border border-border-default bg-surface-primary text-text-primary text-sm focus:outline-none focus:ring-2 focus:ring-desert-green" + /> +
+ {selected.size > 0 && ( + + )} +
+ + {groups.length > 0 && ( +
+

+ Quick picks +

+
+ {groups.map((group) => { + const allIn = + group.countries.length > 0 && + group.countries.every((c) => selected.has(c)) + return ( + + ) + })} +
+
+ )} + +
+ {countriesLoading ? ( +
+ +
+ ) : grouped.length === 0 ? ( +

+ No countries match "{search}". +

+ ) : ( + grouped.map(([continent, list]) => ( +
+
+ {continent} +
+
    + {list.map((country) => { + const isSelected = selected.has(country.code) + return ( +
  • + +
  • + ) + })} +
+
+ )) + )} +
+ + {selectedCountries.length > 0 && ( +
+

+ {selectedCountries.length} selected +

+
+ {selectedCountries.map((country) => ( + + {country.name} + + + ))} +
+
+ )} + +
+ + setMaxzoom(parseInt(e.target.value, 10))} + className="w-full accent-desert-green" + disabled={loading || downloading} + /> +
+ z{EXTRACT_MIN_ZOOM} (world) + z{EXTRACT_MAX_ZOOM} (street) +
+

+ Lower zoom = smaller file, less detail. Zoom 15 shows individual streets; + zoom 10 shows city-level detail. +

+
+ +
+ 0} + /> +
+ +
+
+ ) +} + +type PreflightStatusProps = { + errorMessage: string | null + loading: boolean + preflight: MapExtractPreflight | null + hasSelection: boolean +} + +function PreflightStatus({ errorMessage, loading, preflight, hasSelection }: PreflightStatusProps) { + if (errorMessage) { + return

{errorMessage}

+ } + if (loading) { + return

Estimating size…

+ } + if (preflight) { + return ( +

+ {preflight.tiles.toLocaleString()} tiles, ~{formatBytes(preflight.bytes, 1)}{' '} + (source build {preflight.source.date}) +

+ ) + } + if (!hasSelection) { + return

Pick at least one country to estimate size.

+ } + return

Estimating size…

+} + +export default CountryPickerModal diff --git a/admin/inertia/lib/api.ts b/admin/inertia/lib/api.ts index 2449259d..77e0515d 100644 --- a/admin/inertia/lib/api.ts +++ b/admin/inertia/lib/api.ts @@ -4,6 +4,7 @@ import { ServiceSlim } from '../../types/services' import { FileEntry } from '../../types/files' import { CheckLatestVersionResult, SystemInformationResponse, SystemUpdateStatus } from '../../types/system' import { DownloadJobWithProgress, WikipediaState } from '../../types/downloads' +import type { Country, CountryCode, CountryGroup, MapExtractPreflight } from '../../types/maps' import { EmbedJobWithProgress } from '../../types/rag' import type { CategoryWithStatus, CollectionWithStatus, ContentUpdateCheckResult, ResourceUpdateInfo } from '../../types/collections' import { catchInternal } from './util' @@ -557,6 +558,46 @@ class API { })() } + async listCountries() { + return catchInternal(async () => { + const response = await this.client.get<{ countries: Country[] }>('/maps/countries') + return response.data.countries + })() + } + + async listCountryGroups() { + return catchInternal(async () => { + const response = await this.client.get<{ groups: CountryGroup[] }>('/maps/country-groups') + return response.data.groups + })() + } + + async extractMapPreflight(params: { countries: CountryCode[]; maxzoom?: number }) { + return catchInternal(async () => { + const response = await this.client.post( + '/maps/extract-preflight', + params + ) + return response.data + })() + } + + async extractMapRegion(params: { + countries: CountryCode[] + maxzoom?: number + label?: string + estimatedBytes?: number + }) { + return catchInternal(async () => { + const response = await this.client.post<{ + message: string + filename: string + jobId?: string + }>('/maps/extract', params) + return response.data + })() + } + async listCuratedMapCollections() { return catchInternal(async () => { const response = await this.client.get( diff --git a/admin/inertia/pages/settings/maps.tsx b/admin/inertia/pages/settings/maps.tsx index 47055fa7..dcd945fb 100644 --- a/admin/inertia/pages/settings/maps.tsx +++ b/admin/inertia/pages/settings/maps.tsx @@ -13,6 +13,7 @@ import { useMutation, useQuery, useQueryClient } from '@tanstack/react-query' import useDownloads from '~/hooks/useDownloads' import StyledSectionHeader from '~/components/StyledSectionHeader' import CuratedCollectionCard from '~/components/CuratedCollectionCard' +import CountryPickerModal from '~/components/CountryPickerModal' import type { CollectionWithStatus } from '../../../types/collections' import ActiveDownloads from '~/components/ActiveDownloads' import Alert from '~/components/Alert' @@ -221,6 +222,23 @@ export default function MapsManager(props: { ) } + function openCountryPickerModal() { + openModal( + { + invalidateDownloads() + addNotification({ + type: 'success', + message: 'Download queued. Watch progress below.', + }) + closeAllModals() + }} + />, + 'country-picker-modal' + ) + } + async function openDownloadModal() { openModal( )} + +
Date: Sun, 3 May 2026 14:06:56 -0700 Subject: [PATCH 039/108] fix(UI): Country Picker UX polish + auto-refresh stored files (#817) Three UX issues from manual testing of #780 on NOMAD3. 1. Slider was unusable for multi-step zoom changes `setLoading(true)` fired immediately on every selection or maxzoom change, which disabled the slider until the request returned. Even with the 400ms debounce delaying the network call, the UI was locked the whole time. User couldn't drag through zoom levels to find the right one. Fix: bump debounce to 1500ms, move `setLoading(true)` inside the setTimeout so it only flips after the debounce expires. Slider stays interactive throughout the wait. Slider `disabled` now only ties to `downloading` (active extract dispatch), not `loading` (preflight in flight). The existing requestId stale-safe pattern handles concurrent changes. 2. Newly-downloaded maps didn't show in Stored Map Files until manual refresh `props.maps.regionFiles` is rendered server-side and passed through Inertia props; without a partial reload it stayed stale until the user navigated away and back. Fix: watch `useDownloads({ filetype: 'map' })` count via a ref. When the count drops (a download finished), trigger `router.reload({ only: ['maps'] })` to refresh just the maps prop. Existing pattern from elsewhere in the codebase. 3. Country picker didn't surface already-downloaded countries When a user re-opened "Choose Countries" after downloading UK, UK appeared unchecked with no indication it was already on disk. Fix: pass installed pmtiles filenames into the modal as a prop; parse with regex `^([a-z]{2})_[\w-]+_z\d+\.pmtiles$` to extract country codes from single-country extracts (matching MapService.buildRegionSlug's iso2 lowercase slug pattern). Render an "Installed" badge on those countries with a tooltip explaining they're re-selectable for redownload at a different zoom. Group / custom multi-country extracts don't reverse-map cleanly from filename and are skipped here. Could be a follow-up if useful. Files: admin/inertia/components/CountryPickerModal.tsx - SINGLE_COUNTRY_FILENAME_RE: iso2 + flexible date + zoom - installedFilenames prop with default [] - installedCountrySet derivation via useMemo - "Installed" badge rendering on country list rows - Debounce: 400ms -> 1500ms; setLoading inside setTimeout - Slider disabled: only on `downloading` admin/inertia/pages/settings/maps.tsx - import useEffect/useRef - destructure activeMapDownloads from useDownloads - useEffect on download count drop -> router.reload({ only: ['maps'] }) - pass installedFilenames to CountryPickerModal All three fixes tested end-to-end on NOMAD3. --- .../inertia/components/CountryPickerModal.tsx | 42 ++++++++++++++++--- admin/inertia/pages/settings/maps.tsx | 16 ++++++- 2 files changed, 50 insertions(+), 8 deletions(-) diff --git a/admin/inertia/components/CountryPickerModal.tsx b/admin/inertia/components/CountryPickerModal.tsx index b490360c..1b293348 100644 --- a/admin/inertia/components/CountryPickerModal.tsx +++ b/admin/inertia/components/CountryPickerModal.tsx @@ -30,10 +30,20 @@ export type CountryPickerModalProps = Omit< | 'large' > & { onDownloadStart?: () => void + /** Filenames of pmtiles already on disk; used to badge already-installed countries. */ + installedFilenames?: string[] } +// Single-country extracts use the slug `{iso2 lowercase}_{dateSlug}_z{maxzoom}.pmtiles`, +// matching MapService.buildRegionSlug (which lowercases the alpha-2 country code). +// dateSlug comes from the upstream pmtiles key with `.pmtiles` stripped — currently +// YYYYMMDD but we accept any digits/dashes. Group / custom filenames don't reverse-map +// to country codes, so we skip them here. +const SINGLE_COUNTRY_FILENAME_RE = /^([a-z]{2})_[\w-]+_z\d+\.pmtiles$/ + const CountryPickerModal: React.FC = ({ onDownloadStart, + installedFilenames = [], ...modalProps }) => { const [selected, setSelected] = useState>(new Set()) @@ -78,6 +88,15 @@ const CountryPickerModal: React.FC = ({ [countries, selected] ) + const installedCountrySet = useMemo(() => { + const set = new Set() + for (const filename of installedFilenames) { + const match = SINGLE_COUNTRY_FILENAME_RE.exec(filename) + if (match) set.add(match[1].toUpperCase() as CountryCode) + } + return set + }, [installedFilenames]) + function toggleCountry(code: CountryCode) { setSelected((prev) => { const next = new Set(prev) @@ -105,8 +124,10 @@ const CountryPickerModal: React.FC = ({ } // Auto-refresh the preflight whenever selection or maxzoom changes. Debounced - // so rapid multi-select clicks collapse into a single CDN round-trip, and - // stale-safe via requestId so an earlier slow response can't clobber a later one. + // so rapid multi-select clicks and slider drags collapse into a single CDN + // round-trip. Loading state only flips after the debounce expires so the UI + // stays interactive during the wait. Stale-safe via requestId so an earlier + // slow response can't clobber a later one. useEffect(() => { if (selected.size === 0) { setPreflight(null) @@ -116,10 +137,10 @@ const CountryPickerModal: React.FC = ({ return } - const requestId = ++preflightRequestIdRef.current - setLoading(true) setErrorMessage(null) const timer = setTimeout(async () => { + const requestId = ++preflightRequestIdRef.current + setLoading(true) try { const res = await api.extractMapPreflight({ countries: [...selected], @@ -135,7 +156,7 @@ const CountryPickerModal: React.FC = ({ } finally { if (requestId === preflightRequestIdRef.current) setLoading(false) } - }, 400) + }, 1500) return () => clearTimeout(timer) }, [selected, maxzoom]) @@ -253,6 +274,7 @@ const CountryPickerModal: React.FC = ({
    {list.map((country) => { const isSelected = selected.has(country.code) + const isInstalled = installedCountrySet.has(country.code) return (
  • - -
-
- - {/* Existing markers */} - {markers.map((marker) => ( - { - e.originalEvent.stopPropagation() - setSelectedMarkerId(marker.id === selectedMarkerId ? null : marker.id) - setPlacingMarker(null) - }} - > - c.id === marker.color)?.hex} - active={marker.id === selectedMarkerId} - /> - - ))} - - {/* Popup for selected marker */} - {selectedMarker && ( - setSelectedMarkerId(null)} - closeOnClick={false} - > -
{selectedMarker.name}
-
- )} - - {/* Popup for placing a new marker */} - {placingMarker && ( - setPlacingMarker(null)} - closeOnClick={false} - > -
- setMarkerName(e.target.value)} - onKeyDown={(e) => { - if (e.key === 'Enter') handleSaveMarker() - if (e.key === 'Escape') setPlacingMarker(null) - }} - className="block w-full rounded border border-gray-300 px-2 py-1 text-sm placeholder:text-gray-400 focus:outline-none focus:border-gray-500" + c.id === marker.color)?.hex} + active={marker.id === selectedMarkerId} /> -
- {PIN_COLORS.map((c) => ( + + ))} + + {selectedMarker && ( + setSelectedMarkerId(null)} + closeOnClick={false} + > +
{selectedMarker.name}
+
+ )} + + {placingMarker && ( + setPlacingMarker(null)} + closeOnClick={false} + > +
+ setMarkerName(e.target.value)} + onKeyDown={(e) => { + if (e.key === 'Enter') handleSaveMarker() + if (e.key === 'Escape') setPlacingMarker(null) + }} + className="block w-full rounded border border-gray-300 px-2 py-1 text-sm placeholder:text-gray-400 focus:outline-none focus:border-gray-500" + /> + +
+ {PIN_COLORS.map((c) => ( + + ))} +
+ +
- ))} -
-
- - + + +
-
- - )} - - - {/* Marker panel overlay */} - + + )} + +
+ +
+ +
) } diff --git a/admin/inertia/components/maps/ScaleUnitToggle.tsx b/admin/inertia/components/maps/ScaleUnitToggle.tsx new file mode 100644 index 00000000..ba0bceba --- /dev/null +++ b/admin/inertia/components/maps/ScaleUnitToggle.tsx @@ -0,0 +1,46 @@ +type ScaleUnit = 'imperial' | 'metric' + +type ScaleUnitToggleProps = { + scaleUnit: ScaleUnit + onChange: (unit: ScaleUnit) => void + onMouseEnter?: () => void +} + +export default function ScaleUnitToggle({ + scaleUnit, + onChange, + onMouseEnter, +}: ScaleUnitToggleProps) { + return ( +
+
+ + + +
+
+ ) +} diff --git a/admin/inertia/pages/maps.tsx b/admin/inertia/pages/maps.tsx index 9fe20e0a..a1d8df1e 100644 --- a/admin/inertia/pages/maps.tsx +++ b/admin/inertia/pages/maps.tsx @@ -1,38 +1,66 @@ -import MapsLayout from '~/layouts/MapsLayout' +import { useState } from 'react' import { Head, Link, router } from '@inertiajs/react' +import { IconArrowLeft } from '@tabler/icons-react' + +import MapsLayout from '~/layouts/MapsLayout' import MapComponent from '~/components/maps/MapComponent' import StyledButton from '~/components/StyledButton' -import { IconArrowLeft } from '@tabler/icons-react' -import { FileEntry } from '../../types/files' import Alert from '~/components/Alert' +import { FileEntry } from '../../types/files' + export default function Maps(props: { maps: { baseAssetsExist: boolean; regionFiles: FileEntry[] } }) { + const [isHoveringUI, setIsHoveringUI] = useState(false) + const [showMapCoordinates, setShowMapCoordinates] = useState(true) + const alertMessage = !props.maps.baseAssetsExist ? 'The base map assets have not been installed. Please download them first to enable map functionality.' : props.maps.regionFiles.length === 0 - ? 'No map regions have been downloaded yet. Please download some regions to enable map functionality.' - : null + ? 'No map regions have been downloaded yet. Please download some regions to enable map functionality.' + : null return ( +
- {/* Nav and alerts are overlayed */} -
+ {/* Navbar */} +
setIsHoveringUI(true)} + onMouseLeave={() => setIsHoveringUI(false)} + >

Back to Home

- - - Manage Map Regions - - + +
+ + + + + Manage Map Regions + + +
+ + {/* Alert */} {alertMessage && ( -
+
setIsHoveringUI(true)} + onMouseLeave={() => setIsHoveringUI(false)} + >
)} + + {/* Map */}
- +
From 8ef2c69f56a155830a4c8fb46c8fc0ed78861023 Mon Sep 17 00:00:00 2001 From: Chris Sherwood Date: Wed, 29 Apr 2026 15:52:05 -0700 Subject: [PATCH 041/108] docs: link to new WSL2 install guide from README and FAQ MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The /install/wsl2 community-supported install guide is now live on projectnomad.us. Update the README install section and FAQ to point Windows users at it instead of deflecting WSL2 questions to "see the Debian-only answer." Doesn't change the official-support stance — bare-metal Debian-based Linux remains the supported configuration. Just removes the dead-end deflection so Windows users have a real path forward. Co-Authored-By: Claude Opus 4.7 (1M context) --- FAQ.md | 6 ++++-- README.md | 2 +- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/FAQ.md b/FAQ.md index fee06f85..1823c7b0 100644 --- a/FAQ.md +++ b/FAQ.md @@ -20,7 +20,9 @@ Long answer: Custom storage paths, mount points, and external drives (like iSCSI ## Can I run NOMAD on MAC, WSL2, or a non-Debian-based Distro? -See [Why does NOMAD require a Debian-based OS?](#why-does-nomad-require-a-debian-based-os) +**WSL2 on Windows** is community-supported via the [WSL2 install guide](https://www.projectnomad.us/install/wsl2) — covers two install paths (native Docker and Docker Desktop) with all known gotchas documented and empirical performance numbers comparing WSL2 to bare-metal. + +**macOS and other non-Debian Linux distros** aren't officially supported. See [Why does NOMAD require a Debian-based OS?](#why-does-nomad-require-a-debian-based-os) for details. ## Why does NOMAD require a Debian-based OS? @@ -28,7 +30,7 @@ Project N.O.M.A.D. is currently designed to run on Debian-based Linux distributi Support for other operating systems will come in the future, but because our development resources are limited as a free and open-source project, we needed to prioritize our efforts and focus on a narrower set of supported platforms for the initial release. We chose Debian-based Linux as our starting point because it's widely used, easy to spin up, and provides a stable environment for running Docker containers. -Community members have provided guides for running N.O.M.A.D. on other platforms (e.g. WSL2, Mac, etc.) in our Discord community and [Github Discussions](https://github.com/Crosstalk-Solutions/project-nomad/discussions), so if you're interested in running N.O.M.A.D. on a non-Debian-based system, we recommend checking there for any available resources or guides. However, keep in mind that if you choose to run N.O.M.A.D. on a non-Debian-based system, you may encounter issues that we won't be able to provide support for, and you may need to have a higher level of technical expertise to troubleshoot and resolve any problems that arise. +For Windows users, the [WSL2 install guide](https://www.projectnomad.us/install/wsl2) provides a community-supported path. Community members have also published guides for other platforms (e.g. macOS) in our Discord community and [Github Discussions](https://github.com/Crosstalk-Solutions/project-nomad/discussions), so if you're interested in running N.O.M.A.D. on a non-Debian-based system, we recommend checking there for any available resources or guides. However, keep in mind that if you choose to run N.O.M.A.D. on a non-Debian-based system, you may encounter issues that we won't be able to provide support for, and you may need to have a higher level of technical expertise to troubleshoot and resolve any problems that arise. ## Can I run NOMAD on a Raspberry Pi or other ARM-based device? Project N.O.M.A.D. is currently designed to run on x86-64 architecture, and we have not yet tested or optimized it for ARM-based devices like the Raspberry Pi (and have not published any official images for ARM architecture). diff --git a/README.md b/README.md index 0edd05a3..a7e1100f 100644 --- a/README.md +++ b/README.md @@ -32,7 +32,7 @@ sudo bash install_nomad.sh Project N.O.M.A.D. is now installed on your device! Open a browser and navigate to `http://localhost:8080` (or `http://DEVICE_IP:8080`) to start exploring! -For a complete step-by-step walkthrough (including Ubuntu installation), see the [Installation Guide](https://www.projectnomad.us/install). +For a complete step-by-step walkthrough (including Ubuntu installation), see the [Installation Guide](https://www.projectnomad.us/install). For Windows users, see the [WSL2 install guide](https://www.projectnomad.us/install/wsl2) — community-supported path covering native Docker and Docker Desktop install routes. ### Advanced Installation For more control over the installation process, copy and paste the [Docker Compose template](https://raw.githubusercontent.com/Crosstalk-Solutions/project-nomad/refs/heads/main/install/management_compose.yaml) into a `docker-compose.yml` file and customize it to your liking (be sure to replace any placeholders with your actual values). Then, run `docker compose up -d` to start the Command Center and its dependencies. Note: this method is recommended for advanced users only, as it requires familiarity with Docker and manual configuration before starting. From d81b66bb145cf63e3bec2cfb21adbe3ed0658216 Mon Sep 17 00:00:00 2001 From: Jake Turner Date: Mon, 4 May 2026 17:45:18 +0000 Subject: [PATCH 042/108] chore(deps): pin all deps to exact versions --- admin/package.json | 170 ++++++++++++++++++++++----------------------- 1 file changed, 85 insertions(+), 85 deletions(-) diff --git a/admin/package.json b/admin/package.json index d98e7e02..b273fd00 100644 --- a/admin/package.json +++ b/admin/package.json @@ -38,94 +38,94 @@ "#jobs/*": "./app/jobs/*.js" }, "devDependencies": { - "@adonisjs/assembler": "^7.8.2", - "@adonisjs/eslint-config": "^2.0.0", - "@adonisjs/prettier-config": "^1.4.4", - "@adonisjs/tsconfig": "^1.4.0", - "@japa/assert": "^4.0.1", - "@japa/plugin-adonisjs": "^4.0.0", - "@japa/runner": "^4.2.0", + "@adonisjs/assembler": "7.8.2", + "@adonisjs/eslint-config": "2.1.2", + "@adonisjs/prettier-config": "1.4.5", + "@adonisjs/tsconfig": "1.4.1", + "@japa/assert": "4.2.0", + "@japa/plugin-adonisjs": "4.0.0", + "@japa/runner": "4.5.0", "@swc/core": "1.11.24", - "@tanstack/eslint-plugin-query": "^5.81.2", - "@types/compression": "^1.8.1", - "@types/dockerode": "^4.0.1", - "@types/luxon": "^3.6.2", - "@types/node": "^22.15.18", - "@types/react": "^19.1.8", - "@types/react-dom": "^19.1.6", - "@types/stopword": "^2.0.3", - "eslint": "^9.26.0", - "hot-hook": "^0.4.0", - "prettier": "^3.5.3", - "ts-node-maintained": "^10.9.5", - "typescript": "~5.8.3", - "vite": "^6.4.2" + "@tanstack/eslint-plugin-query": "5.91.4", + "@types/compression": "1.8.1", + "@types/dockerode": "4.0.1", + "@types/luxon": "3.7.1", + "@types/node": "22.19.7", + "@types/react": "19.2.10", + "@types/react-dom": "19.2.3", + "@types/stopword": "2.0.3", + "eslint": "9.39.2", + "hot-hook": "0.4.0", + "prettier": "3.8.1", + "ts-node-maintained": "10.9.6", + "typescript": "5.8.3", + "vite": "6.4.2" }, "dependencies": { - "@adonisjs/auth": "^9.4.0", - "@adonisjs/core": "^6.18.0", - "@adonisjs/cors": "^2.2.1", - "@adonisjs/inertia": "^3.1.1", - "@adonisjs/lucid": "^21.8.2", - "@adonisjs/session": "^7.5.1", - "@adonisjs/shield": "^8.2.0", - "@adonisjs/static": "^1.1.1", - "@adonisjs/transmit": "^2.0.2", - "@adonisjs/transmit-client": "^1.0.0", - "@adonisjs/vite": "^4.0.0", - "@chonkiejs/core": "^0.0.7", - "@headlessui/react": "^2.2.4", - "@inertiajs/react": "^2.0.13", - "@markdoc/markdoc": "^0.5.2", - "@openzim/libzim": "^4.0.0", - "@protomaps/basemaps": "^5.7.0", - "@qdrant/js-client-rest": "^1.16.2", - "@tabler/icons-react": "^3.34.0", - "@tailwindcss/vite": "^4.1.10", - "@tanstack/react-query": "^5.81.5", - "@tanstack/react-query-devtools": "^5.83.0", - "@tanstack/react-virtual": "^3.13.12", - "@uppy/core": "^5.2.0", - "@uppy/dashboard": "^5.1.0", - "@uppy/react": "^5.1.1", - "@vinejs/vine": "^3.0.1", - "@vitejs/plugin-react": "^4.6.0", - "autoprefixer": "^10.4.21", - "axios": "^1.15.0", - "better-sqlite3": "^12.1.1", - "bullmq": "^5.65.1", - "cheerio": "^1.2.0", - "compression": "^1.8.1", - "dockerode": "^4.0.7", - "edge.js": "^6.2.1", - "fast-xml-parser": "^5.5.7", - "fuse.js": "^7.1.0", - "jszip": "^3.10.1", - "luxon": "^3.6.1", - "maplibre-gl": "^4.7.1", - "mysql2": "^3.14.1", - "ollama": "^0.6.3", - "openai": "^6.27.0", - "pdf-parse": "^2.4.5", - "pdf2pic": "^3.2.0", - "pino-pretty": "^13.0.0", - "pmtiles": "^4.4.0", - "postcss": "^8.5.6", - "react": "^19.1.0", - "react-adonis-transmit": "^1.0.1", - "react-dom": "^19.1.0", - "react-map-gl": "^8.1.0", - "react-markdown": "^10.1.0", - "reflect-metadata": "^0.2.2", - "remark-gfm": "^4.0.1", - "sharp": "^0.34.5", - "stopword": "^3.1.5", - "systeminformation": "^5.31.0", - "tailwindcss": "^4.2.1", - "tar": "^7.5.11", - "tesseract.js": "^7.0.0", - "url-join": "^5.0.0", - "yaml": "^2.8.3" + "@adonisjs/auth": "9.6.0", + "@adonisjs/core": "6.19.3", + "@adonisjs/cors": "2.2.1", + "@adonisjs/inertia": "3.1.1", + "@adonisjs/lucid": "21.8.2", + "@adonisjs/session": "7.7.1", + "@adonisjs/shield": "8.2.0", + "@adonisjs/static": "1.1.1", + "@adonisjs/transmit": "2.0.2", + "@adonisjs/transmit-client": "1.1.0", + "@adonisjs/vite": "4.0.0", + "@chonkiejs/core": "0.0.7", + "@headlessui/react": "2.2.9", + "@inertiajs/react": "2.3.13", + "@markdoc/markdoc": "0.5.4", + "@openzim/libzim": "4.0.0", + "@protomaps/basemaps": "5.7.0", + "@qdrant/js-client-rest": "1.16.2", + "@tabler/icons-react": "3.36.1", + "@tailwindcss/vite": "4.1.18", + "@tanstack/react-query": "5.90.20", + "@tanstack/react-query-devtools": "5.91.3", + "@tanstack/react-virtual": "3.13.18", + "@uppy/core": "5.2.0", + "@uppy/dashboard": "5.1.0", + "@uppy/react": "5.1.1", + "@vinejs/vine": "3.0.1", + "@vitejs/plugin-react": "4.7.0", + "autoprefixer": "10.4.24", + "axios": "1.15.0", + "better-sqlite3": "12.6.2", + "bullmq": "5.67.2", + "cheerio": "1.2.0", + "compression": "1.8.1", + "dockerode": "4.0.9", + "edge.js": "6.4.0", + "fast-xml-parser": "5.5.9", + "fuse.js": "7.1.0", + "jszip": "3.10.1", + "luxon": "3.7.2", + "maplibre-gl": "4.7.1", + "mysql2": "3.16.2", + "ollama": "0.6.3", + "openai": "6.27.0", + "pdf-parse": "2.4.5", + "pdf2pic": "3.2.0", + "pino-pretty": "13.1.3", + "pmtiles": "4.4.0", + "postcss": "8.5.6", + "react": "19.2.4", + "react-adonis-transmit": "1.0.1", + "react-dom": "19.2.4", + "react-map-gl": "8.1.0", + "react-markdown": "10.1.0", + "reflect-metadata": "0.2.2", + "remark-gfm": "4.0.1", + "sharp": "0.34.5", + "stopword": "3.1.5", + "systeminformation": "5.31.0", + "tailwindcss": "4.2.2", + "tar": "7.5.11", + "tesseract.js": "7.0.0", + "url-join": "5.0.0", + "yaml": "2.8.3" }, "hotHook": { "boundaries": [ From d66eaa3d423ab049b1c382c3753375f3e8bdad13 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 25 Mar 2026 21:51:08 +0000 Subject: [PATCH 043/108] build(deps): bump picomatch in /admin Bumps and [picomatch](https://github.com/micromatch/picomatch). These dependencies needed to be updated together. Updates `picomatch` from 4.0.3 to 4.0.4 - [Release notes](https://github.com/micromatch/picomatch/releases) - [Changelog](https://github.com/micromatch/picomatch/blob/master/CHANGELOG.md) - [Commits](https://github.com/micromatch/picomatch/compare/4.0.3...4.0.4) Updates `picomatch` from 2.3.1 to 2.3.2 - [Release notes](https://github.com/micromatch/picomatch/releases) - [Changelog](https://github.com/micromatch/picomatch/blob/master/CHANGELOG.md) - [Commits](https://github.com/micromatch/picomatch/compare/4.0.3...4.0.4) --- updated-dependencies: - dependency-name: picomatch dependency-version: 4.0.4 dependency-type: indirect - dependency-name: picomatch dependency-version: 2.3.2 dependency-type: indirect ... Signed-off-by: dependabot[bot] --- admin/package-lock.json | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/admin/package-lock.json b/admin/package-lock.json index ae7ef36b..29f8dd3e 100644 --- a/admin/package-lock.json +++ b/admin/package-lock.json @@ -12174,9 +12174,9 @@ } }, "node_modules/micromatch/node_modules/picomatch": { - "version": "2.3.1", - "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.1.tgz", - "integrity": "sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA==", + "version": "2.3.2", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.2.tgz", + "integrity": "sha512-V7+vQEJ06Z+c5tSye8S+nHUfI51xoXIXjHQ99cQtKUkQqqO1kO/KCJUfZXuB47h/YBlDhah2H3hdUGXn8ie0oA==", "license": "MIT", "engines": { "node": ">=8.6" @@ -13351,9 +13351,9 @@ "license": "ISC" }, "node_modules/picomatch": { - "version": "4.0.3", - "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.3.tgz", - "integrity": "sha512-5gTmgEY/sqK6gFXLIsQNH19lWb4ebPDLA4SdLP7dsWkIXHWlG66oPuVvXSGFPppYZz8ZDZq0dYYrbHfBCVUb1Q==", + "version": "4.0.4", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.4.tgz", + "integrity": "sha512-QP88BAKvMam/3NxH6vj2o21R6MjxZUAd6nlwAS/pnGvN9IVLocLHxGYIzFhg6fUQ+5th6P4dv4eW9jX3DSIj7A==", "license": "MIT", "engines": { "node": ">=12" From a7dbee55c4286e13b3915a88adf74bc259adff82 Mon Sep 17 00:00:00 2001 From: chriscrosstalk <49691103+chriscrosstalk@users.noreply.github.com> Date: Mon, 4 May 2026 11:30:59 -0700 Subject: [PATCH 044/108] feat(Content): custom ZIM library sources with pre-seeded mirrors (#593) * feat(content): add custom ZIM library sources with pre-seeded mirrors Users reported slow download speeds from the default Kiwix CDN. This adds the ability to browse and download ZIM files from alternative Kiwix mirrors or self-hosted repositories, all through the GUI. - Add "Custom Libraries" button next to "Browse the Kiwix Library" - Source dropdown to switch between Default (Kiwix) and custom libraries - Browsable directory structure with breadcrumb navigation - 5 pre-seeded official Kiwix mirrors (US, DE, DK, UK, Global CDN) - Built-in mirrors protected from deletion - Downloads use existing pipeline (progress, cancel, Kiwix restart) - Source selection persists across page loads via localStorage - Scrollable directory browser (600px max) with sticky header - SSRF protection on all custom library URLs Closes #576 Co-Authored-By: Claude Opus 4.6 (1M context) * fix(content): recognize Wikipedia downloads from mirror sources When Wikipedia is downloaded via a custom mirror instead of the default Kiwix server, the completion callback now matches by filename instead of exact URL. This ensures the Wikipedia selector correctly shows "Installed" status and triggers old-version cleanup regardless of which mirror was used. Also handles the case where no Wikipedia selection exists yet (file downloaded before visiting the selector), creating the record automatically. Co-Authored-By: Claude Opus 4.6 (1M context) * fix(ZIM): use cheerio for custom mirror directory parsing * fix(ZIM): use URL constructor for more robust joining --------- Co-authored-by: Claude Opus 4.6 (1M context) Co-authored-by: Jake Turner --- admin/app/controllers/zim_controller.ts | 49 +- admin/app/models/custom_library_source.ts | 24 + admin/app/services/zim_service.ts | 180 +++++- admin/app/validators/zim.ts | 27 + ...001_create_custom_library_sources_table.ts | 42 ++ admin/inertia/lib/api.ts | 36 ++ .../pages/settings/zim/remote-explorer.tsx | 545 +++++++++++++++--- admin/start/routes.ts | 6 + 8 files changed, 814 insertions(+), 95 deletions(-) create mode 100644 admin/app/models/custom_library_source.ts create mode 100644 admin/database/migrations/1775100000001_create_custom_library_sources_table.ts diff --git a/admin/app/controllers/zim_controller.ts b/admin/app/controllers/zim_controller.ts index 96adf636..006e59b9 100644 --- a/admin/app/controllers/zim_controller.ts +++ b/admin/app/controllers/zim_controller.ts @@ -6,7 +6,7 @@ import { remoteDownloadWithMetadataValidator, selectWikipediaValidator, } from '#validators/common' -import { listRemoteZimValidator } from '#validators/zim' +import { addCustomLibraryValidator, browseLibraryValidator, idParamValidator, listRemoteZimValidator } from '#validators/zim' import { inject } from '@adonisjs/core' import type { HttpContext } from '@adonisjs/core/http' @@ -85,4 +85,51 @@ export default class ZimController { const payload = await request.validateUsing(selectWikipediaValidator) return this.zimService.selectWikipedia(payload.optionId) } + + // Custom library endpoints + + async listCustomLibraries({}: HttpContext) { + return this.zimService.listCustomLibraries() + } + + async addCustomLibrary({ request, response }: HttpContext) { + const payload = await request.validateUsing(addCustomLibraryValidator) + assertNotPrivateUrl(payload.base_url) + try { + const source = await this.zimService.addCustomLibrary(payload.name, payload.base_url) + return { message: 'Custom library added', library: source } + } catch (error) { + if (error.message === 'Maximum of 10 custom libraries allowed') { + return response.status(400).send({ message: error.message }) + } + throw error + } + } + + async removeCustomLibrary({ request, response }: HttpContext) { + const payload = await request.validateUsing(idParamValidator) + try { + await this.zimService.removeCustomLibrary(payload.params.id) + return { message: 'Custom library removed' } + } catch (error) { + if (error.message === 'Custom library not found') { + return response.status(404).send({ message: error.message }) + } + throw error + } + } + + async browseLibrary({ request, response }: HttpContext) { + const payload = await request.validateUsing(browseLibraryValidator) + try { + return await this.zimService.browseLibraryUrl(payload.url) + } catch (error) { + if (error.message?.includes('loopback or link-local')) { + return response.status(400).send({ message: error.message }) + } + return response.status(502).send({ + message: 'Could not fetch directory listing from the provided URL', + }) + } + } } diff --git a/admin/app/models/custom_library_source.ts b/admin/app/models/custom_library_source.ts new file mode 100644 index 00000000..478d9a01 --- /dev/null +++ b/admin/app/models/custom_library_source.ts @@ -0,0 +1,24 @@ +import { DateTime } from 'luxon' +import { BaseModel, column, SnakeCaseNamingStrategy } from '@adonisjs/lucid/orm' + +export default class CustomLibrarySource extends BaseModel { + static namingStrategy = new SnakeCaseNamingStrategy() + + @column({ isPrimary: true }) + declare id: number + + @column() + declare name: string + + @column() + declare base_url: string + + @column() + declare is_default: boolean + + @column.dateTime({ autoCreate: true }) + declare created_at: DateTime + + @column.dateTime({ autoCreate: true, autoUpdate: true }) + declare updated_at: DateTime +} diff --git a/admin/app/services/zim_service.ts b/admin/app/services/zim_service.ts index db6b5b77..1cc9e973 100644 --- a/admin/app/services/zim_service.ts +++ b/admin/app/services/zim_service.ts @@ -4,6 +4,7 @@ import { RemoteZimFileEntry, } from '../../types/zim.js' import axios from 'axios' +import * as cheerio from 'cheerio' import { XMLParser } from 'fast-xml-parser' import { isRawListRemoteZimFilesResponse, isRawRemoteZimFileEntry } from '../../util/zim.js' import logger from '@adonisjs/core/services/logger' @@ -27,6 +28,8 @@ import { SERVICE_NAMES } from '../../constants/service_names.js' import { CollectionManifestService } from './collection_manifest_service.js' import { KiwixLibraryService } from './kiwix_library_service.js' import type { CategoryWithStatus } from '../../types/collections.js' +import CustomLibrarySource from '#models/custom_library_source' +import { assertNotPrivateUrl } from '#validators/common' const ZIM_MIME_TYPES = ['application/x-zim', 'application/x-openzim', 'application/octet-stream'] const WIKIPEDIA_OPTIONS_URL = 'https://raw.githubusercontent.com/Crosstalk-Solutions/project-nomad/refs/heads/main/collections/wikipedia.json' @@ -587,25 +590,47 @@ export class ZimService { } async onWikipediaDownloadComplete(url: string, success: boolean): Promise { + const filename = url.split('/').pop() || '' const selection = await this.getWikipediaSelection() - if (!selection || selection.url !== url) { - logger.warn(`[ZimService] Wikipedia download complete callback for unknown URL: ${url}`) - return + // Determine which Wikipedia option this file belongs to by matching filename + let matchedOptionId: string | null = null + try { + const options = await this.getWikipediaOptions() + for (const opt of options) { + if (opt.url && opt.url.split('/').pop() === filename) { + matchedOptionId = opt.id + break + } + } + } catch { + // If we can't fetch options, try to continue with existing selection } if (success) { - // Update status to installed - selection.status = 'installed' - await selection.save() + // Update or create the selection record + // Match by filename (not URL) so mirror downloads are recognized + if (selection) { + selection.option_id = matchedOptionId || selection.option_id + selection.url = url + selection.filename = filename + selection.status = 'installed' + await selection.save() + } else { + await WikipediaSelection.create({ + option_id: matchedOptionId || 'unknown', + url: url, + filename: filename, + status: 'installed', + }) + } - logger.info(`[ZimService] Wikipedia download completed successfully: ${selection.filename}`) + logger.info(`[ZimService] Wikipedia download completed successfully: ${filename}`) - // Delete the old Wikipedia file if it exists and is different - // We need to find what was previously installed + // Delete old Wikipedia files (keep only the newly installed one) const existingFiles = await this.list() const wikipediaFiles = existingFiles.files.filter((f) => - f.name.startsWith('wikipedia_en_') && f.name !== selection.filename + f.name.startsWith('wikipedia_en_') && f.name !== filename ) for (const oldFile of wikipediaFiles) { @@ -617,10 +642,137 @@ export class ZimService { } } } else { - // Download failed - keep the selection record but mark as failed - selection.status = 'failed' - await selection.save() - logger.error(`[ZimService] Wikipedia download failed for: ${selection.filename}`) + // Download failed - update selection if it matches this file + if (selection && (!selection.filename || selection.filename === filename)) { + selection.status = 'failed' + await selection.save() + logger.error(`[ZimService] Wikipedia download failed for: ${filename}`) + } else { + logger.error(`[ZimService] Wikipedia download failed for: ${filename} (no matching selection)`) + } + } + } + + // Custom library source management + + async listCustomLibraries(): Promise { + return CustomLibrarySource.all() + } + + async addCustomLibrary(name: string, baseUrl: string): Promise { + const count = await CustomLibrarySource.query().count('* as total') + const total = Number(count[0].$extras.total) + if (total >= 10) { + throw new Error('Maximum of 10 custom libraries allowed') + } + + // Ensure URL ends with / + const normalizedUrl = baseUrl.endsWith('/') ? baseUrl : baseUrl + '/' + + return CustomLibrarySource.create({ + name, + base_url: normalizedUrl, + }) + } + + async removeCustomLibrary(id: number): Promise { + const source = await CustomLibrarySource.find(id) + if (!source) { + throw new Error('Custom library not found') + } + if (source.is_default) { + throw new Error('Cannot remove a built-in mirror') } + await source.delete() + } + + async browseLibraryUrl(url: string): Promise<{ + directories: { name: string; url: string }[] + files: { name: string; url: string; size_bytes: number | null }[] + }> { + assertNotPrivateUrl(url) + + const normalizedUrl = url.endsWith('/') ? url : url + '/' + + const res = await axios.get(normalizedUrl, { + responseType: 'text', + timeout: 15000, + headers: { + 'Accept': 'text/html', + }, + }) + + const html: string = res.data + const directories: { name: string; url: string }[] = [] + const files: { name: string; url: string; size_bytes: number | null }[] = [] + + const $ = cheerio.load(html) + + $('a').each((_, el) => { + const href = el.attribs?.href + if (!href || href === '../' || href === './' || href === '/' || href.startsWith('?') || href.startsWith('#')) { + return + } + if (href.startsWith('/') || href.startsWith('http://') || href.startsWith('https://')) { + return + } + + if (href.endsWith('/')) { + const dirName = decodeURIComponent(href.replace(/\/$/, '')) + directories.push({ + name: dirName, + url: new URL(href, normalizedUrl).toString(), + }) + return + } + + if (href.endsWith('.zim')) { + const fileName = decodeURIComponent(href) + + // Apache/Nginx autoindex put the date + size in the text node directly + // following within a
. Walk forward across text siblings until
+        // we find a parseable size token.
+        let trailingText = ''
+        let sibling = el.next
+        while (sibling && sibling.type === 'text') {
+          trailingText += sibling.data
+          if (/\n/.test(sibling.data)) break
+          sibling = sibling.next
+        }
+
+        files.push({
+          name: fileName,
+          url: new URL(href, normalizedUrl).toString(),
+          size_bytes: this._parseListingSize(trailingText),
+        })
+      }
+    })
+
+    directories.sort((a, b) => a.name.localeCompare(b.name))
+    files.sort((a, b) => a.name.localeCompare(b.name))
+
+    return { directories, files }
+  }
+
+  /**
+   * Parse a directory-listing size token out of the text that follows an anchor.
+   * Apache renders e.g. `   2024-01-15 10:30  5.1G`; Nginx renders raw bytes.
+   * Returns bytes or null if no size token is found.
+   */
+  private _parseListingSize(text: string): number | null {
+    // Skip the date/time columns; grab the last numeric token (with optional suffix)
+    // before a newline. Matches `5.1G`, `5368709120`, `1.2T`, etc.
+    const sizeMatch = /([\d.]+\s*[KMGT]?B?|\d+)\s*$/i.exec(text.split('\n')[0].trim())
+    if (!sizeMatch) return null
+
+    const sizeStr = sizeMatch[1].replace(/\s|B$/gi, '')
+    const num = parseFloat(sizeStr)
+    if (isNaN(num)) return null
+
+    if (/^\d+$/.test(sizeStr)) return num
+
+    const suffix = sizeStr.slice(-1).toUpperCase()
+    const multipliers: Record = { K: 1024, M: 1024 ** 2, G: 1024 ** 3, T: 1024 ** 4 }
+    return multipliers[suffix] ? Math.round(num * multipliers[suffix]) : null
   }
 }
diff --git a/admin/app/validators/zim.ts b/admin/app/validators/zim.ts
index 2c182710..5463e527 100644
--- a/admin/app/validators/zim.ts
+++ b/admin/app/validators/zim.ts
@@ -7,3 +7,30 @@ export const listRemoteZimValidator = vine.compile(
     query: vine.string().optional(),
   })
 )
+
+export const addCustomLibraryValidator = vine.compile(
+  vine.object({
+    name: vine.string().trim().minLength(1).maxLength(100),
+    base_url: vine
+      .string()
+      .url({ require_tld: false })
+      .trim(),
+  })
+)
+
+export const browseLibraryValidator = vine.compile(
+  vine.object({
+    url: vine
+      .string()
+      .url({ require_tld: false })
+      .trim(),
+  })
+)
+
+export const idParamValidator = vine.compile(
+  vine.object({
+    params: vine.object({
+      id: vine.number(),
+    }),
+  })
+)
diff --git a/admin/database/migrations/1775100000001_create_custom_library_sources_table.ts b/admin/database/migrations/1775100000001_create_custom_library_sources_table.ts
new file mode 100644
index 00000000..baa3c775
--- /dev/null
+++ b/admin/database/migrations/1775100000001_create_custom_library_sources_table.ts
@@ -0,0 +1,42 @@
+import { BaseSchema } from '@adonisjs/lucid/schema'
+
+export default class extends BaseSchema {
+  protected tableName = 'custom_library_sources'
+
+  async up() {
+    this.schema.createTable(this.tableName, (table) => {
+      table.increments('id').primary()
+      table.string('name', 100).notNullable()
+      table.string('base_url', 2048).notNullable()
+      table.boolean('is_default').notNullable().defaultTo(false)
+      table.timestamp('created_at').notNullable()
+      table.timestamp('updated_at').notNullable()
+    })
+
+    // Seed default Kiwix mirrors
+    const now = new Date().toISOString().slice(0, 19).replace('T', ' ')
+    const defaults = [
+      { name: 'Debian CDN (Global)', base_url: 'https://cdimage.debian.org/mirror/kiwix.org/zim/' },
+      { name: 'Your.org (US)', base_url: 'https://ftpmirror.your.org/pub/kiwix/zim/' },
+      { name: 'FAU Erlangen (DE)', base_url: 'https://ftp.fau.de/kiwix/zim/' },
+      { name: 'Dotsrc (DK)', base_url: 'https://mirrors.dotsrc.org/kiwix/zim/' },
+      { name: 'MirrorService (UK)', base_url: 'https://www.mirrorservice.org/sites/download.kiwix.org/zim/' },
+    ]
+
+    for (const d of defaults) {
+      await this.defer(async (db) => {
+        await db.table(this.tableName).insert({
+          name: d.name,
+          base_url: d.base_url,
+          is_default: true,
+          created_at: now,
+          updated_at: now,
+        })
+      })
+    }
+  }
+
+  async down() {
+    this.schema.dropTable(this.tableName)
+  }
+}
diff --git a/admin/inertia/lib/api.ts b/admin/inertia/lib/api.ts
index 77e0515d..3e015d84 100644
--- a/admin/inertia/lib/api.ts
+++ b/admin/inertia/lib/api.ts
@@ -681,6 +681,42 @@ class API {
     })()
   }
 
+  async listCustomLibraries() {
+    return catchInternal(async () => {
+      const response = await this.client.get<{ id: number; name: string; base_url: string; is_default: boolean }[]>(
+        '/zim/custom-libraries'
+      )
+      return response.data
+    })()
+  }
+
+  async addCustomLibrary(name: string, base_url: string) {
+    return catchInternal(async () => {
+      const response = await this.client.post<{
+        message: string
+        library: { id: number; name: string; base_url: string }
+      }>('/zim/custom-libraries', { name, base_url })
+      return response.data
+    })()
+  }
+
+  async removeCustomLibrary(id: number) {
+    return catchInternal(async () => {
+      const response = await this.client.delete<{ message: string }>(`/zim/custom-libraries/${id}`)
+      return response.data
+    })()
+  }
+
+  async browseLibrary(url: string) {
+    return catchInternal(async () => {
+      const response = await this.client.get<{
+        directories: { name: string; url: string }[]
+        files: { name: string; url: string; size_bytes: number | null }[]
+      }>('/zim/browse-library', { params: { url } })
+      return response.data
+    })()
+  }
+
   async deleteZimFile(filename: string) {
     return catchInternal(async () => {
       const response = await this.client.delete<{ message: string }>(`/zim/${filename}`)
diff --git a/admin/inertia/pages/settings/zim/remote-explorer.tsx b/admin/inertia/pages/settings/zim/remote-explorer.tsx
index 9d1ca3eb..7fcfed16 100644
--- a/admin/inertia/pages/settings/zim/remote-explorer.tsx
+++ b/admin/inertia/pages/settings/zim/remote-explorer.tsx
@@ -21,7 +21,16 @@ import useInternetStatus from '~/hooks/useInternetStatus'
 import Alert from '~/components/Alert'
 import useServiceInstalledStatus from '~/hooks/useServiceInstalledStatus'
 import Input from '~/components/inputs/Input'
-import { IconSearch, IconBooks } from '@tabler/icons-react'
+import {
+  IconSearch,
+  IconBooks,
+  IconFolder,
+  IconFileDownload,
+  IconChevronRight,
+  IconPlus,
+  IconTrash,
+  IconLibrary,
+} from '@tabler/icons-react'
 import useDebounce from '~/hooks/useDebounce'
 import CategoryCard from '~/components/CategoryCard'
 import TierSelectionModal from '~/components/TierSelectionModal'
@@ -34,6 +43,13 @@ import { SERVICE_NAMES } from '../../../../constants/service_names'
 
 const CURATED_CATEGORIES_KEY = 'curated-categories'
 const WIKIPEDIA_STATE_KEY = 'wikipedia-state'
+const CUSTOM_LIBRARIES_KEY = 'custom-libraries'
+
+type CustomLibrary = { id: number; name: string; base_url: string; is_default: boolean }
+type BrowseResult = {
+  directories: { name: string; url: string }[]
+  files: { name: string; url: string; size_bytes: number | null }[]
+}
 
 export default function ZimRemoteExplorer() {
   const queryClient = useQueryClient()
@@ -56,6 +72,20 @@ export default function ZimRemoteExplorer() {
   const [selectedWikipedia, setSelectedWikipedia] = useState(null)
   const [isSubmittingWikipedia, setIsSubmittingWikipedia] = useState(false)
 
+  // Custom library state - persist selection to localStorage
+  const [selectedSource, setSelectedSource] = useState<'default' | number>(() => {
+    try {
+      const saved = localStorage.getItem('nomad:zim-library-source')
+      if (saved && saved !== 'default') return parseInt(saved, 10)
+    } catch {}
+    return 'default'
+  })
+  const [browseUrl, setBrowseUrl] = useState(null)
+  const [breadcrumbs, setBreadcrumbs] = useState<{ name: string; url: string }[]>([])
+  const [manageModalOpen, setManageModalOpen] = useState(false)
+  const [newLibraryName, setNewLibraryName] = useState('')
+  const [newLibraryUrl, setNewLibraryUrl] = useState('')
+
   const debouncedSetQuery = debounce((val: string) => {
     setQuery(val)
   }, 400)
@@ -79,6 +109,26 @@ export default function ZimRemoteExplorer() {
     enabled: true,
   })
 
+  // Fetch custom libraries
+  const { data: customLibraries } = useQuery({
+    queryKey: [CUSTOM_LIBRARIES_KEY],
+    queryFn: () => api.listCustomLibraries(),
+    refetchOnWindowFocus: false,
+  })
+
+  // Browse custom library directory
+  const {
+    data: browseData,
+    isLoading: isBrowsing,
+    error: browseError,
+  } = useQuery({
+    queryKey: ['browse-library', browseUrl],
+    queryFn: () => api.browseLibrary(browseUrl!) as Promise,
+    enabled: !!browseUrl && selectedSource !== 'default',
+    refetchOnWindowFocus: false,
+    retry: false,
+  })
+
   const { data, fetchNextPage, isFetching, isLoading } =
     useInfiniteQuery({
       queryKey: ['remote-zim-files', query],
@@ -97,6 +147,7 @@ export default function ZimRemoteExplorer() {
       getNextPageParam: (lastPage) => (lastPage.has_more ? lastPage.next_start : undefined),
       refetchOnWindowFocus: false,
       placeholderData: keepPreviousData,
+      enabled: selectedSource === 'default',
     })
 
   const flatData = useMemo(() => {
@@ -140,6 +191,50 @@ export default function ZimRemoteExplorer() {
     fetchOnBottomReached(tableParentRef.current)
   }, [fetchOnBottomReached])
 
+  // Restore custom library selection on mount when data loads
+  useEffect(() => {
+    if (selectedSource !== 'default' && customLibraries) {
+      const lib = customLibraries.find((l) => l.id === selectedSource)
+      if (lib && !browseUrl) {
+        setBrowseUrl(lib.base_url)
+        setBreadcrumbs([{ name: lib.name, url: lib.base_url }])
+      } else if (!lib) {
+        // Saved library was deleted
+        setSelectedSource('default')
+        localStorage.setItem('nomad:zim-library-source', 'default')
+      }
+    }
+  }, [customLibraries, selectedSource])
+
+  // When selecting a custom library, navigate to its root
+  const handleSourceChange = (value: string) => {
+    localStorage.setItem('nomad:zim-library-source', value)
+    if (value === 'default') {
+      setSelectedSource('default')
+      setBrowseUrl(null)
+      setBreadcrumbs([])
+    } else {
+      const id = parseInt(value, 10)
+      const lib = customLibraries?.find((l) => l.id === id)
+      if (lib) {
+        setSelectedSource(id)
+        setBrowseUrl(lib.base_url)
+        setBreadcrumbs([{ name: lib.name, url: lib.base_url }])
+      }
+    }
+  }
+
+  const navigateToDirectory = (name: string, url: string) => {
+    setBrowseUrl(url)
+    setBreadcrumbs((prev) => [...prev, { name, url }])
+  }
+
+  const navigateToBreadcrumb = (index: number) => {
+    const crumb = breadcrumbs[index]
+    setBrowseUrl(crumb.url)
+    setBreadcrumbs((prev) => prev.slice(0, index + 1))
+  }
+
   async function confirmDownload(record: RemoteZimFileEntry) {
     openModal(
        {
+          downloadCustomFile(file)
+          closeAllModals()
+        }}
+        onCancel={closeAllModals}
+        open={true}
+        confirmText="Download"
+        cancelText="Cancel"
+        confirmVariant="primary"
+      >
+        

+ Are you sure you want to download{' '} + {file.name} + {file.size_bytes ? ` (${formatBytes(file.size_bytes)})` : ''}? The Kiwix + application will be restarted after the download is complete. +

+
, + 'confirm-download-custom-modal' + ) + } + async function downloadFile(record: RemoteZimFileEntry) { try { await api.downloadRemoteZimFile(record.download_url, { @@ -179,6 +299,26 @@ export default function ZimRemoteExplorer() { } } + async function downloadCustomFile(file: { name: string; url: string; size_bytes: number | null }) { + try { + await api.downloadRemoteZimFile(file.url, { + title: file.name.replace(/\.zim$/, ''), + size_bytes: file.size_bytes ?? undefined, + }) + addNotification({ + message: `Started downloading "${file.name}"`, + type: 'success', + }) + invalidateDownloads() + } catch (error) { + console.error('Error downloading file:', error) + addNotification({ + message: 'Failed to start download.', + type: 'error', + }) + } + } + // Category/tier handlers const handleCategoryClick = (category: CategoryWithStatus) => { if (!isOnline) return @@ -264,6 +404,35 @@ export default function ZimRemoteExplorer() { }, }) + // Custom library management + const addLibraryMutation = useMutation({ + mutationFn: () => api.addCustomLibrary(newLibraryName.trim(), newLibraryUrl.trim()), + onSuccess: () => { + addNotification({ message: 'Custom library added.', type: 'success' }) + queryClient.invalidateQueries({ queryKey: [CUSTOM_LIBRARIES_KEY] }) + setNewLibraryName('') + setNewLibraryUrl('') + }, + onError: () => { + addNotification({ message: 'Failed to add custom library.', type: 'error' }) + }, + }) + + const removeLibraryMutation = useMutation({ + mutationFn: (id: number) => api.removeCustomLibrary(id), + onSuccess: (_data, id) => { + addNotification({ message: 'Custom library removed.', type: 'success' }) + queryClient.invalidateQueries({ queryKey: [CUSTOM_LIBRARIES_KEY] }) + if (selectedSource === id) { + setSelectedSource('default') + setBrowseUrl(null) + setBreadcrumbs([]) + } + }, + }) + + const hasCustomLibraries = customLibraries && customLibraries.length > 0 + return ( @@ -302,7 +471,7 @@ export default function ZimRemoteExplorer() { Force Refresh Collections
- + {/* Wikipedia Selector */} {isLoadingWikipedia ? (
@@ -360,87 +529,303 @@ export default function ZimRemoteExplorer() { ) : (

No curated content categories available.

)} - -
- { - setQueryUI(e.target.value) - debouncedSetQuery(e.target.value) - }} - className="w-1/3" - leftIcon={} - /> + + {/* Kiwix Library / Custom Library Browser */} +
+ + setManageModalOpen(true)} + disabled={!isOnline} + icon="IconLibrary" + > + {hasCustomLibraries ? 'Manage Custom Libraries' : 'Add Custom Library'} +
- - data={flatData.map((i, idx) => { - const row = virtualizer.getVirtualItems().find((v) => v.index === idx) - return { - ...i, - height: `${row?.size || 48}px`, // Use the size from the virtualizer - translateY: row?.start || 0, - } - })} - ref={tableParentRef} - loading={isLoading} - columns={[ - { - accessor: 'title', - }, - { - accessor: 'author', - }, - { - accessor: 'summary', - }, - { - accessor: 'updated', - render(record) { - return new Intl.DateTimeFormat('en-US', { - dateStyle: 'medium', - }).format(new Date(record.updated)) - }, - }, - { - accessor: 'size_bytes', - title: 'Size', - render(record) { - return formatBytes(record.size_bytes) - }, - }, - { - accessor: 'actions', - render(record) { - return ( -
- { - confirmDownload(record) - }} + + {/* Source selector dropdown */} + {hasCustomLibraries && ( +
+ + +
+ )} + + {/* Default Kiwix library browser */} + {selectedSource === 'default' && ( + <> +
+ { + setQueryUI(e.target.value) + debouncedSetQuery(e.target.value) + }} + className="w-1/3" + leftIcon={} + /> +
+ + data={flatData.map((i, idx) => { + const row = virtualizer.getVirtualItems().find((v) => v.index === idx) + return { + ...i, + height: `${row?.size || 48}px`, + translateY: row?.start || 0, + } + })} + ref={tableParentRef} + loading={isLoading} + columns={[ + { + accessor: 'title', + }, + { + accessor: 'author', + }, + { + accessor: 'summary', + }, + { + accessor: 'updated', + render(record) { + return new Intl.DateTimeFormat('en-US', { + dateStyle: 'medium', + }).format(new Date(record.updated)) + }, + }, + { + accessor: 'size_bytes', + title: 'Size', + render(record) { + return formatBytes(record.size_bytes) + }, + }, + { + accessor: 'actions', + render(record) { + return ( +
+ { + confirmDownload(record) + }} + > + Download + +
+ ) + }, + }, + ]} + className="relative overflow-x-auto overflow-y-auto h-[600px] w-full mt-4" + tableBodyStyle={{ + position: 'relative', + height: `${virtualizer.getTotalSize()}px`, + }} + containerProps={{ + onScroll: (e) => fetchOnBottomReached(e.currentTarget as HTMLDivElement), + }} + compact + rowLines + /> + + )} + + {/* Custom library directory browser */} + {selectedSource !== 'default' && ( +
+ {/* Breadcrumb navigation */} +
- ) - }, - }, - ]} - className="relative overflow-x-auto overflow-y-auto h-[600px] w-full mt-4" - tableBodyStyle={{ - position: 'relative', - height: `${virtualizer.getTotalSize()}px`, - }} - containerProps={{ - onScroll: (e) => fetchOnBottomReached(e.currentTarget as HTMLDivElement), - }} - compact - rowLines - /> + {crumb.name} + + ) : ( + {crumb.name} + )} + + ))} + + + {isBrowsing && ( +
+
+
+ )} + + {browseError && ( + + )} + + {!isBrowsing && !browseError && browseData && ( +
+ {browseData.directories.length === 0 && browseData.files.length === 0 ? ( +

+ No directories or ZIM files found at this location. +

+ ) : ( + + + + + + + + + + {browseData.directories.map((dir) => ( + navigateToDirectory(dir.name, dir.url)} + > + + + + + ))} + {browseData.files.map((file) => ( + + + + + + ))} + +
NameSize
+ + + {dir.name} + + -- + +
+ + + {file.name} + + + {file.size_bytes ? formatBytes(file.size_bytes) : '--'} + + confirmCustomDownload(file)} + > + Download + +
+ )} +
+ )} +
+ )} + + + {/* Manage Custom Libraries Modal */} + setManageModalOpen(false)} + cancelText="Close" + > +
+
+

+ Add Kiwix mirrors or other ZIM file sources for faster downloads. +

+ + {/* Existing libraries */} + {customLibraries && customLibraries.length > 0 && ( +
+ {customLibraries.map((lib) => ( +
+
+

+ {lib.name} + {lib.is_default && ( + (built-in) + )} +

+

{lib.base_url}

+
+ {!lib.is_default && ( + + )} +
+ ))} +
+ )} + + {/* Add new library form */} +
+ setNewLibraryName(e.target.value)} + /> + setNewLibraryUrl(e.target.value)} + /> + addLibraryMutation.mutate()} + disabled={ + !newLibraryName.trim() || + !newLibraryUrl.trim() || + addLibraryMutation.isPending + } + > + Add Library + +
+
+
+
diff --git a/admin/start/routes.ts b/admin/start/routes.ts index 724734d7..fb0d47d2 100644 --- a/admin/start/routes.ts +++ b/admin/start/routes.ts @@ -183,6 +183,12 @@ router router.get('/wikipedia', [ZimController, 'getWikipediaState']) router.post('/wikipedia/select', [ZimController, 'selectWikipedia']) + + router.get('/custom-libraries', [ZimController, 'listCustomLibraries']) + router.post('/custom-libraries', [ZimController, 'addCustomLibrary']) + router.delete('/custom-libraries/:id', [ZimController, 'removeCustomLibrary']) + router.get('/browse-library', [ZimController, 'browseLibrary']) + router.delete('/:filename', [ZimController, 'delete']) }) .prefix('/api/zim') From 0ddcfe901105f3657b8d03f329471edf4ee58378 Mon Sep 17 00:00:00 2001 From: Jake Turner <52841588+jakeaturner@users.noreply.github.com> Date: Mon, 4 May 2026 11:54:56 -0700 Subject: [PATCH 045/108] fix(System): self-heal stale updateAvailable flag after sidecar-driven update (#825) --- admin/adonisrc.ts | 1 + admin/providers/version_check_provider.ts | 56 +++++++++++++++++++++++ 2 files changed, 57 insertions(+) create mode 100644 admin/providers/version_check_provider.ts diff --git a/admin/adonisrc.ts b/admin/adonisrc.ts index 34586ca4..741b1600 100644 --- a/admin/adonisrc.ts +++ b/admin/adonisrc.ts @@ -56,6 +56,7 @@ export default defineConfig({ () => import('#providers/map_static_provider'), () => import('#providers/kiwix_migration_provider'), () => import('#providers/qdrant_restart_policy_provider'), + () => import('#providers/version_check_provider'), ], /* diff --git a/admin/providers/version_check_provider.ts b/admin/providers/version_check_provider.ts new file mode 100644 index 00000000..20add77a --- /dev/null +++ b/admin/providers/version_check_provider.ts @@ -0,0 +1,56 @@ +import logger from '@adonisjs/core/services/logger' +import type { ApplicationService } from '@adonisjs/core/types' + +/** + * Self-heals stale `system.updateAvailable` after a sidecar-driven update. + * + * When the admin container is recreated on a new image, the KVStore still + * carries pre-update values for `system.updateAvailable` and + * `system.latestVersion`. Without intervention the UI keeps showing the + * "update available" banner until the next scheduled CheckUpdateJob (could be up to ~12h). + * + * Synchronous self-heal (no network): if the cached "latest" is not newer + * than the version we are now running, clear `updateAvailable`. The next + * scheduled CheckUpdateJob refreshes the cache from GitHub — we deliberately + * do not hit the network from boot to avoid coupling container startup to + * a network request to Github (e.g. container restart loop = flooding GitHub with requests). + * + * Note: this provider does not set `updateAvailable` to true if the cached + * "latest" is newer than the current version. We rely on the next scheduled + * CheckUpdateJob to do that, to avoid false positives in case of a stale cache. + */ +export default class VersionCheckProvider { + constructor(protected app: ApplicationService) { } + + async boot() { + if (this.app.getEnvironment() !== 'web') return + + setImmediate(async () => { + try { + const KVStore = (await import('#models/kv_store')).default + const { SystemService } = await import('#services/system_service') + const { isNewerVersion } = await import('../app/utils/version.js') + + const current = SystemService.getAppVersion() + if (current === 'dev' || current === '0.0.0'){ + logger.info(`[VersionCheckProvider] Skipping self-heal for version ${current}. Appears to be a dev build without proper version set.`) + return + } + + logger.info(`[VersionCheckProvider] Checking for stale updateAvailable (current=${current})`) + + const cachedLatest = (await KVStore.getValue('system.latestVersion')) as string | null + const earlyAccess = ((await KVStore.getValue('system.earlyAccess')) ?? false) as boolean + + if (cachedLatest && !isNewerVersion(cachedLatest, current, earlyAccess)) { + await KVStore.setValue('system.updateAvailable', false) + logger.info( + `[VersionCheckProvider] Cleared stale updateAvailable (cached=${cachedLatest}, current=${current})` + ) + } + } catch (err: any) { + logger.warn(`[VersionCheckProvider] Self-heal skipped: ${err?.message ?? err}`) + } + }) + } +} From 0fdf31c2e42874e8f7db2b537a487a9eced113ed Mon Sep 17 00:00:00 2001 From: Jake Turner Date: Mon, 4 May 2026 19:21:51 +0000 Subject: [PATCH 046/108] docs: update release notes --- admin/docs/release-notes.md | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/admin/docs/release-notes.md b/admin/docs/release-notes.md index c8c27662..cc7b6542 100644 --- a/admin/docs/release-notes.md +++ b/admin/docs/release-notes.md @@ -2,21 +2,57 @@ ## Unreleased +### Features +- **AI Assistant**: Added improved support for AMD GPU acceleration for Ollama via ROCm + HSA override. Thanks @chriscrosstalk for the contribution! +- **Content Explorer**: Added support for custom ZIM library sources and pre-seeded ZIM library mirrors in addition to the default Kiwix library. Thanks @chriscrosstalk for the contribution! +- **Content Manager**: Content update sizes and downloads are now properly displayed in Active Downloads with progress bars and friendly names. Thanks @chriscrosstalk for the contribution! +- **Maps**: Map regions can now be extracted and downloaded locally from PMTiles to avoid the need for a full global map download for users who only want specific regions. Thanks @bgauger for the contribution! + +### Bug Fixes +- **API**: Compression is now skipped for Server-Sent Events (SSE) responses to prevent issues with streaming endpoints. Thanks @chriscrosstalk for the fix! +- **Maps**: Fixed logic issues with the global map banner display. Thanks @Gujiassh for the fix! +- **Maps**: The selected map file is now properly deleted after confirming the action in the UI. Thanks @cuyua9 for the fix! +- **System**: Fixed an issue where the a pending update could still be indicated in the UI even after the system was updated successfully. Thanks @jakeaturner for the fix! + +### Improvements +- **Build**: The Command Center image now uses the VERSION build arg to write `app/version.json` with the current version for improved version tracking and debugging, even in RC environments. Thanks @chriscrosstalk for the contribution! +- **Content Manager**: Added a sortable file size column to the ZIM files table in the Content Manager for easier management of storage space. Thanks @chriscrosstalk for the contribution! +- **Dependencies**: All package.json dependencies have been pinned to specific versions to ensure stability and reduce the risk of unexpected breaking changes/supply-chain compromises from upstream packages. Thanks @jakeaturner for the contribution! +- **Dependencies**: Updated various dependencies to close security vulnerabilities and improve stability +- **Docs**: Update CONTIRBUTING.md to require an issue to be opened before submitting a PR for non-trivial changes to ensure proper discussion and review of proposed changes. Thanks @chriscrosstalk for the contribution! +- **Docs**: Added the map markers endpoints to the API reference documentation. Thanks @kennethbrewer3 for the contribution! +- **Docs**: Added a link to the new WSL2 install guide in the README and FAQ. Thanks @chriscrosstalk for the contribution! +- **Install**: The install script now warns loudly if the user is attempting to install on a non-x86_64/amd64 platform to prevent unsupported installations and potential issues. Thanks @chriscrosstalk for the contribution! +- **Maps**: The maps API endpoints now properly accept and validate notes, marker_type, and position data for map markers and persist them in the database for retrieval in the UI. Thanks @jrsphoto for the contribution! +- **Maps**: The current coordinates of the mouse pointer can now be displayed in the map viewer for easier navigation and exploration. Thanks @kennethbrewer3 for the contribution! +- **RAG**: NOMAD now properly passed `num_ctx` and truncation to the Ollama embedding endpoint to ensure that the context window of the model is best utilized for embeddings. Thanks @chriscrosstalk for the contribution! +- **RAG**: Added a manual start button for Qdrant and a self-healing mechanism for Qdrant's restart-policy to ensure that the vector database is running properly for embedding and retrieval tasks. Thanks @hestela for the contribution! + +## Version 1.31.1 - April 21, 2026 + ### Features ### Bug Fixes - **AI Assistant**: In-progress model downloads can now be cancelled properly and the progress UI now matches that of file downloads. Thanks @chriscrosstalk for the contribution! - **AI Assistant**: Fixed an issue where the AI Assistant settings page could crash if a model object did not have a details property. Thanks @hestela for the fix! +- **AI Assistant**: Fixed an issue with non-embeddable files being queued for embedding and flooding logs with errors. Thanks @sbruschke for the bug report and @chriscrosstalk for the fix! +- **AI Assistant**: Fixed an issue with ZIM batch embedding using the wrong batch count and causing remaining batches to be skipped. Thanks @sbruschke for the bug report and @chriscrosstalk for the fix! +- **AI Assistant**: Fixed an issue with ZIM content extraction only extracting the first-level children of the article body and thus missing a lot of content. Thanks @sbruschke for the bug report and @chriscrosstalk for the fix! - **Disk Collector**: Improved reporting for NFS mount stats and display in the UI. Thanks @bgauger and @bravosierra99 for the contribution! - **Downloads**: Downloads are now staged to .tmp files and atomically renamed upon completion to prevent issues with incomplete/corrupt files. Thanks @artbird309 for the contribution! - **Downloads**: Removed a duplicate error listener and improved stability when handling Range requests for file downloads. Thanks @jakeaturner for the contribution! - **Downloads**: Added improved handling for corrupt ZIM file downloads and removed duplicate Ollama download logs. Thanks @aegisman for the contribution! - **Security**: Closed a potential SSRF vulnerability in the map file download functionality by implementing stricter URL validation and blocking private IP ranges. Thanks @LuisMIguelFurlanettoSousa for the fix! - **Security**: Sanitized error messages from the backend to prevent potential information disclosure. Thanks @LuisMIguelFurlanettoSousa for the fix! +- **UI**: Fixed an issue with broken pagination for the Content Explorer that could cause some users to see a "No records found" message indefinitely. Thanks @johno10661 for the bug report and @chriscrosstalk for the fix! +- **UI**: Fixed an issue where all storage devices could report as "NAS Storage" regardless of actual type. Thanks @bgauger for the fix! ### Improvements - **AI Assistant**: Now uses the currently loaded model for query rewriting and chat title generation for improved performance and consistency. Thanks @hestela for the contribution! +- **AI Assistant**: When a remote Ollama URL is configured, the Command Center will now attempt to stop NOMAD's local Ollama container to free up resources and avoid confusion. Thanks @chriscrosstalk for the contribution! - **Dependencies**: Updated various dependencies to close security vulnerabilities and improve stability +- **Docs**: Added a "Community Add-Ons" page to the documentation to highlight some of the amazing community contributions that have been made since launch. Thanks @chriscrosstalk for the contribution! +- **Privacy**: Added the appropriate environment variable to disable telemetry for the Qdrant container. Note that this will only take effect on new installations of if the Qdrant container is force re-installed on existing installations. Thanks @berkdamerc for the find and @chriscrosstalk for the contribution! ## Version 1.31.0 - April 3, 2026 From 1ad898bc8b7222961e07cbaf4e1fc67258372e4e Mon Sep 17 00:00:00 2001 From: chriscrosstalk <49691103+chriscrosstalk@users.noreply.github.com> Date: Tue, 5 May 2026 10:33:18 -0700 Subject: [PATCH 047/108] fix(UI): four fixes for the System Update page (#827) Closes #826. 1. Heading and subtext now read from `versionInfo` state (which the Check Again mutation already populates) instead of the server-rendered `props.system`. Previously the card kept showing "System Up to Date / Your system is running the latest version!" alongside the new `Latest Version` row + Start Update button after a successful recheck. Status icon also switched to `versionInfo` for consistency. 2. The pulling-state heading rendered the lowercase status enum (`pulling`, `pulled`, ...) and relied on a Tailwind `capitalize` class for the visible glyph. Screen readers and other accessible-name consumers got the lowercase value with no transform applied. Replaced with a `STAGE_LABELS` map so visual + accessible names match. 3. The sidecar (install/sidecar-updater/update-watcher.sh) writes `complete` for ~5s, then resets the status file to `idle`. The SPA could miss that window across the admin container restart, leaving the page parked on its last observed progress percentage indefinitely while the upgrade was actually finished on disk. A `seenAdvancedStageRef` now records whether the session ever observed an advanced stage; a later poll seeing `idle` is treated as the missed completion, and the page reloads as advertised in step 3 of the on-screen process. Reset on each Start Update. 4. Toggling Enable Early Access now triggers a recheck on success, so the eligible-version list updates immediately instead of requiring a manual Check Again click. Single file touched: admin/inertia/pages/settings/update.tsx. Typecheck (tsc --noEmit) passes; static UI changes verified in source. Co-authored-by: Claude Opus 4.7 (1M context) --- admin/inertia/pages/settings/update.tsx | 56 +++++++++++++++++++++---- 1 file changed, 47 insertions(+), 9 deletions(-) diff --git a/admin/inertia/pages/settings/update.tsx b/admin/inertia/pages/settings/update.tsx index 348040da..06db3bbd 100644 --- a/admin/inertia/pages/settings/update.tsx +++ b/admin/inertia/pages/settings/update.tsx @@ -5,7 +5,7 @@ import StyledTable from '~/components/StyledTable' import StyledSectionHeader from '~/components/StyledSectionHeader' import ActiveDownloads from '~/components/ActiveDownloads' import Alert from '~/components/Alert' -import { useEffect, useState } from 'react' +import { useEffect, useRef, useState } from 'react' import { IconAlertCircle, IconArrowBigUpLines, IconCheck, IconCircleCheck, IconReload } from '@tabler/icons-react' import { SystemUpdateStatus } from '../../../types/system' import type { ContentUpdateCheckResult, ResourceUpdateInfo } from '../../../types/collections' @@ -24,6 +24,23 @@ type Props = { earlyAccess: boolean } +const STAGE_LABELS: Record = { + idle: 'Preparing Update', + starting: 'Starting Update', + pulling: 'Pulling Images', + pulled: 'Images Pulled', + recreating: 'Recreating Containers', + complete: 'Update Complete', + error: 'Update Failed', +} + +const ADVANCED_STAGES: ReadonlySet = new Set([ + 'pulling', + 'pulled', + 'recreating', + 'complete', +]) + function ContentUpdatesSection() { const { addNotification } = useNotifications() const queryClient = useQueryClient() @@ -251,6 +268,12 @@ export default function SystemUpdatePage(props: { system: Props }) { const [email, setEmail] = useState('') const [versionInfo, setVersionInfo] = useState>(props.system) const [showConnectionLostNotice, setShowConnectionLostNotice] = useState(false) + // Tracks whether this update session has progressed past 'idle'/'starting'. + // The sidecar sits on 'complete' for ~5s before resetting to 'idle' (see + // install/sidecar-updater/update-watcher.sh), and the SPA can miss that + // window across the admin container restart. If we resurface to 'idle' + // after seeing an advanced stage, treat it as the missed completion. + const seenAdvancedStageRef = useRef(false) const earlyAccessSetting = useSystemSetting({ key: 'system.earlyAccess', initialData: { @@ -270,11 +293,22 @@ export default function SystemUpdatePage(props: { system: Props }) { } setUpdateStatus(response) + if (ADVANCED_STAGES.has(response.stage)) { + seenAdvancedStageRef.current = true + } + // If we can connect again, hide the connection lost notice setShowConnectionLostNotice(false) - // Check if update is complete or errored - if (response.stage === 'complete') { + // Check if update is complete or errored. We also treat a return to + // 'idle' as completion if we previously saw an advanced stage — this + // catches the race where the sidecar's brief 'complete' window passes + // while we're disconnected during the admin container restart. + const isComplete = + response.stage === 'complete' || + (response.stage === 'idle' && seenAdvancedStageRef.current) + + if (isComplete) { // Re-check version so the KV store clears the stale "update available" flag // before we reload, otherwise the banner shows "current → current" try { @@ -304,6 +338,7 @@ export default function SystemUpdatePage(props: { system: Props }) { const handleStartUpdate = async () => { try { setError(null) + seenAdvancedStageRef.current = false setIsUpdating(true) const response = await api.startSystemUpdate() if (!response || !response.success) { @@ -368,7 +403,7 @@ export default function SystemUpdatePage(props: { system: Props }) { if (updateStatus?.stage === 'error') return if (isUpdating) return - if (props.system.updateAvailable) + if (versionInfo.updateAvailable) return return } @@ -380,6 +415,9 @@ export default function SystemUpdatePage(props: { system: Props }) { onSuccess: () => { addNotification({ message: 'Setting updated successfully.', type: 'success' }) earlyAccessSetting.refetch() + // Toggling Early Access changes which versions are eligible, so re-evaluate + // immediately rather than making the user click Check Again. + checkVersionMutation.mutate() }, onError: (error) => { console.error('Error updating setting:', error) @@ -461,11 +499,11 @@ export default function SystemUpdatePage(props: { system: Props }) { {!isUpdating && ( <>

- {props.system.updateAvailable ? 'Update Available' : 'System Up to Date'} + {versionInfo.updateAvailable ? 'Update Available' : 'System Up to Date'}

- {props.system.updateAvailable - ? `A new version (${props.system.latestVersion}) is available for your Project N.O.M.A.D. instance.` + {versionInfo.updateAvailable + ? `A new version (${versionInfo.latestVersion}) is available for your Project N.O.M.A.D. instance.` : 'Your system is running the latest version!'}

@@ -473,8 +511,8 @@ export default function SystemUpdatePage(props: { system: Props }) { {isUpdating && updateStatus && ( <> -

- {updateStatus.stage === 'idle' ? 'Preparing Update' : updateStatus.stage} +

+ {STAGE_LABELS[updateStatus.stage] ?? updateStatus.stage}

{updateStatus.message}

From cb47e1e4e35a9ffa7f11f168f716802911b10194 Mon Sep 17 00:00:00 2001 From: cosmistack-bot Date: Mon, 4 May 2026 19:32:01 +0000 Subject: [PATCH 048/108] chore(release): 1.32.0-rc.1 [skip ci] --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 2a8ff50b..86b418cd 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "project-nomad", - "version": "1.31.0", + "version": "1.32.0-rc.1", "description": "\"", "main": "index.js", "scripts": { From 03ab614f9961eb9c8ab968fdaf5540a8723fac11 Mon Sep 17 00:00:00 2001 From: Ben Gauger Date: Mon, 4 May 2026 14:39:10 -0600 Subject: [PATCH 049/108] fix(Maps): send filename instead of full path to delete endpoint --- admin/inertia/pages/settings/maps.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/admin/inertia/pages/settings/maps.tsx b/admin/inertia/pages/settings/maps.tsx index f95fa56f..d2df0f32 100644 --- a/admin/inertia/pages/settings/maps.tsx +++ b/admin/inertia/pages/settings/maps.tsx @@ -138,7 +138,7 @@ export default function MapsManager(props: { try { setDeletingFileKey(file.key) - await api.deleteMapRegionFile(file.key) + await api.deleteMapRegionFile(file.name) addNotification({ type: 'success', message: `${file.name} has been deleted.`, From 63282565a958648064b0fbe35bf2a54ee9d1cad4 Mon Sep 17 00:00:00 2001 From: Chris Sherwood Date: Mon, 4 May 2026 15:41:23 -0700 Subject: [PATCH 050/108] fix(Maps): render notes in marker popup when populated MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Closes #796. The maps API has accepted and persisted `notes` on map markers since PR #770, but the marker popup component still rendered name only and ignored the field. Now the popup shows a notes block beneath the name when it's populated, with whitespace preserved and long text wrapped. Threaded `notes` through the read path: - `api.listMapMarkers` / `api.createMapMarker` response types - `MapMarker` interface in `useMapMarkers` and the data.map projection - `MapComponent`'s selectedMarker popup The create/update UI is unchanged — users still set notes via the API or DB directly, matching the issue's stated scope. A marker entry with empty/whitespace-only notes renders the same as before. Co-Authored-By: Claude Opus 4.7 (1M context) --- admin/inertia/components/maps/MapComponent.tsx | 5 +++++ admin/inertia/hooks/useMapMarkers.ts | 3 +++ admin/inertia/lib/api.ts | 4 ++-- 3 files changed, 10 insertions(+), 2 deletions(-) diff --git a/admin/inertia/components/maps/MapComponent.tsx b/admin/inertia/components/maps/MapComponent.tsx index 07fe1c65..f7a3a8ad 100644 --- a/admin/inertia/components/maps/MapComponent.tsx +++ b/admin/inertia/components/maps/MapComponent.tsx @@ -243,6 +243,11 @@ export default function MapComponent({ closeOnClick={false} >
{selectedMarker.name}
+ {selectedMarker.notes && selectedMarker.notes.trim() && ( +
+ {selectedMarker.notes} +
+ )} )} diff --git a/admin/inertia/hooks/useMapMarkers.ts b/admin/inertia/hooks/useMapMarkers.ts index ba999ebb..a0e818e5 100644 --- a/admin/inertia/hooks/useMapMarkers.ts +++ b/admin/inertia/hooks/useMapMarkers.ts @@ -18,6 +18,7 @@ export interface MapMarker { longitude: number latitude: number color: PinColorId + notes: string | null createdAt: string } @@ -36,6 +37,7 @@ export function useMapMarkers() { longitude: m.longitude, latitude: m.latitude, color: m.color as PinColorId, + notes: m.notes ?? null, createdAt: m.created_at, })) ) @@ -54,6 +56,7 @@ export function useMapMarkers() { longitude: result.longitude, latitude: result.latitude, color: result.color as PinColorId, + notes: result.notes ?? null, createdAt: result.created_at, } setMarkers((prev) => [...prev, marker]) diff --git a/admin/inertia/lib/api.ts b/admin/inertia/lib/api.ts index 3e015d84..aa1369b7 100644 --- a/admin/inertia/lib/api.ts +++ b/admin/inertia/lib/api.ts @@ -631,7 +631,7 @@ class API { async listMapMarkers() { return catchInternal(async () => { const response = await this.client.get< - Array<{ id: number; name: string; longitude: number; latitude: number; color: string; created_at: string }> + Array<{ id: number; name: string; longitude: number; latitude: number; color: string; notes: string | null; created_at: string }> >('/maps/markers') return response.data })() @@ -640,7 +640,7 @@ class API { async createMapMarker(data: { name: string; longitude: number; latitude: number; color?: string }) { return catchInternal(async () => { const response = await this.client.post< - { id: number; name: string; longitude: number; latitude: number; color: string; created_at: string } + { id: number; name: string; longitude: number; latitude: number; color: string; notes: string | null; created_at: string } >('/maps/markers', data) return response.data })() From 0b25638a3e8a069197b996f58daa0cb815b0518f Mon Sep 17 00:00:00 2001 From: Chris Sherwood Date: Tue, 5 May 2026 09:00:38 -0700 Subject: [PATCH 051/108] fix(AI): vendor-aware AMD HSA override + benchmark discrete-GPU detection MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Closes #810. ## Bug A: HSA_OVERRIDE_GFX_VERSION=11.0.0 was unconditional PR #804 set HSA_OVERRIDE_GFX_VERSION=11.0.0 for any AMD GPU. The inline comment claimed this was harmless on supported discrete cards (gfx1030 RX 6800, etc.) — empirically false. With the override, Ollama crashes during GPU discovery on gfx1030 and falls back to CPU silently. Affects every NOMAD user with an RX 6800 or other RDNA 2 discrete card. The correct value depends on the gfx version: - gfx1030, gfx1100, gfx1101, gfx1102: officially supported by ROCm — no override - gfx1031..gfx1036 (RDNA 2 variants + iGPUs like Rembrandt 680M): 10.3.0 - gfx1103, gfx1150, gfx1151 (Phoenix 780M, Strix 890M, Strix Halo): 11.0.0 ### Resolution chain in `_resolveAmdHsaOverride()` 1. KV `ai.amdHsaOverride` — manual override; accepts 'none' to disable, or a semver-style value to force. 2. Marker file `/app/storage/.nomad-amd-gfx` — written by install_nomad.sh based on lspci codename. Mapped to override via `_mapGfxToHsaOverride()`. 3. Default: `11.0.0` — preserves prior behavior so existing iGPU users (780M / 890M, the dominant AMD population today) don't regress on upgrade. Discrete RDNA 2 users on existing installs can opt out via `ai.amdHsaOverride='none'` and force-reinstall AI Assistant, OR re-run install_nomad.sh to refresh the marker file. The helper is used in both `createContainer` (initial install) and `updateContainer` (image update) paths, replacing the unconditional push. ## Bug B: BenchmarkService had no AMD discrete detection path `BenchmarkService.getHardwareInfo()` had three GPU detection fallbacks: 1. `si.graphics()` — empty inside Docker for AMD 2. nvidia-smi — NVIDIA only 3. AMD APU regex from CPU model — integrated only Result: AMD discrete cards (RX 6800, RX 7900 XTX, etc.) showed up as "GPU: Not detected" on the leaderboard despite ROCm working. Corrupts leaderboard data quality for that population. Fix: after the existing fallbacks, call `SystemService.getSystemInfo()` and read `graphics.controllers[0].model`. That path already handles AMD via the marker file + Ollama log probe added in PR #804, so we're reusing existing plumbing rather than duplicating detection logic. ## install_nomad.sh changes The existing AMD detection block already runs lspci. Added a codename parse step that maps Navi 21/22/23/24, Rembrandt, Phoenix1/Phoenix2, Strix/Strix Point/Strix Halo, and Navi 31/32/33 to gfx versions, then writes `/opt/project-nomad/storage/.nomad-amd-gfx`. Unknown codenames write nothing (admin handles missing-marker case via the backward-compat default). ## Validation Both bugs were originally surfaced and validated empirically on RX 6800 / gfx1030 / Ubuntu 24.04 + kernel 6.17 + ollama/ollama:rocm during the #810 filing. Validation grid from that report: | Run | NOMAD Score | tok/s | GPU detected | |-----------------------------------------------|-------------|-------|-------------------------| | Pre-fix (Bug A active) | n/a | 0 | yes, but library=cpu | | HSA_OVERRIDE removed, Bug B unfixed | 73.8 | 221.6 | "Not detected" | | Both fixes hot-patched (this PR's behavior) | 73.7 | 216.0 | AMD Radeon RX 6800 | Local checks: `npm run typecheck` clean, `npm run build` clean. --- admin/app/services/benchmark_service.ts | 17 +++++ admin/app/services/docker_service.ts | 85 ++++++++++++++++++++++--- admin/types/kv_store.ts | 1 + install/install_nomad.sh | 40 ++++++++++++ 4 files changed, 133 insertions(+), 10 deletions(-) diff --git a/admin/app/services/benchmark_service.ts b/admin/app/services/benchmark_service.ts index 80247f70..47e4cf1f 100644 --- a/admin/app/services/benchmark_service.ts +++ b/admin/app/services/benchmark_service.ts @@ -317,6 +317,23 @@ export class BenchmarkService { } } + // Fallback: AMD discrete cards. si.graphics() returns empty inside Docker for AMD, + // the nvidia-smi path doesn't apply, and the APU regex only catches integrated parts. + // SystemService.getSystemInfo() already handles AMD via the marker file + Ollama log + // probe added in PR #804, so reuse that plumbing rather than duplicating it here. + if (!gpuModel) { + try { + const systemService = new (await import('./system_service.js')).SystemService(this.dockerService) + const sysInfo = await systemService.getSystemInfo() + const sysGpuModel = sysInfo?.graphics?.controllers?.[0]?.model + if (sysGpuModel) { + gpuModel = sysGpuModel + } + } catch (sysError: any) { + logger.warn(`[BenchmarkService] system_service AMD fallback failed: ${sysError.message}`) + } + } + return { cpu_model: `${cpu.manufacturer} ${cpu.brand}`, cpu_cores: cpu.physicalCores, diff --git a/admin/app/services/docker_service.ts b/admin/app/services/docker_service.ts index 7501e78c..7e8fc4ee 100644 --- a/admin/app/services/docker_service.ts +++ b/admin/app/services/docker_service.ts @@ -592,10 +592,12 @@ export class DockerService { ollamaEnv.push('OLLAMA_FLASH_ATTENTION=1') } if (amdGpuConfigured) { - // RDNA3 iGPUs (gfx1103: 780M, 880M, 890M, ...) aren't on AMD's official ROCm - // allowlist but work when forced to identify as gfx1100 via HSA_OVERRIDE_GFX_VERSION. - // Harmless on supported discrete cards (gfx1030 RX 6800, etc.) — they ignore the override. - ollamaEnv.push('HSA_OVERRIDE_GFX_VERSION=11.0.0') + // gfx-aware HSA override — only set for cards that actually need it. See + // _resolveAmdHsaOverride() for the resolution order and gfx → version mapping. + const hsaOverride = await this._resolveAmdHsaOverride() + if (hsaOverride) { + ollamaEnv.push(`HSA_OVERRIDE_GFX_VERSION=${hsaOverride}`) + } } } @@ -999,6 +1001,67 @@ export class DockerService { } } + /** + * Resolve the HSA_OVERRIDE_GFX_VERSION value for the host's AMD GPU. + * + * gfx1030 (RX 6800/6700/etc.), gfx1100/1101/1102 (RX 7900/7800/7600) are on AMD's + * official ROCm allowlist — forcing an override on these breaks GPU discovery. + * gfx1035 / gfx1036 (RDNA 2 iGPUs like 680M) need 10.3.0 to coerce to gfx1030. + * gfx1103 / gfx1150 / gfx1151 (RDNA 3/3.5 iGPUs like 780M / 890M / Strix Halo) need 11.0.0. + * + * Resolution order: + * 1. KV `ai.amdHsaOverride` — manual user override; accepts 'none' (disable) or a semver-style value. + * 2. Marker file `/app/storage/.nomad-amd-gfx` written by install_nomad.sh. + * 3. Default: '11.0.0' — preserves prior behavior so existing iGPU users don't regress on + * upgrade. Discrete-card users on existing installs can opt out via the KV. + * + * Returns null when no override should be applied. + */ + private async _resolveAmdHsaOverride(): Promise { + const manualRaw = await KVStore.getValue('ai.amdHsaOverride') + if (manualRaw !== null && manualRaw !== undefined && String(manualRaw).trim() !== '') { + const manual = String(manualRaw).trim().toLowerCase() + if (manual === 'none' || manual === 'off' || manual === 'false') { + logger.info('[DockerService] HSA override disabled via ai.amdHsaOverride') + return null + } + if (/^\d+\.\d+\.\d+$/.test(manual)) { + logger.info(`[DockerService] HSA override forced to ${manual} via ai.amdHsaOverride`) + return manual + } + logger.warn(`[DockerService] Ignoring invalid ai.amdHsaOverride value: ${manualRaw}`) + } + + try { + const gfx = (await readFile('/app/storage/.nomad-amd-gfx', 'utf8')).trim() + const mapped = this._mapGfxToHsaOverride(gfx) + logger.info(`[DockerService] AMD gfx marker '${gfx}' → HSA override ${mapped ?? 'none'}`) + return mapped + } catch { + // Marker absent — most likely an existing install upgraded without re-running + // install_nomad.sh. Fall through to the default. + } + + logger.info('[DockerService] No AMD gfx marker; defaulting HSA override to 11.0.0 for backward compatibility') + return '11.0.0' + } + + private _mapGfxToHsaOverride(gfx: string): string | null { + // Officially supported by ROCm — no override needed + if (gfx === 'gfx1030' || gfx === 'gfx1100' || gfx === 'gfx1101' || gfx === 'gfx1102') { + return null + } + // RDNA 2 variants + iGPUs (gfx1031..gfx1036, e.g. Rembrandt 680M) + if (/^gfx103[1-6]$/.test(gfx)) { + return '10.3.0' + } + // RDNA 3 / 3.5 mobile parts (Phoenix 780M = gfx1103, Strix 890M = gfx1150, Strix Halo = gfx1151) + if (gfx === 'gfx1103' || gfx === 'gfx1150' || gfx === 'gfx1151') { + return '11.0.0' + } + return '11.0.0' + } + /** * Build the Docker Devices array for AMD GPU passthrough. * @@ -1132,12 +1195,14 @@ export class DockerService { // and whether HSA_OVERRIDE needs injection. For AMD, replace any prior HSA_OVERRIDE in // the inspect-captured env so updates from older containers pick up the current value. const baseEnv = inspectData.Config?.Env || [] - const finalEnv = updatedAmdGpuConfigured - ? [ - ...baseEnv.filter((e: string) => !e.startsWith('HSA_OVERRIDE_GFX_VERSION=')), - 'HSA_OVERRIDE_GFX_VERSION=11.0.0', - ] - : baseEnv + let finalEnv = baseEnv + if (updatedAmdGpuConfigured) { + const hsaOverride = await this._resolveAmdHsaOverride() + finalEnv = baseEnv.filter((e: string) => !e.startsWith('HSA_OVERRIDE_GFX_VERSION=')) + if (hsaOverride) { + finalEnv.push(`HSA_OVERRIDE_GFX_VERSION=${hsaOverride}`) + } + } const newContainerConfig: any = { Image: newImage, diff --git a/admin/types/kv_store.ts b/admin/types/kv_store.ts index a3632ab0..7974e952 100644 --- a/admin/types/kv_store.ts +++ b/admin/types/kv_store.ts @@ -13,6 +13,7 @@ export const KV_STORE_SCHEMA = { 'ai.remoteOllamaUrl': 'string', 'ai.ollamaFlashAttention': 'boolean', 'ai.amdGpuAcceleration': 'boolean', + 'ai.amdHsaOverride': 'string', } as const type KVTagToType = T extends 'boolean' ? boolean : string diff --git a/install/install_nomad.sh b/install/install_nomad.sh index ef501a0d..484cea1f 100644 --- a/install/install_nomad.sh +++ b/install/install_nomad.sh @@ -520,10 +520,40 @@ verify_gpu_setup() { # Check for AMD GPU — restrict to display controller classes to avoid false positives # from AMD CPU host bridges, PCI bridges, and chipset devices. local has_amd_gpu='false' + local amd_gfx_version='' if command -v lspci &> /dev/null; then if lspci 2>/dev/null | grep -iE "VGA|3D controller|Display" | grep -iE "amd|radeon" &> /dev/null; then has_amd_gpu='true' echo -e "${GREEN}✓${RESET} AMD GPU detected — ROCm acceleration will be configured automatically when AI Assistant is installed.\\n" + + # Map AMD codename → gfx version so the admin can pick the right HSA_OVERRIDE_GFX_VERSION. + # gfx1030/1100/1101/1102 are on AMD's official ROCm allowlist and need NO override — + # forcing one (e.g. 11.0.0) breaks GPU discovery on these. Other variants do need it. + local amd_devices + amd_devices=$(lspci -vmm 2>/dev/null | awk -F'\t' '/^Class:.*(VGA|3D|Display)/{c=1} c && /^Device:/{print $2; c=0}') + if echo "${amd_devices}" | grep -iq 'Navi 21'; then + amd_gfx_version='gfx1030' + elif echo "${amd_devices}" | grep -iq 'Navi 22'; then + amd_gfx_version='gfx1031' + elif echo "${amd_devices}" | grep -iq 'Navi 23'; then + amd_gfx_version='gfx1032' + elif echo "${amd_devices}" | grep -iq 'Navi 24'; then + amd_gfx_version='gfx1034' + elif echo "${amd_devices}" | grep -iq 'Rembrandt'; then + amd_gfx_version='gfx1035' + elif echo "${amd_devices}" | grep -iEq 'Phoenix1?|Phoenix2'; then + amd_gfx_version='gfx1103' + elif echo "${amd_devices}" | grep -iEq 'Strix Halo'; then + amd_gfx_version='gfx1151' + elif echo "${amd_devices}" | grep -iEq 'Strix( Point)?'; then + amd_gfx_version='gfx1150' + elif echo "${amd_devices}" | grep -iq 'Navi 31'; then + amd_gfx_version='gfx1100' + elif echo "${amd_devices}" | grep -iq 'Navi 32'; then + amd_gfx_version='gfx1101' + elif echo "${amd_devices}" | grep -iq 'Navi 33'; then + amd_gfx_version='gfx1102' + fi fi fi @@ -539,6 +569,16 @@ verify_gpu_setup() { sudo rm -f "${gpu_marker_path}" 2>/dev/null || true fi + # Companion marker used by the admin to pick the right HSA_OVERRIDE_GFX_VERSION for + # the detected card. Absence of this file means "unknown gfx" — the admin falls back + # to its built-in default. Always rewrite (or remove) on install to keep state fresh. + local amd_gfx_marker_path="${NOMAD_DIR}/storage/.nomad-amd-gfx" + if [[ -n "${amd_gfx_version}" ]]; then + echo "${amd_gfx_version}" | sudo tee "${amd_gfx_marker_path}" > /dev/null 2>&1 || true + else + sudo rm -f "${amd_gfx_marker_path}" 2>/dev/null || true + fi + echo -e "${YELLOW}===========================================${RESET}\\n" # Summary From c8f675f56276f5f9b7fa762bcc3a0d19db77c35d Mon Sep 17 00:00:00 2001 From: Jake Turner Date: Tue, 5 May 2026 19:25:30 +0000 Subject: [PATCH 052/108] chore: trigger release [rc2] From 38b714283d30d1564bbb4ab42b4dea0750fff9b3 Mon Sep 17 00:00:00 2001 From: Jake Turner Date: Tue, 5 May 2026 19:33:08 +0000 Subject: [PATCH 053/108] chore(release): 1.32.0-rc.2 [skip ci] --- package-lock.json | 4 ++-- package.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/package-lock.json b/package-lock.json index 1ee61ece..0a83eae0 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "project-nomad", - "version": "1.27.0", + "version": "1.32.0-rc.2", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "project-nomad", - "version": "1.27.0", + "version": "1.32.0-rc.2", "license": "ISC" } } diff --git a/package.json b/package.json index 86b418cd..88a4d88e 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "project-nomad", - "version": "1.32.0-rc.1", + "version": "1.32.0-rc.2", "description": "\"", "main": "index.js", "scripts": { From 525a1f1789a479180c660a44492ffe716528efb0 Mon Sep 17 00:00:00 2001 From: Ben Gauger Date: Thu, 7 May 2026 18:39:58 -0600 Subject: [PATCH 054/108] fix(System): correct NVIDIA VRAM in Graphics card (#835) PR #804 added pciutils to the admin image so AMD detection could fall back to lspci. Side effect for NVIDIA: si.graphics() now finds the card via lspci but reads the BAR0 region size (16-32 MiB on most NVIDIA cards) as VRAM, since nvidia-smi isn't installed in the admin image to enrich the result. A GTX 1050 Ti showed 32 MB instead of 4096. The nvidia-smi-via-Ollama and Ollama log probes already give the right number, but they only ran when graphics.controllers came back empty. Extend the trigger so they also run when the only entries are NVIDIA controllers reporting under 256 MiB (no real dGPU has that little). If the probes can't reach a value either (Ollama not installed, passthrough broken), VRAM now falls back to N/A instead of the bogus 32. Verified locally on an RTX 4070 Ti by simulating the container condition (lspci available, nvidia-smi unreachable). Before the fix: vram 32, model "AD104 [GeForce RTX 4070 Ti]". After: vram 12288, model "NVIDIA GeForce RTX 4070 Ti", from the Ollama inference-compute log line. Also confirmed the same result inside the actual admin Docker image. --- admin/app/services/system_service.ts | 35 +++++++++++++++++++++++++--- 1 file changed, 32 insertions(+), 3 deletions(-) diff --git a/admin/app/services/system_service.ts b/admin/app/services/system_service.ts index 1a55cfb2..850e85a5 100644 --- a/admin/app/services/system_service.ts +++ b/admin/app/services/system_service.ts @@ -399,9 +399,38 @@ export class SystemService { os.kernel = dockerInfo.KernelVersion } - // If si.graphics() returned no controllers (common inside Docker), - // fall back to runtime + Ollama log probe to figure out what's accessible. - if (!graphics.controllers || graphics.controllers.length === 0) { + // si.graphics() in the admin container uses lspci (pciutils ships in + // the image for AMD detection). lspci has no real VRAM info for NVIDIA + // cards, so systeminformation parses the first PCI memory Region (BAR0, + // 16-32 MiB on most NVIDIA cards) as `vram`. nvidia-smi enrichment also + // can't run since the binary isn't in the admin image. No real dGPU + // has under 256 MiB, so any NVIDIA controller below that needs the + // probes below to give us real data. + const NVIDIA_BOGUS_VRAM_THRESHOLD_MIB = 256 + const isBogusNvidiaVram = (c: { vendor?: string; vram?: number | null }) => + /nvidia/i.test(c.vendor || '') && + typeof c.vram === 'number' && + c.vram < NVIDIA_BOGUS_VRAM_THRESHOLD_MIB + + // Clear the bogus value up front. If a probe replaces the entry below + // we get the real VRAM; if no probe succeeds (Ollama not installed, + // passthrough_failed) the UI falls back to "N/A" instead of showing + // "32 MB". The lspci model/vendor strings stay since they're still + // useful for identifying the card. + const hasLspciBogusNvidiaVram = (graphics.controllers || []).some(isBogusNvidiaVram) + if (hasLspciBogusNvidiaVram) { + for (const c of graphics.controllers) { + if (isBogusNvidiaVram(c)) c.vram = null + } + } + + // Run the probes when controllers are empty (common inside Docker) or + // when lspci gave us bogus NVIDIA BAR0 values that need replacing. + if ( + !graphics.controllers || + graphics.controllers.length === 0 || + hasLspciBogusNvidiaVram + ) { const runtimes = dockerInfo.Runtimes || {} gpuHealth.hasNvidiaRuntime = 'nvidia' in runtimes From 0a7bd9b11b76ced186555f0f703883a65a0afbe8 Mon Sep 17 00:00:00 2001 From: Chris Sherwood Date: Mon, 11 May 2026 10:12:47 -0700 Subject: [PATCH 055/108] fix(AI): preserve semver tag in DB on AMD Ollama updates Closes #855. PR #804's AMD branch in `updateContainer()` overrode `newImage` to `ollama/ollama:rocm` and then persisted that literal string to `service.container_image` (line 1273). Two downstream consequences for every AMD user who clicked Update on AI Assistant: 1. Apps page (`apps.tsx`) extracts the displayed version from `container_image` and rendered the literal string "rocm". 2. `ContainerRegistryService.getAvailableUpdates()` parsed `currentTag = "rocm"`, which isn't semver, so `parseMajorVersion` returned NaN, the filter didn't reject newer tags by major-version, and `isNewerVersion` treated any future tag as newer. Result: the same update reappeared on every check, forever. Fix: separate "what we run" from "what we persist". `runtimeImage` holds the tag passed to `docker.pull()` and `createContainer()` (still `:rocm` for AMD), while `newImage` keeps the semver tag and is the value written to the DB. Surgical: 3 references renamed plus 1 declaration added. The install path (`_createContainer`) already had the right shape (runtime-only override, no DB write of the override), so this PR only touches `updateContainer`. Test plan: - `npm run typecheck` passes locally. - Manual repro on NOMAD2 (AMD HX 370 / 890M, rc.2): before fix, DB shows `container_image = ollama/ollama:rocm` after triggering an Ollama update via Settings > Apps; Apps page shows version "rocm"; `/api/system/services/check-updates` immediately re-reports the same update available. After fix, DB shows `container_image = ollama/ollama:`; Apps page shows the semver; check- updates does not re-report the same update. - nomad_ollama container itself still runs the `:rocm` image (verified via `docker inspect`). Co-Authored-By: Claude Opus 4.7 (1M context) --- admin/app/services/docker_service.ts | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/admin/app/services/docker_service.ts b/admin/app/services/docker_service.ts index 7e8fc4ee..d0540685 100644 --- a/admin/app/services/docker_service.ts +++ b/admin/app/services/docker_service.ts @@ -1103,13 +1103,17 @@ export class DockerService { this.activeInstallations.add(serviceName) - // Compute new image string. AMD-on-Ollama overrides this to the rolling :rocm tag - // (set during GPU detection below) since per-version ROCm tags aren't always published. + // newImage = the semver tag we record in the DB after the update (e.g. ollama/ollama:0.23.2). + // runtimeImage = the tag we actually pull and run. For AMD-on-Ollama these diverge: we run + // the rolling :rocm tag because per-version ROCm tags aren't always published, but the DB + // must keep the semver tag so the Apps page shows the actual version (not literally "rocm") + // and the registry update-check parses a valid tag (instead of looping on the same update). const currentImage = service.container_image const imageBase = currentImage.includes(':') ? currentImage.substring(0, currentImage.lastIndexOf(':')) : currentImage - let newImage = `${imageBase}:${targetVersion}` + const newImage = `${imageBase}:${targetVersion}` + let runtimeImage = newImage // GPU detection runs before the pull so AMD updates pull ollama/ollama:rocm rather // than the standard tag. Detection result is reused below when building the new @@ -1137,7 +1141,7 @@ export class DockerService { 'update-gpu-config', `AMD GPU detected. Using ROCm image with /dev/kfd and /dev/dri passthrough...` ) - newImage = 'ollama/ollama:rocm' + runtimeImage = 'ollama/ollama:rocm' updatedAmdDevices = await this._discoverAMDDevices() updatedAmdGpuConfigured = true } else { @@ -1158,9 +1162,9 @@ export class DockerService { } } - // Step 1: Pull new image - this._broadcast(serviceName, 'update-pulling', `Pulling image ${newImage}...`) - const pullStream = await this.docker.pull(newImage) + // Step 1: Pull new image (runtimeImage diverges from newImage for AMD, see above) + this._broadcast(serviceName, 'update-pulling', `Pulling image ${runtimeImage}...`) + const pullStream = await this.docker.pull(runtimeImage) await new Promise((res) => this.docker.modem.followProgress(pullStream, res)) // Step 2: Find and stop existing container @@ -1205,7 +1209,7 @@ export class DockerService { } const newContainerConfig: any = { - Image: newImage, + Image: runtimeImage, name: serviceName, Env: finalEnv.length > 0 ? finalEnv : undefined, Cmd: inspectData.Config?.Cmd || undefined, From e42f9331b67c0b966000300893b34d83ce555474 Mon Sep 17 00:00:00 2001 From: Ben Gauger Date: Fri, 8 May 2026 13:28:39 -0600 Subject: [PATCH 056/108] fix(Downloads): treat missing Content-Type as octet-stream (#848) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit download.kiwix.org (and some of its mirrors) don't always set a Content-Type header on .zim responses. The MIME validator was reading `headers['content-type'] || ''`, then running each allowlist entry through `''.includes(...)` which is always false, so every download from those hosts was rejected with `MIME type is not allowed`. RFC 7231 §3.1.1.5 says missing Content-Type may be treated as application/octet-stream by the recipient, and that's already in every binary-content allowlist we use (ZIM, PMTILES, base assets). Default the missing case to that and the validator does the right thing. Strict callers that don't list octet-stream still reject as before. --- admin/app/utils/downloads.ts | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/admin/app/utils/downloads.ts b/admin/app/utils/downloads.ts index 5d91d619..23988da0 100644 --- a/admin/app/utils/downloads.ts +++ b/admin/app/utils/downloads.ts @@ -47,7 +47,14 @@ export async function doResumableDownload({ timeout, }) - const contentType = headResponse.headers['content-type'] || '' + // Some upstream hosts (notably download.kiwix.org for .zim files) don't set a + // Content-Type header at all. Per RFC 7231 §3.1.1.5, "if no Content-Type is + // provided" the recipient may treat it as application/octet-stream — which is + // already in every binary-content allowlist we use (ZIM, PMTILES, base assets). + // Without this default, the validator below throws `MIME type is not allowed` + // and breaks all downloads from kiwix's primary host (#848). + const contentType = + headResponse.headers['content-type'] || 'application/octet-stream' const totalBytes = parseInt(headResponse.headers['content-length'] || '0') const supportsRangeRequests = headResponse.headers['accept-ranges'] === 'bytes' From adb132c286f5093e19aa18427e9b6177927259ef Mon Sep 17 00:00:00 2001 From: cosmistack-bot Date: Tue, 12 May 2026 04:10:35 +0000 Subject: [PATCH 057/108] chore(release): 1.32.0-rc.1 [skip ci] --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 88a4d88e..86b418cd 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "project-nomad", - "version": "1.32.0-rc.2", + "version": "1.32.0-rc.1", "description": "\"", "main": "index.js", "scripts": { From b43fea3c2e040d819a1fcc8343c219143e4a3320 Mon Sep 17 00:00:00 2001 From: Jake Turner Date: Tue, 12 May 2026 04:20:59 +0000 Subject: [PATCH 058/108] chore(release): 1.32.0-rc.3 [skip ci] --- package-lock.json | 4 ++-- package.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/package-lock.json b/package-lock.json index 0a83eae0..8584c097 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "project-nomad", - "version": "1.32.0-rc.2", + "version": "1.32.0-rc.3", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "project-nomad", - "version": "1.32.0-rc.2", + "version": "1.32.0-rc.3", "license": "ISC" } } diff --git a/package.json b/package.json index 86b418cd..d2bc5063 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "project-nomad", - "version": "1.32.0-rc.1", + "version": "1.32.0-rc.3", "description": "\"", "main": "index.js", "scripts": { From 5a72560929f0c72d8183c15d3208a6e7ddb87108 Mon Sep 17 00:00:00 2001 From: Chris Sherwood Date: Tue, 12 May 2026 12:47:34 -0700 Subject: [PATCH 059/108] fix(AI): rewrite RAG query on first follow-up (off-by-one in skip-rewrite threshold) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The short-conversation skip in `rewriteQueryWithContext` used `userMessages.length <= 2`, which short-circuits both the very first turn AND the first follow-up. The follow-up is the moment the rewriter matters most — it's where pronouns and shorthand ("the bars", "how long does it last?") need to be resolved against earlier turns before the embedding search runs. With the rewriter skipped, RAG queries against the raw last message, scores nothing above the 0.3 threshold, and no context gets injected for that turn. The visible symptom is the assistant treating the first follow-up in any chat as a brand-new question — e.g. "great - they threw up 2 of the bars it looks like" answered as if it were a recipe-bars question, with no carry-forward of the prior chocolate- poisoning context. Threshold lowered to `< 2`: skip only when there's exactly one user message (nothing to rewrite from). From the first follow-up onward the rewriter runs, as originally intended before commit 96e5027. Validated against `mistral-nemo:12b` on NOMAD3 by hot-patching the compiled controller and replaying the dog-chocolate scenario. Post-patch response correctly threads "3 Hershey's bars" from turn 1 into turn 2's answer; pre-patch (per reporter's screenshot) pivoted to peanut butter bar recipes. Co-Authored-By: Claude Opus 4.7 (1M context) --- admin/app/controllers/ollama_controller.ts | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/admin/app/controllers/ollama_controller.ts b/admin/app/controllers/ollama_controller.ts index 68b6b785..cac39943 100644 --- a/admin/app/controllers/ollama_controller.ts +++ b/admin/app/controllers/ollama_controller.ts @@ -383,10 +383,15 @@ export default class OllamaController { // Get recent conversation history (last 6 messages for 3 turns) const recentMessages = messages.slice(-6) - // Skip rewriting for short conversations. Rewriting adds latency with - // little RAG benefit until there is enough context to matter. + // Skip rewriting on the very first turn — with only one user message + // there is no prior context to fold in, so the rewrite would just echo + // the message back at the cost of an extra LLM round-trip. From the + // first follow-up onward we need the rewrite so the RAG query carries + // entities and topics from earlier turns ("the bars" → "Hershey's bars + // chocolate poisoning dog"); without it, embeddings match nothing and + // the assistant loses the thread. const userMessages = recentMessages.filter(msg => msg.role === 'user') - if (userMessages.length <= 2) { + if (userMessages.length < 2) { return lastUserMessage?.content || null } From 7acc53444dc3a79ebeac7a532c5e1fe26f893786 Mon Sep 17 00:00:00 2001 From: Chris Sherwood Date: Tue, 12 May 2026 18:05:18 -0700 Subject: [PATCH 060/108] fix(RAG): unbreak multi-batch ZIM ingestion (jobId dedupe) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit EmbedFileJob.dispatch() uses a deterministic per-file jobId (sha256(filePath).slice(0,16)) for every batch. The parent batch's handle() calls EmbedFileJob.dispatch({ batchOffset }) before returning, so the parent is still in `active` state and locked when the continuation tries to enqueue. BullMQ silently returns the locked parent instead of creating a new job — and in newer BullMQ versions it does so without throwing, so the existing `catch (error.message.includes('job already exists'))` branch never fires. After the parent completes, its entry stays in the `completed` ZSET (held by `removeOnComplete: { count: 50 }`), continuing to trip jobId dedupe for any subsequent re-dispatch attempts. Result: every NOMAD install since 2026-02-08 (feat: zim content embedding) with a multi-batch ZIM (wikipedia, cooking SE, ifixit, lrnselfreliance, etc.) has only the first 50 articles indexed in qdrant. The RAG feature has been silently degraded for ~3 months — the user sees the file appear in their KB, qdrant accumulates ~50 articles' worth of vectors, and pagination quietly halts. No error surfaces anywhere. Fix: dispatch() skips the deterministic jobId for continuation batches (batchOffset > 0), letting BullMQ auto-generate a unique one so each batch stacks as an independent queue entry. Initial dispatches keep the deterministic jobId so re-triggering an install (UI re-click, sync rescan) remains idempotent. The existing 'job already exists' branch is now gated on !isContinuation, since by construction continuation batches will never hit dedupe. Validated on NOMAD8 (RX 6800 / Threadripper 3960X, rc.3 + this patch): devdocs_en_python (~1,500 chunks across multiple batches) correctly paginates end-to-end. admin.log shows the expected sequence of "Dispatched embedding job for file: X (continuation @ offset N)" followed by "Starting embedding process for: X (batch offset: N)" for each batch. Co-Authored-By: Claude Opus 4.7 (1M context) --- admin/app/jobs/embed_file_job.ts | 54 ++++++++++++++++++++++---------- 1 file changed, 38 insertions(+), 16 deletions(-) diff --git a/admin/app/jobs/embed_file_job.ts b/admin/app/jobs/embed_file_job.ts index eef45a86..c7566fac 100644 --- a/admin/app/jobs/embed_file_job.ts +++ b/admin/app/jobs/embed_file_job.ts @@ -207,36 +207,58 @@ export class EmbedFileJob { static async dispatch(params: EmbedFileJobParams) { const queueService = new QueueService() const queue = queueService.getQueue(this.queue) - const jobId = this.getJobId(params.filePath) + + // Continuation batches (batchOffset > 0) must NOT reuse the deterministic + // per-file jobId. Two BullMQ dedupe paths would otherwise silently swallow them: + // 1) The parent batch's handle() calls dispatch() before returning, so the + // parent job is still `active` and locked — queue.add() with the same + // jobId returns the locked parent rather than enqueueing the new batch. + // 2) After the parent completes, its entry stays in `completed` (held by + // `removeOnComplete: { count: 50 }`), still tripping jobId dedupe. + // Letting BullMQ auto-generate a unique jobId for continuation batches stacks + // them as independent queue entries that each process via handle(). + // Initial dispatches keep the deterministic jobId so re-triggering an install + // (UI re-click, sync rescan, etc.) is still idempotent. + const isContinuation = !!(params.batchOffset && params.batchOffset > 0) + const initialJobId = this.getJobId(params.filePath) + + const jobOptions: Parameters[2] = { + attempts: 30, + backoff: { + type: 'fixed', + delay: 60000, // Check every 60 seconds for service readiness + }, + removeOnComplete: { count: 50 }, // Keep last 50 completed jobs for history + removeOnFail: { count: 20 }, // Keep last 20 failed jobs for debugging + } + if (!isContinuation) { + jobOptions.jobId = initialJobId + } try { - const job = await queue.add(this.key, params, { - jobId, - attempts: 30, - backoff: { - type: 'fixed', - delay: 60000, // Check every 60 seconds for service readiness - }, - removeOnComplete: { count: 50 }, // Keep last 50 completed jobs for history - removeOnFail: { count: 20 } // Keep last 20 failed jobs for debugging - }) + const job = await queue.add(this.key, params, jobOptions) - logger.info(`[EmbedFileJob] Dispatched embedding job for file: ${params.fileName}`) + const continuationLabel = isContinuation + ? ` (continuation @ offset ${params.batchOffset})` + : '' + logger.info( + `[EmbedFileJob] Dispatched embedding job for file: ${params.fileName}${continuationLabel}` + ) return { job, created: true, - jobId, + jobId: job.id ?? initialJobId, message: `File queued for embedding: ${params.fileName}`, } } catch (error) { - if (error.message && error.message.includes('job already exists')) { - const existing = await queue.getJob(jobId) + if (!isContinuation && error.message && error.message.includes('job already exists')) { + const existing = await queue.getJob(initialJobId) logger.info(`[EmbedFileJob] Job already exists for file: ${params.fileName}`) return { job: existing, created: false, - jobId, + jobId: initialJobId, message: `Embedding job already exists for: ${params.fileName}`, } } From 6778fe6dd0ed15a689a234deb940d76f7be60133 Mon Sep 17 00:00:00 2001 From: Jake Turner Date: Wed, 13 May 2026 03:42:34 +0000 Subject: [PATCH 061/108] chore(release): 1.32.0-rc.4 [skip ci] --- package-lock.json | 4 ++-- package.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/package-lock.json b/package-lock.json index 8584c097..f12f0e58 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "project-nomad", - "version": "1.32.0-rc.3", + "version": "1.32.0-rc.4", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "project-nomad", - "version": "1.32.0-rc.3", + "version": "1.32.0-rc.4", "license": "ISC" } } diff --git a/package.json b/package.json index d2bc5063..475a818b 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "project-nomad", - "version": "1.32.0-rc.3", + "version": "1.32.0-rc.4", "description": "\"", "main": "index.js", "scripts": { From e51ead616feb3c7b20c89e6ce38d513bcb41b8ad Mon Sep 17 00:00:00 2001 From: Chris Sherwood Date: Wed, 13 May 2026 08:49:29 -0700 Subject: [PATCH 062/108] fix(queue): singleton QueueService to stop ioredis connection leak Every static call site instantiated a fresh QueueService (24 call sites across 8 files). QueueService.getQueue() opens a BullMQ Queue per call when not cached, and each Queue opens two ioredis connections (one for commands, one blocking). Because every static call constructed a new QueueService, its internal `queues` cache was never shared, every call opened a fresh pair, and none were ever closed. In normal operation this leaked a few connections per API hit. During multi-batch ZIM ingestion after PR #872 (where EmbedFileJob.handle() dispatches the next batch every 50 articles), every batch completion opened two new connections. On NOMAD3 at ~one batch every 4s sustained, that's ~1800 leaked connections/hour. Redis hit its 10,000-maxclient ceiling in ~5 hours and the admin container fell into an EPIPE flood that required a restart to recover. Fix: collapse QueueService to a true process-wide singleton with a private constructor and getInstance() accessor. The existing per-queue Map is now shared across every dispatch / status / cleanup call, so each queue's underlying connections are opened exactly once for the lifetime of the process. close() now clears the map so the singleton can be torn down cleanly if a graceful-shutdown hook is ever wired up. Validated on NOMAD3 (RTX 5060, v1.32.0-rc.4 + this patch hot-applied): under sustained multi-batch wikipedia_en_simple_all_nopic ingestion, connected_clients held flat at 21-22 across a 5-minute window. Pre-fix the same scenario climbed to 10,000+ over hours. --- admin/app/jobs/check_service_updates_job.ts | 4 ++-- admin/app/jobs/check_update_job.ts | 4 ++-- admin/app/jobs/download_model_job.ts | 8 ++++---- admin/app/jobs/embed_file_job.ts | 10 +++++----- admin/app/jobs/run_benchmark_job.ts | 4 ++-- admin/app/jobs/run_download_job.ts | 8 ++++---- admin/app/jobs/run_extract_pmtiles_job.ts | 8 ++++---- admin/app/services/queue_service.ts | 17 +++++++++++++++++ admin/app/services/zim_service.ts | 2 +- 9 files changed, 41 insertions(+), 24 deletions(-) diff --git a/admin/app/jobs/check_service_updates_job.ts b/admin/app/jobs/check_service_updates_job.ts index 6fb7335d..58be73c7 100644 --- a/admin/app/jobs/check_service_updates_job.ts +++ b/admin/app/jobs/check_service_updates_job.ts @@ -95,7 +95,7 @@ export class CheckServiceUpdatesJob { } static async scheduleNightly() { - const queueService = new QueueService() + const queueService = QueueService.getInstance() const queue = queueService.getQueue(this.queue) await queue.upsertJobScheduler( @@ -114,7 +114,7 @@ export class CheckServiceUpdatesJob { } static async dispatch() { - const queueService = new QueueService() + const queueService = QueueService.getInstance() const queue = queueService.getQueue(this.queue) const job = await queue.add( diff --git a/admin/app/jobs/check_update_job.ts b/admin/app/jobs/check_update_job.ts index 046d9c02..aaac08d3 100644 --- a/admin/app/jobs/check_update_job.ts +++ b/admin/app/jobs/check_update_job.ts @@ -42,7 +42,7 @@ export class CheckUpdateJob { } static async scheduleNightly() { - const queueService = new QueueService() + const queueService = QueueService.getInstance() const queue = queueService.getQueue(this.queue) await queue.upsertJobScheduler( @@ -61,7 +61,7 @@ export class CheckUpdateJob { } static async dispatch() { - const queueService = new QueueService() + const queueService = QueueService.getInstance() const queue = queueService.getQueue(this.queue) const job = await queue.add(this.key, {}, { diff --git a/admin/app/jobs/download_model_job.ts b/admin/app/jobs/download_model_job.ts index f1890215..2ba0080c 100644 --- a/admin/app/jobs/download_model_job.ts +++ b/admin/app/jobs/download_model_job.ts @@ -34,7 +34,7 @@ export class DownloadModelJob { /** Signal cancellation via Redis so the worker process can pick it up on its next poll tick */ static async signalCancel(jobId: string): Promise { - const queueService = new QueueService() + const queueService = QueueService.getInstance() const queue = queueService.getQueue(this.queue) const client = await queue.client await client.set(this.cancelKey(jobId), '1', 'EX', 300) // 5 min TTL @@ -66,7 +66,7 @@ export class DownloadModelJob { DownloadModelJob.abortControllers.set(job.id!, abortController) // Get Redis client for checking cancel signals from the API process - const queueService = new QueueService() + const queueService = QueueService.getInstance() const cancelRedis = await queueService.getQueue(DownloadModelJob.queue).client // Track whether cancellation was explicitly requested by the user. Only user-initiated @@ -154,14 +154,14 @@ export class DownloadModelJob { } static async getByModelName(modelName: string): Promise { - const queueService = new QueueService() + const queueService = QueueService.getInstance() const queue = queueService.getQueue(this.queue) const jobId = this.getJobId(modelName) return await queue.getJob(jobId) } static async dispatch(params: DownloadModelJobParams) { - const queueService = new QueueService() + const queueService = QueueService.getInstance() const queue = queueService.getQueue(this.queue) const jobId = this.getJobId(params.modelName) diff --git a/admin/app/jobs/embed_file_job.ts b/admin/app/jobs/embed_file_job.ts index c7566fac..771cedfd 100644 --- a/admin/app/jobs/embed_file_job.ts +++ b/admin/app/jobs/embed_file_job.ts @@ -184,7 +184,7 @@ export class EmbedFileJob { } static async listActiveJobs(): Promise { - const queueService = new QueueService() + const queueService = QueueService.getInstance() const queue = queueService.getQueue(this.queue) const jobs = await queue.getJobs(['waiting', 'active', 'delayed']) @@ -198,14 +198,14 @@ export class EmbedFileJob { } static async getByFilePath(filePath: string): Promise { - const queueService = new QueueService() + const queueService = QueueService.getInstance() const queue = queueService.getQueue(this.queue) const jobId = this.getJobId(filePath) return await queue.getJob(jobId) } static async dispatch(params: EmbedFileJobParams) { - const queueService = new QueueService() + const queueService = QueueService.getInstance() const queue = queueService.getQueue(this.queue) // Continuation batches (batchOffset > 0) must NOT reuse the deterministic @@ -267,7 +267,7 @@ export class EmbedFileJob { } static async listFailedJobs(): Promise { - const queueService = new QueueService() + const queueService = QueueService.getInstance() const queue = queueService.getQueue(this.queue) // Jobs that have failed at least once are in 'delayed' (retrying) or terminal 'failed' state. // We identify them by job.data.status === 'failed' set in the catch block of handle(). @@ -286,7 +286,7 @@ export class EmbedFileJob { } static async cleanupFailedJobs(): Promise<{ cleaned: number; filesDeleted: number }> { - const queueService = new QueueService() + const queueService = QueueService.getInstance() const queue = queueService.getQueue(this.queue) const allJobs = await queue.getJobs(['waiting', 'delayed', 'failed']) const failedJobs = allJobs.filter((job) => (job.data as any).status === 'failed') diff --git a/admin/app/jobs/run_benchmark_job.ts b/admin/app/jobs/run_benchmark_job.ts index 0ae41e86..962e663d 100644 --- a/admin/app/jobs/run_benchmark_job.ts +++ b/admin/app/jobs/run_benchmark_job.ts @@ -53,7 +53,7 @@ export class RunBenchmarkJob { } static async dispatch(params: RunBenchmarkJobParams) { - const queueService = new QueueService() + const queueService = QueueService.getInstance() const queue = queueService.getQueue(this.queue) try { @@ -89,7 +89,7 @@ export class RunBenchmarkJob { } static async getJob(benchmarkId: string): Promise { - const queueService = new QueueService() + const queueService = QueueService.getInstance() const queue = queueService.getQueue(this.queue) return await queue.getJob(benchmarkId) } diff --git a/admin/app/jobs/run_download_job.ts b/admin/app/jobs/run_download_job.ts index 12b35326..5fad2b64 100644 --- a/admin/app/jobs/run_download_job.ts +++ b/admin/app/jobs/run_download_job.ts @@ -31,7 +31,7 @@ export class RunDownloadJob { /** Signal cancellation via Redis so the worker process can pick it up */ static async signalCancel(jobId: string): Promise { - const queueService = new QueueService() + const queueService = QueueService.getInstance() const queue = queueService.getQueue(this.queue) const client = await queue.client await client.set(this.cancelKey(jobId), '1', 'EX', 300) // 5 min TTL @@ -46,7 +46,7 @@ export class RunDownloadJob { RunDownloadJob.abortControllers.set(job.id!, abortController) // Get Redis client for checking cancel signals from the API process - const queueService = new QueueService() + const queueService = QueueService.getInstance() const cancelRedis = await queueService.getQueue(RunDownloadJob.queue).client let lastKnownProgress: Pick = { @@ -199,7 +199,7 @@ export class RunDownloadJob { } static async getByUrl(url: string): Promise { - const queueService = new QueueService() + const queueService = QueueService.getInstance() const queue = queueService.getQueue(this.queue) const jobId = this.getJobId(url) return await queue.getJob(jobId) @@ -229,7 +229,7 @@ export class RunDownloadJob { } static async dispatch(params: RunDownloadJobParams) { - const queueService = new QueueService() + const queueService = QueueService.getInstance() const queue = queueService.getQueue(this.queue) const jobId = this.getJobId(params.url) diff --git a/admin/app/jobs/run_extract_pmtiles_job.ts b/admin/app/jobs/run_extract_pmtiles_job.ts index 73c4eed3..de7049ff 100644 --- a/admin/app/jobs/run_extract_pmtiles_job.ts +++ b/admin/app/jobs/run_extract_pmtiles_job.ts @@ -49,7 +49,7 @@ export class RunExtractPmtilesJob { } static async signalCancel(jobId: string): Promise { - const queueService = new QueueService() + const queueService = QueueService.getInstance() const queue = queueService.getQueue(this.queue) const client = await queue.client await client.set(this.cancelKey(jobId), '1', 'EX', 300) @@ -77,7 +77,7 @@ export class RunExtractPmtilesJob { `maxzoom=${maxzoom ?? 'source-max'} out=${outputFilepath}` ) - const queueService = new QueueService() + const queueService = QueueService.getInstance() const cancelRedis = await queueService.getQueue(RunExtractPmtilesJob.queue).client let userCancelled = false @@ -249,13 +249,13 @@ export class RunExtractPmtilesJob { } static async getById(jobId: string): Promise { - const queueService = new QueueService() + const queueService = QueueService.getInstance() const queue = queueService.getQueue(this.queue) return await queue.getJob(jobId) } static async dispatch(params: RunExtractPmtilesJobParams) { - const queueService = new QueueService() + const queueService = QueueService.getInstance() const queue = queueService.getQueue(this.queue) const jobId = this.getJobId(params.sourceUrl, params.regionFilepath, params.maxzoom) diff --git a/admin/app/services/queue_service.ts b/admin/app/services/queue_service.ts index fa3a0503..bad976b9 100644 --- a/admin/app/services/queue_service.ts +++ b/admin/app/services/queue_service.ts @@ -1,9 +1,25 @@ import { Queue } from 'bullmq' import queueConfig from '#config/queue' +// Process-wide singleton. Each `Queue` opens two ioredis connections (one for +// commands, one blocking). Instantiating a fresh QueueService per dispatch / +// status lookup leaks both, and under sustained job churn (e.g. multi-batch ZIM +// ingestion enqueueing a continuation every few seconds) it saturates Redis's +// maxclients within hours. export class QueueService { private queues: Map = new Map() + private static _instance: QueueService | null = null + + private constructor() {} + + static getInstance(): QueueService { + if (!QueueService._instance) { + QueueService._instance = new QueueService() + } + return QueueService._instance + } + getQueue(name: string): Queue { if (!this.queues.has(name)) { const queue = new Queue(name, { @@ -18,5 +34,6 @@ export class QueueService { for (const queue of this.queues.values()) { await queue.close() } + this.queues.clear() } } diff --git a/admin/app/services/zim_service.ts b/admin/app/services/zim_service.ts index 1cc9e973..538d59fb 100644 --- a/admin/app/services/zim_service.ts +++ b/admin/app/services/zim_service.ts @@ -314,7 +314,7 @@ export class ZimService { if (restart) { // Check if there are any remaining ZIM download jobs before restarting const { QueueService } = await import('./queue_service.js') - const queueService = new QueueService() + const queueService = QueueService.getInstance() const queue = queueService.getQueue('downloads') // Get all active and waiting jobs From ba661a9da1b20de8b2d5f32becf1021d3525142f Mon Sep 17 00:00:00 2001 From: Chris Sherwood Date: Tue, 12 May 2026 20:35:57 -0700 Subject: [PATCH 063/108] fix(RAG): pace continuation batches when embedding is CPU-only MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Stacks on top of the multi-batch ZIM ingestion fix. After that fix, multi-batch ZIM ingestion completes correctly — but on installs where Ollama runs the embedding model on CPU (currently every AMD ROCm install, since Ollama's ROCm build doesn't accelerate nomic-bert), the now-correct sustained 100% CPU saturation across all cores can starve other services hard enough to take the box down. Confirmed on a Threadripper 3960X + RX 6800 NOMAD: a wikipedia-class ZIM ingestion pegged 48 threads cleanly enough that sshd lost banner-exchange responsiveness and the box ultimately required a power-cycle. NVIDIA installs aren't affected — nomic-embed-text:v1.5 runs at 100% GPU on RTX 5060 (verified via `ollama ps`). Detect placement at runtime, pace only when needed: 1. OllamaService.isEmbeddingGpuAccelerated() — queries /api/ps and returns true if any loaded embedding model reports size_vram > 0. Fails closed (returns false) if /api/ps is unreachable or no embed model is loaded yet — over-pacing is safer than crashing. 2. EmbedFileJob.handle() — between batches (hasMoreBatches: true branch), check placement and `await setTimeout(CPU_BATCH_DELAY_MS)` when CPU-only. CPU_BATCH_DELAY_MS = 1000 (1s) — enough to give the OS scheduler a window for sshd/disk-collector/etc., small enough that total ingestion time isn't meaningfully affected (each batch is ~60-90s of work). GPU-accelerated installs see zero behavior change. Co-Authored-By: Claude Opus 4.7 (1M context) --- admin/app/jobs/embed_file_job.ts | 19 +++++++++++++++ admin/app/services/ollama_service.ts | 36 ++++++++++++++++++++++++++++ 2 files changed, 55 insertions(+) diff --git a/admin/app/jobs/embed_file_job.ts b/admin/app/jobs/embed_file_job.ts index 771cedfd..426608a1 100644 --- a/admin/app/jobs/embed_file_job.ts +++ b/admin/app/jobs/embed_file_job.ts @@ -27,6 +27,12 @@ export class EmbedFileJob { return 'embed-file' } + // Delay between continuation batches when embedding runs CPU-only. Gives the OS + // scheduler a brief idle window so sshd / disk-collector / other services don't + // starve during long multi-batch ZIM ingestions. Skipped entirely when the + // embedding model is GPU-offloaded — see OllamaService.isEmbeddingGpuAccelerated(). + static readonly CPU_BATCH_DELAY_MS = 1000 + static getJobId(filePath: string): string { return createHash('sha256').update(filePath).digest('hex').slice(0, 16) } @@ -114,6 +120,19 @@ export class EmbedFileJob { `[EmbedFileJob] Batch complete. Dispatching next batch at offset ${nextOffset}` ) + // Pace continuation batches when embedding is CPU-bound. Sustained 100% CPU + // saturation across all cores during multi-batch ZIM ingestion can starve + // other services (sshd has been seen to lose responsiveness hard enough to + // require a power-cycle). When GPU-accelerated, embeddings stream through + // the GPU and CPUs stay free — no pacing needed. + const isGpuAccelerated = await ollamaService.isEmbeddingGpuAccelerated() + if (!isGpuAccelerated) { + logger.info( + `[EmbedFileJob] Embedding is CPU-only — pacing ${EmbedFileJob.CPU_BATCH_DELAY_MS}ms before dispatching next batch` + ) + await new Promise((resolve) => setTimeout(resolve, EmbedFileJob.CPU_BATCH_DELAY_MS)) + } + // Dispatch next batch (not final yet) await EmbedFileJob.dispatch({ filePath, diff --git a/admin/app/services/ollama_service.ts b/admin/app/services/ollama_service.ts index fe0cb1c8..78fbf69c 100644 --- a/admin/app/services/ollama_service.ts +++ b/admin/app/services/ollama_service.ts @@ -513,6 +513,42 @@ export class OllamaService { } } + /** + * Returns true if Ollama is currently running an embedding model with non-zero VRAM + * (i.e., GPU-offloaded). Returns false if the model is running CPU-only OR if it's + * not currently loaded OR if /api/ps is unreachable. + * + * Used by EmbedFileJob to pace continuation batches when the embedding model is + * CPU-bound — sustained 100% CPU on a multi-batch ZIM ingestion can starve other + * services (sshd, etc.) hard enough to require a power-cycle. AMD ROCm installs + * hit this today because Ollama's ROCm build doesn't accelerate nomic-bert; on + * NVIDIA, nomic-embed-text runs at 100% GPU and pacing is unnecessary. + * + * Only the Ollama-native endpoint is supported — backends that expose + * `/v1/embeddings` (LM Studio, llama.cpp) don't surface placement info. + */ + public async isEmbeddingGpuAccelerated(): Promise { + await this._ensureDependencies() + if (!this.baseUrl) return false + + try { + const response = await axios.get(`${this.baseUrl}/api/ps`, { timeout: 5000 }) + const models: Array<{ name?: string; size_vram?: number }> = response.data?.models ?? [] + // Match any loaded model whose name signals it's an embedding model. + // nomic-embed-text, mxbai-embed-large, snowflake-arctic-embed, etc. all follow this convention. + return models.some( + (m) => m.name?.toLowerCase().includes('embed') && (m.size_vram ?? 0) > 0 + ) + } catch (err: any) { + // /api/ps unreachable (Ollama down, non-native backend, etc.) — fail closed: assume CPU, + // which means we'll pace. Better to over-pace than risk box-killing CPU saturation. + logger.warn( + `[OllamaService] Could not check embedding placement via /api/ps: ${err?.message ?? err}` + ) + return false + } + } + public async getModels(includeEmbeddings = false): Promise { await this._ensureDependencies() if (!this.baseUrl) { From fe51dc49b0bf958918d412aa2e4bd307f71e2aff Mon Sep 17 00:00:00 2001 From: Chris Sherwood Date: Wed, 13 May 2026 10:28:32 -0700 Subject: [PATCH 064/108] feat(GPU): auto-remediate nomad_ollama passthrough loss on admin boot (#755) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit After an update, container recreate, or docker daemon restart, nomad_ollama's HostConfig.DeviceRequests still lists the nvidia driver — but the NVIDIA Container Toolkit binding inside the container is torn. `nvidia-smi` returns "Failed to initialize NVML: Unknown Error" and Ollama silently falls back to CPU inference. PR #208 detects this and shows a banner with a "Fix: Reinstall AI Assistant" button. This change does that click automatically on admin boot. New provider GpuPassthroughRemediationProvider runs once on web env boot: 1. Skip when KV `ai.autoFixGpuPassthrough = false` (default true). 2. Skip when Docker has no `nvidia` runtime registered (AMD-only and CPU-only hosts unaffected). 3. Skip when nomad_ollama isn't running. 4. Exec `nvidia-smi --query-gpu=name --format=csv,noheader` inside the container with an 8-second timeout. If the output matches "Failed to initialize NVML", "Unknown Error", "TIMEOUT", or contains no alphabetic characters, treat the passthrough as broken. 5. On broken: call DockerService.forceReinstall('nomad_ollama'). The existing force-reinstall preserves the Ollama volume + installed models. Stamp `gpu.autoRemediatedAt` on success. 6. On healthy: log and exit. AMD passthrough_failed is intentionally not handled — its fix path is HSA override handling (PR #804) rather than a simple service recreate, and false positives during AMD startup log parsing would loop a recreate without fixing anything. Left to a follow-up if it proves to be a recurring AMD issue. Validated on NOMAD3 (RTX 5060, v1.32.0-rc.3 + this patch hot-applied): - After admin restart with passthrough healthy: log line "[GpuPassthroughRemediationProvider] NVIDIA passthrough healthy — no action needed." Provider exits cleanly without touching the container. - The broken-state branch hits the existing forceReinstall path, which was manually invoked earlier in the same session to fix this exact box and recovered GPU access in ~45s with model volume intact. No new failure mode is introduced — the auto-trigger removes the user click but the underlying operation is the same one the banner Fix button already calls. Closes #755. --- admin/adonisrc.ts | 1 + .../gpu_passthrough_remediation_provider.ts | 122 ++++++++++++++++++ admin/types/kv_store.ts | 2 + 3 files changed, 125 insertions(+) create mode 100644 admin/providers/gpu_passthrough_remediation_provider.ts diff --git a/admin/adonisrc.ts b/admin/adonisrc.ts index 741b1600..9b82ee0f 100644 --- a/admin/adonisrc.ts +++ b/admin/adonisrc.ts @@ -57,6 +57,7 @@ export default defineConfig({ () => import('#providers/kiwix_migration_provider'), () => import('#providers/qdrant_restart_policy_provider'), () => import('#providers/version_check_provider'), + () => import('#providers/gpu_passthrough_remediation_provider'), ], /* diff --git a/admin/providers/gpu_passthrough_remediation_provider.ts b/admin/providers/gpu_passthrough_remediation_provider.ts new file mode 100644 index 00000000..e06aa44f --- /dev/null +++ b/admin/providers/gpu_passthrough_remediation_provider.ts @@ -0,0 +1,122 @@ +import logger from '@adonisjs/core/services/logger' +import type { ApplicationService } from '@adonisjs/core/types' + +/** + * Auto-remediates NVIDIA GPU passthrough loss after admin / host restart. + * + * After an update or container recreate, nomad_ollama's HostConfig.DeviceRequests + * still lists the nvidia driver, but the NVIDIA Container Toolkit binding inside + * the container is torn. `nvidia-smi` inside the container returns + * "Failed to initialize NVML: Unknown Error" and Ollama silently falls back to + * CPU inference. PR #208 added detection + a one-click "Fix: Reinstall AI Assistant" + * banner. This provider does that click automatically on admin boot when the + * condition is detected. + * + * Guards: + * - NVIDIA-only. AMD passthrough_failed has a different fix path (HSA override + * handling in PR #804) and is left to the user. + * - One-shot per admin boot. The provider runs once on startup; if the recreate + * itself fails the banner remains as a fallback. + * - Opt-out via KV `ai.autoFixGpuPassthrough = false`. + * - Skipped entirely when no NVIDIA runtime is registered with Docker. + */ +export default class GpuPassthroughRemediationProvider { + constructor(protected app: ApplicationService) {} + + async boot() { + if (this.app.getEnvironment() !== 'web') return + + setImmediate(async () => { + try { + const KVStore = (await import('#models/kv_store')).default + const { DockerService } = await import('#services/docker_service') + const { SERVICE_NAMES } = await import('../constants/service_names.js') + const Docker = (await import('dockerode')).default + + const enabledRaw = await KVStore.getValue('ai.autoFixGpuPassthrough') + if (String(enabledRaw) === 'false') { + logger.info( + '[GpuPassthroughRemediationProvider] Auto-fix disabled via KV — skipping.' + ) + return + } + + const docker = new Docker({ socketPath: '/var/run/docker.sock' }) + const dockerInfo = await docker.info() + const runtimes = dockerInfo.Runtimes || {} + const hasNvidiaRuntime = 'nvidia' in runtimes + + if (!hasNvidiaRuntime) { + logger.info( + '[GpuPassthroughRemediationProvider] No NVIDIA runtime registered — skipping.' + ) + return + } + + const containers = await docker.listContainers({ all: false }) + const ollama = containers.find((c) => c.Names.includes(`/${SERVICE_NAMES.OLLAMA}`)) + + if (!ollama) { + logger.info( + '[GpuPassthroughRemediationProvider] nomad_ollama not running — skipping.' + ) + return + } + + // Probe: exec nvidia-smi inside the Ollama container. NVML init failure + // is the signature of a broken passthrough that DeviceRequests can't see. + const container = docker.getContainer(ollama.Id) + const exec = await container.exec({ + Cmd: ['nvidia-smi', '--query-gpu=name', '--format=csv,noheader'], + AttachStdout: true, + AttachStderr: true, + }) + const stream = await exec.start({ Tty: true }) + const output = await new Promise((resolve) => { + let buf = '' + const timer = setTimeout(() => resolve(buf || 'TIMEOUT'), 8000) + stream.on('data', (chunk: Buffer) => (buf += chunk.toString('utf8'))) + stream.on('end', () => { + clearTimeout(timer) + resolve(buf) + }) + }) + + const passthroughBroken = + /Failed to initialize NVML|Unknown Error|TIMEOUT/i.test(output) || + !/[A-Za-z]/.test(output) + + if (!passthroughBroken) { + logger.info( + '[GpuPassthroughRemediationProvider] NVIDIA passthrough healthy — no action needed.' + ) + return + } + + logger.warn( + '[GpuPassthroughRemediationProvider] NVIDIA passthrough broken (nvidia-smi inside nomad_ollama failed). ' + + 'Auto-reinstalling nomad_ollama; volumes and installed models are preserved.' + ) + + const dockerService = new DockerService() + const result = await dockerService.forceReinstall(SERVICE_NAMES.OLLAMA) + + if (result.success) { + await KVStore.setValue('gpu.autoRemediatedAt', new Date().toISOString()) + logger.info( + '[GpuPassthroughRemediationProvider] nomad_ollama force-reinstall completed successfully.' + ) + } else { + logger.error( + `[GpuPassthroughRemediationProvider] Force-reinstall failed: ${result.message}. ` + + 'User can still click the "Fix: Reinstall AI Assistant" banner manually.' + ) + } + } catch (err: any) { + logger.error( + `[GpuPassthroughRemediationProvider] Auto-remediation check failed: ${err?.message ?? err}` + ) + } + }) + } +} diff --git a/admin/types/kv_store.ts b/admin/types/kv_store.ts index 7974e952..381b367d 100644 --- a/admin/types/kv_store.ts +++ b/admin/types/kv_store.ts @@ -14,6 +14,8 @@ export const KV_STORE_SCHEMA = { 'ai.ollamaFlashAttention': 'boolean', 'ai.amdGpuAcceleration': 'boolean', 'ai.amdHsaOverride': 'string', + 'ai.autoFixGpuPassthrough': 'boolean', + 'gpu.autoRemediatedAt': 'string', } as const type KVTagToType = T extends 'boolean' ? boolean : string From 63170df6f0a42405351c9e5d308b44b06569e691 Mon Sep 17 00:00:00 2001 From: Jake Turner Date: Wed, 13 May 2026 21:24:33 +0000 Subject: [PATCH 065/108] fix(DockerService): improve volume logic and documentation in forceReinstall --- admin/app/services/docker_service.ts | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/admin/app/services/docker_service.ts b/admin/app/services/docker_service.ts index d0540685..aad3927c 100644 --- a/admin/app/services/docker_service.ts +++ b/admin/app/services/docker_service.ts @@ -291,8 +291,12 @@ export class DockerService { /** * Force reinstall a service by stopping, removing, and recreating its container. - * This method will also clear any associated volumes/data. - * Handles edge cases gracefully (e.g., container not running, container not found). + * + * Volume handling: removes Docker-managed named volumes whose name equals + * `serviceName`, starts with `${serviceName}_`, or carries a `service=${serviceName}` + * label. Host bind mounts are NOT touched — any data living on a bind-mounted + * host path (ZIM stores, model caches, MySQL data dir, etc.) survives the reinstall. + * Anonymous volumes (random hash names) are also not matched. */ async forceReinstall(serviceName: string): Promise<{ success: boolean; message: string }> { try { @@ -365,7 +369,10 @@ export class DockerService { const volumes = await this.docker.listVolumes() const serviceVolumes = volumes.Volumes?.filter( - (v) => v.Name.includes(serviceName) || v.Labels?.service === serviceName + (v) => + v.Name === serviceName || + v.Name.startsWith(`${serviceName}_`) || + v.Labels?.service === serviceName ) || [] for (const vol of serviceVolumes) { From 0390e9584e5d34ba3395a87a9c56ad4b461cee5e Mon Sep 17 00:00:00 2001 From: Chris Sherwood Date: Wed, 13 May 2026 12:23:10 -0700 Subject: [PATCH 066/108] fix(System): correct AMD VRAM in Graphics card + harden log probe MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two related fixes to make the System Information page reliably show real GPU info instead of misleading lspci BAR0 readings or N/A. 1. Generalize bogus-VRAM detection to AMD. Same root cause as #835 (NVIDIA showing 32 MB), this time for AMD: lspci parses the first PCI memory Region (BAR0, typically 1-16 MiB on Navi cards) as `vram`. On NOMAD8 (Threadripper 3960X + Radeon RX 6800), the System Information page showed "1 MB" instead of "16 GB". PR #850 fixed this for NVIDIA by clearing the bogus value and re-running the Ollama log probe; the check was vendor-gated to NVIDIA only. `isBogusNvidiaVram` becomes `isBogusDgpuVram` with a `isDiscreteGpuVendor` helper matching /nvidia|advanced micro devices|amd|ati/i. Same 256-MiB threshold — no real discrete GPU has less than that, while Intel iGPUs (which legitimately report small shared-memory VRAM via lspci) are left untouched. The probe gate condition is similarly renamed. 2. Read Ollama logs from the startup window, not tail:N. `getOllamaInferenceComputeFromLogs()` was reading the last 500 log lines and grepping for the "inference compute" line. That line is written once during Ollama's GPU discovery phase within seconds of startup. Under active embedding workloads we measured >1000 log lines/min, which pushes the line past any reasonable tail within minutes — at which point the probe returns null and the UI flips to "GPU Not Accessible" even though Ollama is happily using the GPU (size_vram > 0 in /api/ps). Switch from `tail: 500` to `since: containerStartedAt, until: containerStartedAt + 300s`. The 5-minute window is bounded regardless of container uptime and always captures Ollama's GPU discovery output. The inference-compute line is emitted in the first few seconds of startup, so 5 min is generous headroom. Validated on NOMAD8 (RX 6800, container uptime ~10 min with sustained ingestion that generated 6,345 log lines): Before: controllers[0]: { model: "Navi 21 ...", vram: 1 } After (bogus AMD VRAM cleared, log probe stale due to tail:500 churn): controllers[0]: { model: "Navi 21 ...", vram: null } gpuHealth: { status: "passthrough_failed" } -> UI shows "N/A" and the banner from PR #208 After (bogus cleared + log probe reads startup window): controllers[0]: { model: "AMD Radeon RX 6800", vram: 16384 } gpuHealth: { status: "ok", hasRocmRuntime: true, ollamaGpuAccessible: true } -> UI shows "16 GB", no banner Both branches of the fix exercise correctly: NVIDIA path unchanged (same code, just renamed identifiers), AMD path now triggers the probe and the probe reliably finds the GPU info regardless of container age. --- admin/app/services/system_service.ts | 53 ++++++++++++++++++---------- 1 file changed, 35 insertions(+), 18 deletions(-) diff --git a/admin/app/services/system_service.ts b/admin/app/services/system_service.ts index 850e85a5..2cd59ba3 100644 --- a/admin/app/services/system_service.ts +++ b/admin/app/services/system_service.ts @@ -95,10 +95,23 @@ export class SystemService { if (!ollamaContainer) return null const container = this.dockerService.docker.getContainer(ollamaContainer.Id) + + // Read logs only from the first 5 minutes after container start. The + // "inference compute" line is written once during Ollama's GPU discovery + // phase, within seconds of startup. Using tail:N here is fragile: under + // active embedding workloads we've seen >1000 lines/min, which pushes the + // line past any reasonable tail in minutes. Pinning to the startup window + // is bounded (~5 min of logs regardless of container uptime) and never + // ages out. + const inspect = await container.inspect() + const startedAtMs = new Date(inspect.State.StartedAt).getTime() + const startedAtSec = Math.floor(startedAtMs / 1000) + const startupWindowSec = startedAtSec + 300 // 5-minute window const buf = (await container.logs({ stdout: true, stderr: true, - tail: 500, + since: startedAtSec, + until: startupWindowSec, follow: false, })) as unknown as Buffer const logs = buf.toString('utf8') @@ -400,36 +413,40 @@ export class SystemService { } // si.graphics() in the admin container uses lspci (pciutils ships in - // the image for AMD detection). lspci has no real VRAM info for NVIDIA - // cards, so systeminformation parses the first PCI memory Region (BAR0, - // 16-32 MiB on most NVIDIA cards) as `vram`. nvidia-smi enrichment also - // can't run since the binary isn't in the admin image. No real dGPU - // has under 256 MiB, so any NVIDIA controller below that needs the - // probes below to give us real data. - const NVIDIA_BOGUS_VRAM_THRESHOLD_MIB = 256 - const isBogusNvidiaVram = (c: { vendor?: string; vram?: number | null }) => - /nvidia/i.test(c.vendor || '') && + // the image for AMD detection). lspci has no real VRAM info for + // discrete GPUs, so systeminformation parses the first PCI memory + // Region (BAR0, typically 1-32 MiB) as `vram`. nvidia-smi / ROCm + // tooling enrichment also can't run since neither is in the admin + // image. No real dGPU has under 256 MiB, so any discrete-GPU controller + // below that threshold needs the probes below to give us real data. + // Applies to both NVIDIA and AMD; Intel iGPUs are exempt because their + // shared-system-memory VRAM reading via lspci can legitimately be small. + const DGPU_BOGUS_VRAM_THRESHOLD_MIB = 256 + const isDiscreteGpuVendor = (vendor: string) => + /nvidia|advanced micro devices|amd|ati/i.test(vendor) + const isBogusDgpuVram = (c: { vendor?: string; vram?: number | null }) => + isDiscreteGpuVendor(c.vendor || '') && typeof c.vram === 'number' && - c.vram < NVIDIA_BOGUS_VRAM_THRESHOLD_MIB + c.vram < DGPU_BOGUS_VRAM_THRESHOLD_MIB // Clear the bogus value up front. If a probe replaces the entry below // we get the real VRAM; if no probe succeeds (Ollama not installed, // passthrough_failed) the UI falls back to "N/A" instead of showing - // "32 MB". The lspci model/vendor strings stay since they're still - // useful for identifying the card. - const hasLspciBogusNvidiaVram = (graphics.controllers || []).some(isBogusNvidiaVram) - if (hasLspciBogusNvidiaVram) { + // "1 MB" / "32 MB". The lspci model/vendor strings stay since they're + // still useful for identifying the card. + const hasLspciBogusDgpuVram = (graphics.controllers || []).some(isBogusDgpuVram) + if (hasLspciBogusDgpuVram) { for (const c of graphics.controllers) { - if (isBogusNvidiaVram(c)) c.vram = null + if (isBogusDgpuVram(c)) c.vram = null } } // Run the probes when controllers are empty (common inside Docker) or - // when lspci gave us bogus NVIDIA BAR0 values that need replacing. + // when lspci gave us bogus discrete-GPU BAR0 values that need replacing. if ( !graphics.controllers || graphics.controllers.length === 0 || - hasLspciBogusNvidiaVram + hasLspciBogusDgpuVram ) { const runtimes = dockerInfo.Runtimes || {} gpuHealth.hasNvidiaRuntime = 'nvidia' in runtimes From 4f82b69572fd6b7d11ce0fe854aa926265c93c29 Mon Sep 17 00:00:00 2001 From: Chris Sherwood Date: Wed, 13 May 2026 15:00:59 -0700 Subject: [PATCH 067/108] fix(System): validate StartedAt with fallback to tail:500 (PR review) Jake noted that `inspect.State.StartedAt` could be missing/malformed, which would land NaN inside `container.logs({ since, until })`. Add defensive validation that the parsed timestamp is finite and positive before using it, with a fallback to the previous tail:500 strategy (plus a warn log) when it isn't. Happy path is unchanged. --- admin/app/services/system_service.ts | 29 +++++++++++++++++++++------- 1 file changed, 22 insertions(+), 7 deletions(-) diff --git a/admin/app/services/system_service.ts b/admin/app/services/system_service.ts index 2cd59ba3..42daac63 100644 --- a/admin/app/services/system_service.ts +++ b/admin/app/services/system_service.ts @@ -103,17 +103,32 @@ export class SystemService { // line past any reasonable tail in minutes. Pinning to the startup window // is bounded (~5 min of logs regardless of container uptime) and never // ages out. + // + // Fall back to the previous tail:500 strategy if StartedAt is missing or + // unparseable — we can't construct a since/until window without it, but + // tail:500 is still useful when the container just started and the line + // is still recent. const inspect = await container.inspect() - const startedAtMs = new Date(inspect.State.StartedAt).getTime() - const startedAtSec = Math.floor(startedAtMs / 1000) - const startupWindowSec = startedAtSec + 300 // 5-minute window - const buf = (await container.logs({ + const startedAtRaw = inspect?.State?.StartedAt + const startedAtMs = startedAtRaw ? new Date(startedAtRaw).getTime() : NaN + const hasValidStartedAt = Number.isFinite(startedAtMs) && startedAtMs > 0 + + const logsOpts: { stdout: true; stderr: true; follow: false; since?: number; until?: number; tail?: number } = { stdout: true, stderr: true, - since: startedAtSec, - until: startupWindowSec, follow: false, - })) as unknown as Buffer + } + if (hasValidStartedAt) { + const startedAtSec = Math.floor(startedAtMs / 1000) + logsOpts.since = startedAtSec + logsOpts.until = startedAtSec + 300 // 5-minute window + } else { + logger.warn( + `[SystemService] nomad_ollama State.StartedAt missing or invalid (${startedAtRaw ?? 'undefined'}); falling back to tail:500 for inference-compute probe` + ) + logsOpts.tail = 500 + } + const buf = (await container.logs(logsOpts)) as unknown as Buffer const logs = buf.toString('utf8') const lines = logs.split('\n').filter((l) => l.includes('msg="inference compute"')) From fe599173ef4e764f8c088625b61e77b917e81004 Mon Sep 17 00:00:00 2001 From: Chris Sherwood Date: Wed, 13 May 2026 12:43:24 -0700 Subject: [PATCH 068/108] fix(RAG): report ZIM ingestion progress in overall-file frame MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Before this change, the Active Downloads / Processing Queue UI showed the ingestion progress gauge jumping wildly during multi-batch ZIM ingestion (e.g. 5% → 88% → 27% → 5% → 56% → 36% over ~60 seconds for cooking SE). Each continuation batch is a separate BullMQ job, and `EmbedFileJob.handle()` reported `job.progress` in two different reference frames depending on where it was in the batch lifecycle: - During-batch (via the onProgress callback): 5% → 95% scaled across "% through this batch's chunks" - End-of-batch (just before dispatching the next): overwritten to `(nextOffset / totalArticles) * 100` — % through the whole file - Next continuation batch starts with progress = 5% explicitly, then climbs through the per-batch range again `listActiveJobs()` returns the latest active BullMQ job's progress. With GPU-accelerated ingestion completing a batch every ~4 seconds, the UI saw the jobId rotate constantly and the gauge whipsaw between the two reference frames. `totalArticles` was already wired through the EmbedFileJob params shape and used end-of-batch — but RagService never actually populated it, so any frame-scaling that depended on it silently fell back to the per-batch range. Two fixes together: 1. `ZIMExtractionService.extractZIMContent()` now returns `{ chunks: ZIMContentChunk[]; totalArticles: number }` instead of a raw chunks array, surfacing `archive.articleCount` to the caller. Single caller (rag_service) updated to destructure. 2. `RagService.processZimFile()` includes `totalArticles` in its result so `EmbedFileJob.dispatch()` can propagate it to the continuation batch (which the existing code already does via `totalArticles: totalArticles || result.totalArticles`). 3. `EmbedFileJob`'s onProgress callback scales the service-reported per-batch percent into the overall-file frame when `totalArticles` is known: `((batchOffset + (percent/100) * ZIM_BATCH_SIZE) / totalArticles) * 100`. Capped at 99% to leave room for the explicit 100% set at file completion. Falls back to the original 5-95% range for single-batch files (uploaded PDFs/txts) where totalArticles is undefined — the gauge then represents % through the only batch, which is what the UI expects for one-shot files. Validated on NOMAD8 (RX 6800, ROCm-accelerated nomic): - devdocs python (small, ~1500 articles): batch progressions seen monotonically across continuation jobIds: 1501@30% → 1510@33% → 1514@43% → 1518@52%. - ifixit (huge, ~100k articles): stays near 3% for the first many batches at offset 0..3000 — correct, the file is enormous. - wikipedia_en_medicine (large, ~70k articles): stays near 0-1% for the first batches — also correct. - Brief 0-5% blip on continuation handoff (the explicit `safeUpdateProgress(job, 5)` at batch start, before the first onProgress callback fires) — visible but quickly resolves to the overall-frame value. No more 5% ↔ 88% chaos. --- admin/app/jobs/embed_file_job.ts | 21 ++++++++++++++++++-- admin/app/services/rag_service.ts | 11 +++++----- admin/app/services/zim_extraction_service.ts | 7 +++++-- 3 files changed, 30 insertions(+), 9 deletions(-) diff --git a/admin/app/jobs/embed_file_job.ts b/admin/app/jobs/embed_file_job.ts index 426608a1..5a3e682b 100644 --- a/admin/app/jobs/embed_file_job.ts +++ b/admin/app/jobs/embed_file_job.ts @@ -7,6 +7,7 @@ import { OllamaService } from '#services/ollama_service' import { createHash } from 'crypto' import logger from '@adonisjs/core/services/logger' import fs from 'node:fs/promises' +import { ZIM_BATCH_SIZE } from '../../constants/zim_extraction.js' export interface EmbedFileJobParams { filePath: string @@ -93,9 +94,25 @@ export class EmbedFileJob { logger.info(`[EmbedFileJob] Processing file: ${filePath}`) - // Progress callback: maps service-reported 0-100% into the 5-95% job range + // Progress callback. For multi-batch ZIM ingestions, scale the service-reported + // 0-100% (which is % through the current batch's chunks) into the overall-file + // frame so the UI gauge climbs monotonically across the many continuation jobs + // BullMQ creates per file. Without this, every new continuation jobId resets the + // gauge to ~5% and the user sees ingestion progress "jumping around" between + // each batch's local frame and the end-of-batch overall-file overwrite below. + // + // For single-batch files (uploaded PDFs, txts) totalArticles is undefined and + // we fall back to the original 5-95% per-job range, which is what the UI expects + // for a one-shot file with no continuations. const onProgress = async (percent: number) => { - await this.safeUpdateProgress(job, Math.min(95, Math.round(5 + percent * 0.9))) + const useOverallFrame = totalArticles && totalArticles > 0 + if (useOverallFrame) { + const articlesDone = (batchOffset || 0) + (percent / 100) * ZIM_BATCH_SIZE + const overallPercent = Math.min(99, Math.round((articlesDone / totalArticles) * 100)) + await this.safeUpdateProgress(job, overallPercent) + } else { + await this.safeUpdateProgress(job, Math.min(95, Math.round(5 + percent * 0.9))) + } } // Process and embed the file diff --git a/admin/app/services/rag_service.ts b/admin/app/services/rag_service.ts index bd5371d1..2a31bd1c 100644 --- a/admin/app/services/rag_service.ts +++ b/admin/app/services/rag_service.ts @@ -500,13 +500,13 @@ export class RagService { `[RAG] Extracting ZIM content (batch: offset=${startOffset}, size=${ZIM_BATCH_SIZE})` ) - const zimChunks = await zimExtractionService.extractZIMContent(filepath, { - startOffset, - batchSize: ZIM_BATCH_SIZE, - }) + const { chunks: zimChunks, totalArticles } = await zimExtractionService.extractZIMContent( + filepath, + { startOffset, batchSize: ZIM_BATCH_SIZE } + ) logger.info( - `[RAG] Extracted ${zimChunks.length} chunks from ZIM file with enhanced metadata` + `[RAG] Extracted ${zimChunks.length} chunks from ZIM file with enhanced metadata (file totalArticles=${totalArticles})` ) // Process each chunk individually with its metadata @@ -582,6 +582,7 @@ export class RagService { chunks: totalChunks, hasMoreBatches, articlesProcessed: articlesInBatch, + totalArticles, } } diff --git a/admin/app/services/zim_extraction_service.ts b/admin/app/services/zim_extraction_service.ts index c48b72d0..196add9e 100644 --- a/admin/app/services/zim_extraction_service.ts +++ b/admin/app/services/zim_extraction_service.ts @@ -40,7 +40,10 @@ export class ZIMExtractionService { * @param filePath - Path to the ZIM file * @param opts - Options including maxArticles, strategy, onProgress, startOffset, and batchSize */ - async extractZIMContent(filePath: string, opts: ExtractZIMContentOptions = {}): Promise { + async extractZIMContent( + filePath: string, + opts: ExtractZIMContentOptions = {} + ): Promise<{ chunks: ZIMContentChunk[]; totalArticles: number }> { try { logger.info(`[ZIMExtractionService]: Processing ZIM file at path: ${filePath}`) @@ -161,7 +164,7 @@ export class ZIMExtractionService { textPreview: c.text.substring(0, 100) }))) logger.debug("Total structured sections extracted:", toReturn.length) - return toReturn + return { chunks: toReturn, totalArticles: archive.articleCount } } catch (error) { logger.error('Error processing ZIM file:', error) throw error From d62176141217c07fef737026577c87798016f20e Mon Sep 17 00:00:00 2001 From: Jake Turner <52841588+jakeaturner@users.noreply.github.com> Date: Fri, 15 May 2026 22:05:19 -0700 Subject: [PATCH 069/108] fix(KB): add re-embed and reset & rebuild opts to fix broken embeddings (#886) --- admin/app/controllers/rag_controller.ts | 20 + admin/app/jobs/embed_file_job.ts | 23 +- admin/app/services/rag_service.ts | 369 ++++++++++++++---- .../components/chat/KnowledgeBaseModal.tsx | 180 ++++++++- admin/inertia/lib/api.ts | 24 ++ admin/start/routes.ts | 2 + 6 files changed, 517 insertions(+), 101 deletions(-) diff --git a/admin/app/controllers/rag_controller.ts b/admin/app/controllers/rag_controller.ts index c836393a..205fb699 100644 --- a/admin/app/controllers/rag_controller.ts +++ b/admin/app/controllers/rag_controller.ts @@ -98,6 +98,26 @@ export default class RagController { } } + public async reembedAll({ response }: HttpContext) { + try { + const result = await this.ragService.reembedAll() + return response.status(200).json(result) + } catch (error) { + logger.error({ err: error }, '[RagController] Error during re-embed all') + return response.status(500).json({ error: 'Error during re-embed all' }) + } + } + + public async resetAndRebuild({ response }: HttpContext) { + try { + const result = await this.ragService.resetAndRebuild() + return response.status(200).json(result) + } catch (error) { + logger.error({ err: error }, '[RagController] Error during reset and rebuild') + return response.status(500).json({ error: 'Error during reset and rebuild' }) + } + } + public async health({ response }: HttpContext) { const result = await this.ragService.checkQdrantHealth() return response.status(200).json(result) diff --git a/admin/app/jobs/embed_file_job.ts b/admin/app/jobs/embed_file_job.ts index 5a3e682b..8da37101 100644 --- a/admin/app/jobs/embed_file_job.ts +++ b/admin/app/jobs/embed_file_job.ts @@ -240,7 +240,7 @@ export class EmbedFileJob { return await queue.getJob(jobId) } - static async dispatch(params: EmbedFileJobParams) { + static async dispatch(params: EmbedFileJobParams, options?: { force?: boolean }) { const queueService = QueueService.getInstance() const queue = queueService.getQueue(this.queue) @@ -255,7 +255,11 @@ export class EmbedFileJob { // them as independent queue entries that each process via handle(). // Initial dispatches keep the deterministic jobId so re-triggering an install // (UI re-click, sync rescan, etc.) is still idempotent. + // `force` skips the deterministic jobId for bulk callers (reembedAll / + // resetAndRebuild) where historical entries in :completed would otherwise + // silently swallow the new dispatch. const isContinuation = !!(params.batchOffset && params.batchOffset > 0) + const force = !!options?.force const initialJobId = this.getJobId(params.filePath) const jobOptions: Parameters[2] = { @@ -267,18 +271,20 @@ export class EmbedFileJob { removeOnComplete: { count: 50 }, // Keep last 50 completed jobs for history removeOnFail: { count: 20 }, // Keep last 20 failed jobs for debugging } - if (!isContinuation) { + if (!isContinuation && !force) { jobOptions.jobId = initialJobId } try { const job = await queue.add(this.key, params, jobOptions) - const continuationLabel = isContinuation + const label = isContinuation ? ` (continuation @ offset ${params.batchOffset})` - : '' + : force + ? ' (forced re-dispatch)' + : '' logger.info( - `[EmbedFileJob] Dispatched embedding job for file: ${params.fileName}${continuationLabel}` + `[EmbedFileJob] Dispatched embedding job for file: ${params.fileName}${label}` ) return { @@ -288,7 +294,12 @@ export class EmbedFileJob { message: `File queued for embedding: ${params.fileName}`, } } catch (error) { - if (!isContinuation && error.message && error.message.includes('job already exists')) { + if ( + !isContinuation && + !force && + error.message && + error.message.includes('job already exists') + ) { const existing = await queue.getJob(initialJobId) logger.info(`[EmbedFileJob] Job already exists for file: ${params.fileName}`) return { diff --git a/admin/app/services/rag_service.ts b/admin/app/services/rag_service.ts index 2a31bd1c..ad0e118a 100644 --- a/admin/app/services/rag_service.ts +++ b/admin/app/services/rag_service.ts @@ -1182,12 +1182,114 @@ export class RagService { } } + /** + * Walk kb_uploads and zim storage directories, returning the full path of + * every embeddable file. Non-embeddable types (e.g. kiwix-library.xml) are + * filtered out so they aren't dispatched only to fail with "Unsupported file + * type" and retry on every sync. + */ + private async _discoverKbFiles(): Promise { + const KB_UPLOADS_PATH = join(process.cwd(), RagService.UPLOADS_STORAGE_PATH) + const ZIM_PATH = join(process.cwd(), ZIM_STORAGE_PATH) + const filesInStorage: string[] = [] + + for (const [label, dirPath] of [ + [RagService.UPLOADS_STORAGE_PATH, KB_UPLOADS_PATH] as const, + [ZIM_STORAGE_PATH, ZIM_PATH] as const, + ]) { + try { + const contents = await listDirectoryContentsRecursive(dirPath) + contents.forEach((entry) => { + if (entry.type === 'file') filesInStorage.push(entry.key) + }) + logger.debug(`[RAG] Found ${contents.length} files in ${label}`) + } catch (error) { + if (error.code === 'ENOENT') { + logger.debug(`[RAG] ${label} directory does not exist, skipping`) + } else { + throw error + } + } + } + + return filesInStorage.filter((f) => determineFileType(f) !== 'unknown') + } + + /** + * Dispatch one EmbedFileJob per file path. Returns honest counts: `queuedCount` + * is jobs newly enqueued, `dedupedCount` is jobs that hit BullMQ's per-file + * jobId dedupe (an existing :completed/:waiting/etc. entry was returned + * instead of a new enqueue), and `failedPaths` lists files whose dispatch + * threw. Pass `force: true` for bulk callers that need to bypass dedupe + * entirely. Per-file errors are logged but don't abort the batch — callers + * must inspect `failedPaths` to surface partial failure to the operator. + */ + private async _dispatchEmbedJobsFor( + filePaths: string[], + options?: { force?: boolean } + ): Promise<{ queuedCount: number; dedupedCount: number; failedPaths: string[] }> { + const { EmbedFileJob } = await import('#jobs/embed_file_job') + let queuedCount = 0 + let dedupedCount = 0 + const failedPaths: string[] = [] + for (const filePath of filePaths) { + try { + const fileName = filePath.split(/[/\\]/).pop() || filePath + const stats = await getFileStatsIfExists(filePath) + const result = await EmbedFileJob.dispatch( + { + filePath, + fileName, + fileSize: stats?.size, + }, + { force: options?.force } + ) + if (result.created) { + queuedCount++ + } else { + dedupedCount++ + } + } catch (fileError) { + failedPaths.push(filePath) + logger.error(`[RAG] Error dispatching job for file ${filePath}:`, fileError) + } + } + return { queuedCount, dedupedCount, failedPaths } + } + + /** + * Delete all Qdrant points whose `source` payload matches the given path. + * Unlike deleteFileBySource(), this does NOT touch the file on disk — used + * by reembedAll() where the file must remain so it can be re-ingested. + */ + private async _deletePointsBySource(source: string): Promise { + await this._ensureCollection( + RagService.CONTENT_COLLECTION_NAME, + RagService.EMBEDDING_DIMENSION + ) + await this.qdrant!.delete(RagService.CONTENT_COLLECTION_NAME, { + filter: { must: [{ key: 'source', match: { value: source } }] }, + }) + } + + /** + * Returns true if the file-embeddings queue has any in-flight work + * (waiting, active, delayed, or paused). Bulk re-embed actions use this + * to refuse mid-flight to avoid racing with deletes/dispatches already + * in progress. + */ + private async _hasInflightEmbedJobs(): Promise { + const { EmbedFileJob } = await import('#jobs/embed_file_job') + const { QueueService } = await import('#services/queue_service') + const queue = QueueService.getInstance().getQueue(EmbedFileJob.queue) + const counts = await queue.getJobCounts('waiting', 'active', 'delayed', 'paused') + return (counts.waiting || 0) + (counts.active || 0) + (counts.delayed || 0) + (counts.paused || 0) > 0 + } + /** * Scans the knowledge base storage directories and syncs with Qdrant. * Identifies files that exist in storage but haven't been embedded yet, * and dispatches EmbedFileJob for each missing file. - * - * @returns Object containing success status, message, and counts of scanned/queued files */ public async scanAndSyncStorage(): Promise<{ success: boolean @@ -1198,91 +1300,38 @@ export class RagService { try { logger.info('[RAG] Starting knowledge base sync scan') - const KB_UPLOADS_PATH = join(process.cwd(), RagService.UPLOADS_STORAGE_PATH) - const ZIM_PATH = join(process.cwd(), ZIM_STORAGE_PATH) - - const filesInStorage: string[] = [] - - // Force resync of Nomad docs await this.discoverNomadDocs(true).catch((error) => { logger.error('[RAG] Error during Nomad docs discovery in sync process:', error) }) - // Scan kb_uploads directory - try { - const kbContents = await listDirectoryContentsRecursive(KB_UPLOADS_PATH) - kbContents.forEach((entry) => { - if (entry.type === 'file') { - filesInStorage.push(entry.key) - } - }) - logger.debug(`[RAG] Found ${kbContents.length} files in ${RagService.UPLOADS_STORAGE_PATH}`) - } catch (error) { - if (error.code === 'ENOENT') { - logger.debug(`[RAG] ${RagService.UPLOADS_STORAGE_PATH} directory does not exist, skipping`) - } else { - throw error - } - } - - // Scan zim directory - try { - const zimContents = await listDirectoryContentsRecursive(ZIM_PATH) - zimContents.forEach((entry) => { - if (entry.type === 'file') { - filesInStorage.push(entry.key) - } - }) - logger.debug(`[RAG] Found ${zimContents.length} files in ${ZIM_STORAGE_PATH}`) - } catch (error) { - if (error.code === 'ENOENT') { - logger.debug(`[RAG] ${ZIM_STORAGE_PATH} directory does not exist, skipping`) - } else { - throw error - } - } - - logger.info(`[RAG] Found ${filesInStorage.length} total files in storage directories`) + const filesInStorage = await this._discoverKbFiles() + logger.info(`[RAG] Found ${filesInStorage.length} embeddable files in storage`) - // Get all stored sources from Qdrant await this._ensureCollection( RagService.CONTENT_COLLECTION_NAME, RagService.EMBEDDING_DIMENSION ) + // Collect every unique `source` already in Qdrant so we can skip files + // that have already been embedded. const sourcesInQdrant = new Set() let offset: string | number | null | Record = null - const batchSize = 100 - - // Scroll through all points to get sources do { const scrollResult = await this.qdrant!.scroll(RagService.CONTENT_COLLECTION_NAME, { - limit: batchSize, - offset: offset, - with_payload: ['source'], // Only fetch source field for efficiency + limit: 100, + offset, + with_payload: ['source'], with_vector: false, }) - scrollResult.points.forEach((point) => { const source = point.payload?.source - if (source && typeof source === 'string') { - sourcesInQdrant.add(source) - } + if (source && typeof source === 'string') sourcesInQdrant.add(source) }) - offset = scrollResult.next_page_offset || null } while (offset !== null) - logger.info(`[RAG] Found ${sourcesInQdrant.size} unique sources in Qdrant`) - - // Find files that are in storage, not already in Qdrant, and have an embeddable type. - // Non-embeddable files (e.g. kiwix-library.xml in /storage/zim) would otherwise be - // dispatched to EmbedFileJob, fail with "Unsupported file type", and retry on every sync. - const filesToEmbed = filesInStorage.filter( - (filePath) => !sourcesInQdrant.has(filePath) && determineFileType(filePath) !== 'unknown' - ) - - logger.info(`[RAG] Found ${filesToEmbed.length} files that need embedding`) + const filesToEmbed = filesInStorage.filter((f) => !sourcesInQdrant.has(f)) + logger.info(`[RAG] ${filesToEmbed.length} of ${filesInStorage.length} files need embedding`) if (filesToEmbed.length === 0) { return { @@ -1293,41 +1342,193 @@ export class RagService { } } - // Import EmbedFileJob dynamically to avoid circular dependencies - const { EmbedFileJob } = await import('#jobs/embed_file_job') + const { queuedCount, dedupedCount } = await this._dispatchEmbedJobsFor(filesToEmbed) + const dedupeNote = dedupedCount > 0 ? ` (${dedupedCount} already queued)` : '' + return { + success: true, + message: `Scanned ${filesInStorage.length} files, queued ${queuedCount} for embedding${dedupeNote}`, + filesScanned: filesInStorage.length, + filesQueued: queuedCount, + } + } catch (error) { + logger.error('[RAG] Error scanning and syncing knowledge base:', error) + return { success: false, message: 'Error scanning and syncing knowledge base' } + } + } + + /** + * Re-embed every file on disk (per-file replace). For each discovered file: + * delete its existing Qdrant points by `source` match, then dispatch a fresh + * EmbedFileJob. Files are NOT removed from disk. Any orphan points (points + * whose source file no longer exists) are intentionally preserved — use + * resetAndRebuild() if a clean slate is required. + * + * Refuses to run if the embeddings queue already has in-flight work. + */ + public async reembedAll(): Promise<{ + success: boolean + message: string + filesScanned?: number + filesQueued?: number + failedPaths?: string[] + }> { + try { + if (await this._hasInflightEmbedJobs()) { + return { + success: false, + message: 'Embed jobs are already in progress. Wait for the queue to drain (or clean up failed jobs) before triggering a bulk re-embed.', + } + } + + logger.info('[RAG] Starting full re-embed (per-file replace)') - // Dispatch jobs for files that need embedding + await this.discoverNomadDocs(true).catch((error) => { + logger.error('[RAG] Error re-running Nomad docs discovery during re-embed:', error) + }) + + const filesInStorage = await this._discoverKbFiles() + + await this._ensureCollection( + RagService.CONTENT_COLLECTION_NAME, + RagService.EMBEDDING_DIMENSION + ) + + // Per-file: delete-then-dispatch. We tried dispatch-then-delete but that + // opens a race where a fast worker can write new points before our + // delete-by-source runs, wiping both. Instead we delete first, then + // dispatch — and if dispatch fails, we surface the failed paths in the + // response so the operator knows which files dropped out (rather than + // silently leaving them unindexed). A subsequent sync rescan picks them + // back up. Note: a delete-failure aborts the per-file pair (we don't + // dispatch a job whose old points are still present, since they'd live + // alongside the new vectors forever). + const { EmbedFileJob } = await import('#jobs/embed_file_job') let queuedCount = 0 - for (const filePath of filesToEmbed) { + const failedPaths: string[] = [] + for (const filePath of filesInStorage) { + try { + await this._deletePointsBySource(filePath) + } catch (err) { + logger.error(`[RAG] Failed to delete prior points for ${filePath}; skipping dispatch:`, err) + failedPaths.push(filePath) + continue + } try { const fileName = filePath.split(/[/\\]/).pop() || filePath const stats = await getFileStatsIfExists(filePath) - - logger.info(`[RAG] Dispatching embed job for: ${fileName}`) - await EmbedFileJob.dispatch({ - filePath: filePath, - fileName: fileName, - fileSize: stats?.size, - }) - queuedCount++ - logger.debug(`[RAG] Successfully dispatched job for ${fileName}`) + const result = await EmbedFileJob.dispatch( + { filePath, fileName, fileSize: stats?.size }, + { force: true } + ) + if (result.created) queuedCount++ } catch (fileError) { - logger.error(`[RAG] Error dispatching job for file ${filePath}:`, fileError) + // Old points already deleted but the new job never made it onto the + // queue. Logged + surfaced so an operator can rerun a sync. + logger.error(`[RAG] Re-embed dispatch failed for ${filePath} after delete; file is now unindexed until next sync:`, fileError) + failedPaths.push(filePath) } } + logger.info( + `[RAG] Re-embed dispatched ${queuedCount}/${filesInStorage.length} files` + + (failedPaths.length > 0 ? ` (${failedPaths.length} failed)` : '') + ) + + const failureSuffix = + failedPaths.length > 0 + ? ` ${failedPaths.length} file${failedPaths.length === 1 ? '' : 's'} failed to dispatch and are temporarily unindexed — run a sync rescan to recover.` + : '' + return { - success: true, - message: `Scanned ${filesInStorage.length} files, queued ${queuedCount} for embedding`, + success: failedPaths.length === 0, + message: + `Re-embedding ${queuedCount} file${queuedCount === 1 ? '' : 's'}. Existing points were replaced.` + + failureSuffix, filesScanned: filesInStorage.length, filesQueued: queuedCount, + ...(failedPaths.length > 0 ? { failedPaths } : {}), } } catch (error) { - logger.error('[RAG] Error scanning and syncing knowledge base:', error) + logger.error('[RAG] Error during re-embed:', error) + return { success: false, message: 'Error during re-embed' } + } + } + + /** + * Destructive rebuild. Drops the entire Qdrant collection (wiping every + * point including orphans), recreates it with the correct dimension, clears + * the Nomad-docs discovery flag, then dispatches an EmbedFileJob for every + * file currently on disk. + * + * Refuses to run if the embeddings queue already has in-flight work. + */ + public async resetAndRebuild(): Promise<{ + success: boolean + message: string + filesScanned?: number + filesQueued?: number + failedPaths?: string[] + }> { + try { + if (await this._hasInflightEmbedJobs()) { + return { + success: false, + message: 'Embed jobs are already in progress. Wait for the queue to drain (or clean up failed jobs) before triggering a reset.', + } + } + + logger.info('[RAG] Starting destructive reset & rebuild') + + await this._initializeQdrantClient() + try { + await this.qdrant!.deleteCollection(RagService.CONTENT_COLLECTION_NAME) + logger.info(`[RAG] Dropped collection ${RagService.CONTENT_COLLECTION_NAME}`) + } catch (err) { + // Collection may not exist yet on a fresh install — log and continue. + logger.warn(`[RAG] deleteCollection failed (may not exist): ${(err as Error).message}`) + } + + await this._ensureCollection( + RagService.CONTENT_COLLECTION_NAME, + RagService.EMBEDDING_DIMENSION + ) + + // Force Nomad docs to be re-dispatched. + await KVStore.setValue('rag.docsEmbedded', false) + await this.discoverNomadDocs(true).catch((error) => { + logger.error('[RAG] Error re-running Nomad docs discovery after reset:', error) + }) + + const filesInStorage = await this._discoverKbFiles() + const { queuedCount, failedPaths } = await this._dispatchEmbedJobsFor(filesInStorage, { + force: true, + }) + + logger.info( + `[RAG] Reset complete — dispatched ${queuedCount}/${filesInStorage.length} files` + + (failedPaths.length > 0 ? ` (${failedPaths.length} failed)` : '') + ) + + // Collection was already dropped, so dispatch failures here mean the + // file is gone from Qdrant with no pending job to repopulate it. Surface + // the count + paths so the operator can rerun a sync rescan to recover. + const failureSuffix = + failedPaths.length > 0 + ? ` ${failedPaths.length} file${failedPaths.length === 1 ? '' : 's'} failed to dispatch and are temporarily unindexed — run a sync rescan to recover.` + : '' + return { - success: false, - message: 'Error scanning and syncing knowledge base', + success: failedPaths.length === 0, + message: + `Collection wiped. Queued ${queuedCount} file${queuedCount === 1 ? '' : 's'} for a full rebuild.` + + failureSuffix, + filesScanned: filesInStorage.length, + filesQueued: queuedCount, + ...(failedPaths.length > 0 ? { failedPaths } : {}), } + } catch (error) { + logger.error('[RAG] Error during reset & rebuild:', error) + return { success: false, message: 'Error during reset & rebuild' } } } } diff --git a/admin/inertia/components/chat/KnowledgeBaseModal.tsx b/admin/inertia/components/chat/KnowledgeBaseModal.tsx index 62303984..d3e39984 100644 --- a/admin/inertia/components/chat/KnowledgeBaseModal.tsx +++ b/admin/inertia/components/chat/KnowledgeBaseModal.tsx @@ -27,6 +27,8 @@ export default function KnowledgeBaseModal({ aiAssistantName = "AI Assistant", o const [files, setFiles] = useState([]) const [isUploading, setIsUploading] = useState(false) const [confirmDeleteSource, setConfirmDeleteSource] = useState(null) + const [bulkMode, setBulkMode] = useState(null) + const [resetTyped, setResetTyped] = useState('') const fileUploaderRef = useRef>(null) const { openModal, closeModal } = useModals() const queryClient = useQueryClient() @@ -105,6 +107,44 @@ export default function KnowledgeBaseModal({ aiAssistantName = "AI Assistant", o }, }) + const reembedMutation = useMutation({ + mutationFn: () => api.reembedAllRAG(), + onSuccess: (data) => { + addNotification({ + type: data?.success ? 'success' : 'error', + message: data?.message || 'Re-embed completed.', + }) + queryClient.invalidateQueries({ queryKey: ['storedFiles'] }) + queryClient.invalidateQueries({ queryKey: ['embed-jobs'] }) + setBulkMode(null) + setResetTyped('') + }, + onError: () => { + addNotification({ type: 'error', message: 'Failed to re-embed knowledge base.' }) + setBulkMode(null) + }, + }) + + const resetMutation = useMutation({ + mutationFn: () => api.resetAndRebuildRAG(), + onSuccess: (data) => { + addNotification({ + type: data?.success ? 'success' : 'error', + message: data?.message || 'Reset complete.', + }) + queryClient.invalidateQueries({ queryKey: ['storedFiles'] }) + queryClient.invalidateQueries({ queryKey: ['embed-jobs'] }) + setBulkMode(null) + setResetTyped('') + }, + onError: () => { + addNotification({ type: 'error', message: 'Failed to reset knowledge base.' }) + setBulkMode(null) + }, + }) + + const bulkBusy = reembedMutation.isPending || resetMutation.isPending + const handleUpload = async () => { if (files.length === 0) return setIsUploading(true) @@ -286,18 +326,41 @@ export default function KnowledgeBaseModal({ aiAssistantName = "AI Assistant", o
-
+
- - Sync Storage - +
+ { setResetTyped(''); setBulkMode('reset') }} + disabled={isUploading || qdrantOffline || bulkBusy} + loading={resetMutation.isPending} + > + Reset & Rebuild + + setBulkMode('reembed')} + disabled={isUploading || qdrantOffline || bulkBusy || storedFiles.length === 0} + loading={reembedMutation.isPending} + > + Re-embed All + + + Sync Storage + + +
className="font-semibold" @@ -360,6 +423,101 @@ export default function KnowledgeBaseModal({ aiAssistantName = "AI Assistant", o
+ + {bulkMode === 'reembed' && ( + reembedMutation.mutate()} + onCancel={() => setBulkMode(null)} + > +
+

+ This will re-process every document currently in your knowledge base — about + {storedFiles.length} file{storedFiles.length === 1 ? '' : 's'}. + For each file, NOMAD will delete the existing embeddings from Qdrant and queue a fresh + embedding job using the current chunking and embedding model. +

+
+

What this is for

+

+ Use this when the embedding model or chunking logic has changed, or when you suspect + stored vectors are stale. Files on disk are not deleted, and any orphan + points whose source file is no longer present will be preserved untouched (see + Reset & Rebuild if you want a fully clean slate). +

+
+
+

Heads up

+
    +
  • Embedding {storedFiles.length} file{storedFiles.length === 1 ? '' : 's'} may take a long time, especially for large PDFs or ZIM archives.
  • +
  • On systems without GPU acceleration, expect sustained high CPU usage for the duration.
  • +
  • Knowledge Base search results may be incomplete until every file finishes re-embedding.
  • +
  • If embed jobs are already in progress, this action will be refused — wait for the queue to drain first.
  • +
+
+
+
+ )} + + {bulkMode === 'reset' && ( + { + if (resetTyped === 'RESET') resetMutation.mutate() + }} + onCancel={() => { setBulkMode(null); setResetTyped('') }} + > +
+

+ This will permanently delete every point in the + nomad_knowledge_base Qdrant collection and rebuild from the + {storedFiles.length} file{storedFiles.length === 1 ? '' : 's'} currently + on disk. The collection is dropped, recreated, and every file is re-queued for embedding. +

+
+

How this differs from Re-embed All

+
    +
  • Re-embed All replaces vectors file-by-file. Any orphan points (vectors whose source file was deleted from disk at some point) are preserved.
  • +
  • Reset & Rebuild drops the entire collection. Orphan points are gone forever. Only files currently on disk will exist in Qdrant afterwards.
  • +
+
+
+

This action is destructive and cannot be undone

+
    +
  • Knowledge Base search will be empty until embedding finishes (potentially hours on CPU-only systems).
  • +
  • For a few seconds during the reset, the Qdrant collection does not exist — any chat-with-RAG queries in that window may return a "collection not found" error. Avoid using chat until the rebuild has begun.
  • +
  • If embed jobs are already in progress, this action will be refused — wait for the queue to drain first.
  • +
+
+
+ + setResetTyped(e.target.value)} + placeholder='RESET' + autoFocus + className='w-full rounded border border-border-subtle bg-surface-primary px-3 py-2 text-text-primary focus:outline-none focus:ring-2 focus:ring-red-500' + /> + {resetTyped.length > 0 && resetTyped !== 'RESET' && ( +

Type RESET exactly (uppercase, no spaces) to enable the confirm button.

+ )} +
+
+
+ )}
) } diff --git a/admin/inertia/lib/api.ts b/admin/inertia/lib/api.ts index aa1369b7..a859cf5d 100644 --- a/admin/inertia/lib/api.ts +++ b/admin/inertia/lib/api.ts @@ -811,6 +811,30 @@ class API { })() } + async reembedAllRAG() { + return catchInternal(async () => { + const response = await this.client.post<{ + success: boolean + message: string + filesScanned?: number + filesQueued?: number + }>('/rag/re-embed-all') + return response.data + })() + } + + async resetAndRebuildRAG() { + return catchInternal(async () => { + const response = await this.client.post<{ + success: boolean + message: string + filesScanned?: number + filesQueued?: number + }>('/rag/reset-and-rebuild') + return response.data + })() + } + // Wikipedia selector methods async getWikipediaState(): Promise { diff --git a/admin/start/routes.ts b/admin/start/routes.ts index fb0d47d2..4d214f3e 100644 --- a/admin/start/routes.ts +++ b/admin/start/routes.ts @@ -147,6 +147,8 @@ router router.delete('/failed-jobs', [RagController, 'cleanupFailedJobs']) router.get('/job-status', [RagController, 'getJobStatus']) router.post('/sync', [RagController, 'scanAndSync']) + router.post('/re-embed-all', [RagController, 'reembedAll']) + router.post('/reset-and-rebuild', [RagController, 'resetAndRebuild']) router.get('/health', [RagController, 'health']) }) .prefix('/api/rag') From 5193f74410f21e98cccaa1dd302a1db0766bca8f Mon Sep 17 00:00:00 2001 From: Chris Sherwood Date: Thu, 14 May 2026 09:53:56 -0700 Subject: [PATCH 070/108] fix(ZIM): preserve co-existing Wikipedia corpora on cleanup (#884) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit onWikipediaDownloadComplete was deleting every file whose name starts with `wikipedia_en_`, treating distinct corpora (simple, medicine, wikivoyage, climate_change, etc.) as competing versions of the same selection slot. Whichever wiki finished second silently wiped the other from disk. Match by filename stem instead — strip the trailing `_YYYY-MM(-DD).zim` date suffix and only delete files with the same stem as the new download. Different release dates of the same variant still get cleaned up; distinct variants are preserved. Extracted the predicate to `app/utils/zim_filename.ts` so the boundary is covered by unit tests (8 cases incl. the #884 repro scenario). --- admin/app/services/zim_service.ts | 16 +++--- admin/app/utils/zim_filename.ts | 26 ++++++++++ admin/tests/unit/zim_filename.spec.ts | 73 +++++++++++++++++++++++++++ 3 files changed, 109 insertions(+), 6 deletions(-) create mode 100644 admin/app/utils/zim_filename.ts create mode 100644 admin/tests/unit/zim_filename.spec.ts diff --git a/admin/app/services/zim_service.ts b/admin/app/services/zim_service.ts index 538d59fb..4c8d4301 100644 --- a/admin/app/services/zim_service.ts +++ b/admin/app/services/zim_service.ts @@ -7,6 +7,7 @@ import axios from 'axios' import * as cheerio from 'cheerio' import { XMLParser } from 'fast-xml-parser' import { isRawListRemoteZimFilesResponse, isRawRemoteZimFileEntry } from '../../util/zim.js' +import { findReplacedWikipediaFiles } from '../utils/zim_filename.js' import logger from '@adonisjs/core/services/logger' import { DockerService } from './docker_service.js' import { inject } from '@adonisjs/core' @@ -627,18 +628,21 @@ export class ZimService { logger.info(`[ZimService] Wikipedia download completed successfully: ${filename}`) - // Delete old Wikipedia files (keep only the newly installed one) + // Delete prior versions of THIS specific Wikipedia variant only. + // Earlier logic deleted anything starting with `wikipedia_en_`, which silently + // wiped distinct corpora the user had installed independently (issue #884). const existingFiles = await this.list() - const wikipediaFiles = existingFiles.files.filter((f) => - f.name.startsWith('wikipedia_en_') && f.name !== filename + const wikipediaFiles = findReplacedWikipediaFiles( + filename, + existingFiles.files.map((f) => f.name) ) for (const oldFile of wikipediaFiles) { try { - await this.delete(oldFile.name) - logger.info(`[ZimService] Deleted old Wikipedia file: ${oldFile.name}`) + await this.delete(oldFile) + logger.info(`[ZimService] Deleted old Wikipedia file: ${oldFile}`) } catch (error) { - logger.warn(`[ZimService] Could not delete old Wikipedia file: ${oldFile.name}`, error) + logger.warn(`[ZimService] Could not delete old Wikipedia file: ${oldFile}`, error) } } } else { diff --git a/admin/app/utils/zim_filename.ts b/admin/app/utils/zim_filename.ts new file mode 100644 index 00000000..a2fb5b2d --- /dev/null +++ b/admin/app/utils/zim_filename.ts @@ -0,0 +1,26 @@ +/** + * Strip the trailing `_YYYY-MM(-DD).zim` date suffix from a Kiwix-style ZIM + * filename so different release dates of the same variant share a stem + * (e.g., `wikipedia_en_all_nopic`) while distinct corpora keep distinct stems + * (`wikipedia_en_simple_all_nopic`, `wikipedia_en_medicine_nopic`, etc.). + */ +export function zimFilenameStem(name: string): string { + return name.replace(/_\d{4}-\d{2}(?:-\d{2})?\.zim$/i, '') +} + +/** + * Of the existing files, return only those that are prior-version replacements + * of `currentFilename` — same Wikipedia variant stem, different release. Used + * by the post-download cleanup to avoid deleting unrelated Wikipedia corpora + * the user has installed independently (issue #884). + */ +export function findReplacedWikipediaFiles( + currentFilename: string, + existingNames: string[] +): string[] { + const currentStem = zimFilenameStem(currentFilename) + return existingNames.filter( + (n) => + n.startsWith('wikipedia_en_') && n !== currentFilename && zimFilenameStem(n) === currentStem + ) +} diff --git a/admin/tests/unit/zim_filename.spec.ts b/admin/tests/unit/zim_filename.spec.ts new file mode 100644 index 00000000..4bc4733d --- /dev/null +++ b/admin/tests/unit/zim_filename.spec.ts @@ -0,0 +1,73 @@ +import * as assert from 'node:assert/strict' +import { test } from 'node:test' + +import { findReplacedWikipediaFiles, zimFilenameStem } from '../../app/utils/zim_filename.js' + +test('zimFilenameStem strips YYYY-MM date suffix', () => { + assert.equal(zimFilenameStem('wikipedia_en_all_nopic_2026-02.zim'), 'wikipedia_en_all_nopic') +}) + +test('zimFilenameStem strips YYYY-MM-DD date suffix', () => { + assert.equal(zimFilenameStem('wikipedia_en_all_nopic_2026-02-15.zim'), 'wikipedia_en_all_nopic') +}) + +test('zimFilenameStem returns input unchanged when no date suffix present', () => { + assert.equal( + zimFilenameStem('wikipedia_en_my_custom_extract.zim'), + 'wikipedia_en_my_custom_extract.zim' + ) +}) + +test('findReplacedWikipediaFiles cleans up older version of same variant', () => { + assert.deepEqual( + findReplacedWikipediaFiles('wikipedia_en_all_nopic_2026-04.zim', [ + 'wikipedia_en_all_nopic_2026-02.zim', + 'wikipedia_en_all_nopic_2026-04.zim', + ]), + ['wikipedia_en_all_nopic_2026-02.zim'] + ) +}) + +test('findReplacedWikipediaFiles preserves co-existing distinct corpora — the #884 regression case', () => { + assert.deepEqual( + findReplacedWikipediaFiles('wikipedia_en_medicine_nopic_2026-04.zim', [ + 'wikipedia_en_simple_all_nopic_2026-02.zim', + 'wikipedia_en_medicine_nopic_2026-04.zim', + ]), + [] + ) +}) + +test('findReplacedWikipediaFiles preserves all unrelated variants when a new variant lands', () => { + assert.deepEqual( + findReplacedWikipediaFiles('wikipedia_en_all_nopic_2026-04.zim', [ + 'wikipedia_en_simple_all_nopic_2026-02.zim', + 'wikipedia_en_medicine_nopic_2026-04.zim', + 'wikipedia_en_wikivoyage_2026-02.zim', + 'wikipedia_en_climate_change_2025-08.zim', + 'wikipedia_en_all_nopic_2026-04.zim', + ]), + [] + ) +}) + +test('findReplacedWikipediaFiles ignores files without wikipedia_en_ prefix', () => { + assert.deepEqual( + findReplacedWikipediaFiles('wikipedia_en_all_nopic_2026-04.zim', [ + 'wiktionary_en_all_2026-02.zim', + 'gutenberg_en_all_2026-01.zim', + 'wikipedia_en_all_nopic_2026-04.zim', + ]), + [] + ) +}) + +test('findReplacedWikipediaFiles preserves manually-named files without a date suffix', () => { + assert.deepEqual( + findReplacedWikipediaFiles('wikipedia_en_all_nopic_2026-04.zim', [ + 'wikipedia_en_my_custom_extract.zim', + 'wikipedia_en_all_nopic_2026-04.zim', + ]), + [] + ) +}) From 69cf66c1f31369871494fb33bfe2242659e99012 Mon Sep 17 00:00:00 2001 From: chriscrosstalk <49691103+chriscrosstalk@users.noreply.github.com> Date: Fri, 15 May 2026 22:51:06 -0700 Subject: [PATCH 071/108] feat(KB): per-file ingest state machine (Phase 1 of RFC #883) (#888) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds a persistent state machine for AI knowledge-base ingestion so the scanner can distinguish "fully indexed", "user opted out", "failed", and "stalled" from each other — none of which were derivable from the prior binary "any chunks in Qdrant ⇒ embedded" check. ## What lands - New table `kb_ingest_state` keyed by `file_path` with enum state column (`pending_decision | indexed | browse_only | failed | stalled`). Independent of `installed_resources` so it covers both curated downloads and manually-uploaded KB files. - New KV key `rag.defaultIngestPolicy` (string: `Always | Manual`). Registered now but not consumed yet — JIT prompt + wizard step land in Phase 3 of the RFC. - `EmbedFileJob.handle` writes state on terminal outcomes: - Success (final batch) → `indexed` + chunks count - `UnrecoverableError` → `failed` + error message - Retryable errors are left to BullMQ's existing retry path - `scanAndSyncStorage` swaps the binary qdrant check for a state-aware decision tree (see `decideScanAction`). Existing installs auto-backfill on first scan: files with chunks in Qdrant but no state row become `indexed`; new files start as `pending_decision`. - `deleteFileBySource` drops the state row last, so removed files disappear entirely instead of leaving an orphan that the next scan would re-dispatch into nothing. ## What does NOT land here - Ratio registry (separate PR) — needed for partial-stall detection and cost estimates, but a separable concern. - #880 follow-up initial-progress anchor (separate tiny PR). - Phase 2 UI (status pill, per-card actions, conditional warnings). - Phase 3 policy surfaces (wizard step, JIT prompt, guardrail modal). - PR #886's bulk-action hookup — `_deletePointsBySource` / Re-embed All / Reset & Rebuild would also want to set state, but #886 isn't merged yet; that wiring goes in a follow-up once #886 lands. ## Target This is forward work for v1.40.0 (RFC #883). Branching off `rc` because that's the current latest base and post-GA Jake will sync rc→dev; a retarget at PR-open time is a fast-forward if requested. ## Tests - 9 new unit tests for `decideScanAction` covering all five states plus the no-row / chunks-present / chunks-missing combinations - Type-check clean - Smoke-tested end-to-end on NOMAD3 via hot-patch: - Backfill: 5 ZIMs + 2 KB uploads with existing chunks in Qdrant all came back `indexed` on first scan - Pending dispatch: a video-only ZIM with no chunks (`lrnselfreliance`) came back `pending_decision` and was correctly re-dispatched (Bull deduped to its historical `:completed` jobId — bgauger's #886 fix drains that) - Delete hook: deleting a KB upload via `DELETE /api/rag/files` removed both the disk file and the state row Co-authored-by: Jake Turner <52841588+jakeaturner@users.noreply.github.com> --- admin/app/jobs/embed_file_job.ts | 30 ++++++++ admin/app/models/kb_ingest_state.ts | 77 +++++++++++++++++++ admin/app/services/rag_service.ts | 67 +++++++++++++++- admin/app/utils/kb_ingest_decision.ts | 53 +++++++++++++ ...6000000001_create_kb_ingest_state_table.ts | 26 +++++++ admin/tests/unit/kb_ingest_decision.spec.ts | 46 +++++++++++ admin/types/kb_ingest_state.ts | 9 +++ admin/types/kv_store.ts | 1 + 8 files changed, 307 insertions(+), 2 deletions(-) create mode 100644 admin/app/models/kb_ingest_state.ts create mode 100644 admin/app/utils/kb_ingest_decision.ts create mode 100644 admin/database/migrations/1776000000001_create_kb_ingest_state_table.ts create mode 100644 admin/tests/unit/kb_ingest_decision.spec.ts create mode 100644 admin/types/kb_ingest_state.ts diff --git a/admin/app/jobs/embed_file_job.ts b/admin/app/jobs/embed_file_job.ts index 8da37101..4607a9a2 100644 --- a/admin/app/jobs/embed_file_job.ts +++ b/admin/app/jobs/embed_file_job.ts @@ -4,6 +4,7 @@ import { EmbedJobWithProgress } from '../../types/rag.js' import { RagService } from '#services/rag_service' import { DockerService } from '#services/docker_service' import { OllamaService } from '#services/ollama_service' +import KbIngestState from '#models/kb_ingest_state' import { createHash } from 'crypto' import logger from '@adonisjs/core/services/logger' import fs from 'node:fs/promises' @@ -193,6 +194,18 @@ export class EmbedFileJob { chunks: totalChunks, }) + // Persist the post-job state so scanAndSyncStorage knows this file is done. + // BullMQ's :completed retention (50 jobs) ages out, so the state row is + // the only durable record of "this file finished embedding". + try { + await KbIngestState.markIndexed(filePath, totalChunks) + } catch (stateErr) { + logger.warn( + `[EmbedFileJob] Failed to persist ingest state for ${fileName}: %s`, + stateErr instanceof Error ? stateErr.message : String(stateErr) + ) + } + const batchMsg = isZimBatch ? ` (final batch, total chunks: ${totalChunks})` : '' logger.info( `[EmbedFileJob] Successfully embedded ${result.chunks} chunks from file: ${fileName}${batchMsg}` @@ -215,6 +228,23 @@ export class EmbedFileJob { error: error instanceof Error ? error.message : 'Unknown error', }) + // Only persist `failed` for unrecoverable errors. Retryable errors get + // automatic BullMQ retries (30 attempts); marking state failed on every + // transient blip would suppress the retry-driven recovery path. + if (error instanceof UnrecoverableError) { + try { + await KbIngestState.markFailed( + filePath, + error instanceof Error ? error.message : 'Unknown error' + ) + } catch (stateErr) { + logger.warn( + `[EmbedFileJob] Failed to persist failed state for ${fileName}: %s`, + stateErr instanceof Error ? stateErr.message : String(stateErr) + ) + } + } + throw error } } diff --git a/admin/app/models/kb_ingest_state.ts b/admin/app/models/kb_ingest_state.ts new file mode 100644 index 00000000..2add4077 --- /dev/null +++ b/admin/app/models/kb_ingest_state.ts @@ -0,0 +1,77 @@ +import { DateTime } from 'luxon' +import { BaseModel, column, SnakeCaseNamingStrategy } from '@adonisjs/lucid/orm' +import type { KbIngestStateValue } from '../../types/kb_ingest_state.js' + +const LAST_ERROR_MAX_LEN = 1024 + +/** + * Tracks the per-file decision and outcome of AI knowledge-base ingestion. + * + * The row exists for any embeddable file the scanner has seen and is independent + * of `installed_resources` (which only covers curated downloads). Replaces the + * earlier "any chunks in qdrant ⇒ embedded" binary check, which conflated + * partially-stalled ingestions with fully-indexed files. See RFC #883. + */ +export default class KbIngestState extends BaseModel { + static table = 'kb_ingest_state' + static namingStrategy = new SnakeCaseNamingStrategy() + + @column({ isPrimary: true }) + declare id: number + + @column() + declare file_path: string + + @column() + declare state: KbIngestStateValue + + @column() + declare chunks_embedded: number + + @column() + declare last_error: string | null + + @column.dateTime({ autoCreate: true }) + declare created_at: DateTime + + @column.dateTime({ autoCreate: true, autoUpdate: true }) + declare updated_at: DateTime + + static async getOrCreate(filePath: string): Promise { + return this.firstOrCreate( + { file_path: filePath }, + { file_path: filePath, state: 'pending_decision', chunks_embedded: 0 } + ) + } + + static async markIndexed(filePath: string, chunksEmbedded: number): Promise { + const row = await this.getOrCreate(filePath) + row.state = 'indexed' + row.chunks_embedded = chunksEmbedded + row.last_error = null + await row.save() + } + + static async markFailed(filePath: string, errorMessage: string): Promise { + const row = await this.getOrCreate(filePath) + row.state = 'failed' + row.last_error = errorMessage.slice(0, LAST_ERROR_MAX_LEN) + await row.save() + } + + static async markBrowseOnly(filePath: string): Promise { + const row = await this.getOrCreate(filePath) + row.state = 'browse_only' + await row.save() + } + + static async markStalled(filePath: string): Promise { + const row = await this.getOrCreate(filePath) + row.state = 'stalled' + await row.save() + } + + static async remove(filePath: string): Promise { + await this.query().where('file_path', filePath).delete() + } +} diff --git a/admin/app/services/rag_service.ts b/admin/app/services/rag_service.ts index ad0e118a..a18b4614 100644 --- a/admin/app/services/rag_service.ts +++ b/admin/app/services/rag_service.ts @@ -16,6 +16,8 @@ import { removeStopwords } from 'stopword' import { randomUUID } from 'node:crypto' import { join, resolve, sep } from 'node:path' import KVStore from '#models/kv_store' +import KbIngestState from '#models/kb_ingest_state' +import { decideScanAction } from '../utils/kb_ingest_decision.js' import { ZIMExtractionService } from './zim_extraction_service.js' import { ZIM_BATCH_SIZE } from '../../constants/zim_extraction.js' import { ProcessAndEmbedFileResponse, ProcessZIMFileResponse, RAGResult, RerankedRAGResult } from '../../types/rag.js' @@ -1118,6 +1120,11 @@ export class RagService { logger.warn(`[RAG] File was removed from knowledge base but doesn't live in Nomad's uploads directory, so it can't be safely removed. Skipping deletion of physical file...`) } + // Drop the ingest state row last so the file disappears entirely. Without + // this, the next scanAndSyncStorage would see `indexed + no chunks` for a + // path that no longer exists in storage and try to re-embed nothing. + await KbIngestState.remove(source) + return { success: true, message: 'File removed from knowledge base.' } } catch (error) { logger.error('[RAG] Error deleting file from knowledge base:', error) @@ -1330,8 +1337,64 @@ export class RagService { offset = scrollResult.next_page_offset || null } while (offset !== null) - const filesToEmbed = filesInStorage.filter((f) => !sourcesInQdrant.has(f)) - logger.info(`[RAG] ${filesToEmbed.length} of ${filesInStorage.length} files need embedding`) + logger.info(`[RAG] Found ${sourcesInQdrant.size} unique sources in Qdrant`) + + // Load all known per-file ingest states. The state row is authoritative + // over the "any chunks in Qdrant" heuristic — it captures user choices + // (browse_only) and terminal outcomes (failed, stalled) that aren't visible + // from Qdrant alone. See RFC #883 for the full state machine. + const stateRows = await KbIngestState.all() + const stateByPath = new Map(stateRows.map((row) => [row.file_path, row])) + + // Non-embeddable files (e.g. kiwix-library.xml in /storage/zim) would otherwise + // be dispatched to EmbedFileJob, fail with "Unsupported file type", and retry + // on every sync — filter them out before state decisions. + const embeddableFiles = filesInStorage.filter( + (filePath) => determineFileType(filePath) !== 'unknown' + ) + + const filesToEmbed: string[] = [] + let backfilled = 0 + let createdRows = 0 + let skipped = 0 + + for (const filePath of embeddableFiles) { + const stateRow = stateByPath.get(filePath) ?? null + const action = decideScanAction(stateRow, sourcesInQdrant.has(filePath)) + + switch (action.kind) { + case 'skip': + skipped++ + break + case 'backfill_indexed': + // Pre-RFC install (or a fresh admin pointed at an existing Qdrant volume): + // chunks already exist with no state row, so trust Qdrant and record + // `indexed` without re-embedding. chunks_embedded is left 0 because + // we don't count points-per-source during the scroll above. + await KbIngestState.create({ + file_path: filePath, + state: 'indexed', + chunks_embedded: 0, + }) + backfilled++ + break + case 'dispatch': + if (action.createStateRow) { + await KbIngestState.create({ + file_path: filePath, + state: 'pending_decision', + chunks_embedded: 0, + }) + createdRows++ + } + filesToEmbed.push(filePath) + break + } + } + + logger.info( + `[RAG] Scan results: ${filesToEmbed.length} to embed, ${backfilled} backfilled, ${createdRows} new pending, ${skipped} skipped` + ) if (filesToEmbed.length === 0) { return { diff --git a/admin/app/utils/kb_ingest_decision.ts b/admin/app/utils/kb_ingest_decision.ts new file mode 100644 index 00000000..f9417c3d --- /dev/null +++ b/admin/app/utils/kb_ingest_decision.ts @@ -0,0 +1,53 @@ +import type { KbIngestStateValue } from '../../types/kb_ingest_state.js' + +/** + * Decision returned by `decideScanAction` describing what scanAndSyncStorage + * should do for one file given its current state row (if any) and whether + * Qdrant already has chunks for it. + * + * - `skip` — file is in a settled state (already indexed, deliberately not + * indexed, or in a manual-recovery state); no auto-dispatch. + * - `dispatch` — file needs to be (re-)embedded; an EmbedFileJob should be + * dispatched. `createStateRow` indicates whether a new state row needs to + * be created before dispatch (i.e. first time the scanner has seen it). + * - `backfill_indexed` — Qdrant has chunks but no state row exists yet + * (pre-RFC install, or new admin instance pointed at an existing Qdrant + * volume). Create a row in `indexed` state without re-embedding. + */ +export type ScanAction = + | { kind: 'skip' } + | { kind: 'dispatch'; createStateRow: boolean } + | { kind: 'backfill_indexed' } + +export interface KbIngestStateRow { + state: KbIngestStateValue +} + +/** + * Decide what scanAndSyncStorage should do for a single embeddable file. + * + * Replaces the earlier `!sourcesInQdrant.has(filePath)` binary check, which + * couldn't tell a fully-indexed file from a stalled mid-batch ingestion, and + * couldn't honor a user's "browse only" choice. The state row is now the + * authoritative answer; Qdrant chunk presence is corroborating evidence. + */ +export function decideScanAction( + stateRow: KbIngestStateRow | null, + hasChunksInQdrant: boolean +): ScanAction { + if (!stateRow) { + if (hasChunksInQdrant) return { kind: 'backfill_indexed' } + return { kind: 'dispatch', createStateRow: true } + } + + switch (stateRow.state) { + case 'indexed': + return hasChunksInQdrant ? { kind: 'skip' } : { kind: 'dispatch', createStateRow: false } + case 'pending_decision': + return { kind: 'dispatch', createStateRow: false } + case 'browse_only': + case 'failed': + case 'stalled': + return { kind: 'skip' } + } +} diff --git a/admin/database/migrations/1776000000001_create_kb_ingest_state_table.ts b/admin/database/migrations/1776000000001_create_kb_ingest_state_table.ts new file mode 100644 index 00000000..18e8ab42 --- /dev/null +++ b/admin/database/migrations/1776000000001_create_kb_ingest_state_table.ts @@ -0,0 +1,26 @@ +import { BaseSchema } from '@adonisjs/lucid/schema' + +export default class extends BaseSchema { + protected tableName = 'kb_ingest_state' + + async up() { + this.schema.createTable(this.tableName, (table) => { + table.increments('id').primary() + // utf8mb4 caps an indexed varchar at 768 chars (3072 byte InnoDB key limit); + // 512 leaves headroom and is plenty for any NOMAD-managed file path. + table.string('file_path', 512).notNullable().unique() + table + .enum('state', ['pending_decision', 'indexed', 'browse_only', 'failed', 'stalled']) + .notNullable() + .defaultTo('pending_decision') + table.integer('chunks_embedded').notNullable().defaultTo(0) + table.text('last_error').nullable() + table.timestamp('created_at').notNullable() + table.timestamp('updated_at').notNullable() + }) + } + + async down() { + this.schema.dropTable(this.tableName) + } +} diff --git a/admin/tests/unit/kb_ingest_decision.spec.ts b/admin/tests/unit/kb_ingest_decision.spec.ts new file mode 100644 index 00000000..1f9336d5 --- /dev/null +++ b/admin/tests/unit/kb_ingest_decision.spec.ts @@ -0,0 +1,46 @@ +import * as assert from 'node:assert/strict' +import { test } from 'node:test' + +import { decideScanAction } from '../../app/utils/kb_ingest_decision.js' + +test('no state row, no chunks → dispatch and create row (new file)', () => { + assert.deepEqual(decideScanAction(null, false), { kind: 'dispatch', createStateRow: true }) +}) + +test('no state row, chunks present → backfill_indexed (pre-RFC install, existing Qdrant volume)', () => { + assert.deepEqual(decideScanAction(null, true), { kind: 'backfill_indexed' }) +}) + +test('indexed + chunks present → skip', () => { + assert.deepEqual(decideScanAction({ state: 'indexed' }, true), { kind: 'skip' }) +}) + +test('indexed + chunks missing → re-dispatch (state stale, Qdrant collection rebuilt or chunks deleted)', () => { + assert.deepEqual(decideScanAction({ state: 'indexed' }, false), { + kind: 'dispatch', + createStateRow: false, + }) +}) + +test('pending_decision → dispatch (preserves current Always behavior until policy is consumed)', () => { + assert.deepEqual(decideScanAction({ state: 'pending_decision' }, false), { + kind: 'dispatch', + createStateRow: false, + }) +}) + +test('browse_only → skip (user opted out of indexing)', () => { + assert.deepEqual(decideScanAction({ state: 'browse_only' }, false), { kind: 'skip' }) +}) + +test('browse_only + chunks present → skip (do not silently re-index after un-index)', () => { + assert.deepEqual(decideScanAction({ state: 'browse_only' }, true), { kind: 'skip' }) +}) + +test('failed → skip (manual retry needed, do not auto-redispatch)', () => { + assert.deepEqual(decideScanAction({ state: 'failed' }, false), { kind: 'skip' }) +}) + +test('stalled → skip (manual retry needed)', () => { + assert.deepEqual(decideScanAction({ state: 'stalled' }, false), { kind: 'skip' }) +}) diff --git a/admin/types/kb_ingest_state.ts b/admin/types/kb_ingest_state.ts new file mode 100644 index 00000000..c59d006f --- /dev/null +++ b/admin/types/kb_ingest_state.ts @@ -0,0 +1,9 @@ +export const KB_INGEST_STATES = [ + 'pending_decision', + 'indexed', + 'browse_only', + 'failed', + 'stalled', +] as const + +export type KbIngestStateValue = (typeof KB_INGEST_STATES)[number] diff --git a/admin/types/kv_store.ts b/admin/types/kv_store.ts index 381b367d..e41f910e 100644 --- a/admin/types/kv_store.ts +++ b/admin/types/kv_store.ts @@ -3,6 +3,7 @@ export const KV_STORE_SCHEMA = { 'chat.suggestionsEnabled': 'boolean', 'chat.lastModel': 'string', 'rag.docsEmbedded': 'boolean', + 'rag.defaultIngestPolicy': 'string', 'system.updateAvailable': 'boolean', 'system.latestVersion': 'string', 'system.earlyAccess': 'boolean', From 68c0a37cab71206d0e8697ef65420e023eee0c6e Mon Sep 17 00:00:00 2001 From: chriscrosstalk <49691103+chriscrosstalk@users.noreply.github.com> Date: Fri, 15 May 2026 23:01:45 -0700 Subject: [PATCH 072/108] fix(RAG): anchor continuation-batch initial progress to overall-file frame (#889) Each continuation batch of a multi-batch ZIM embed runs as a fresh BullMQ job, so handle() ran the hardcoded `safeUpdateProgress(job, 5)` even when the file was already 100k articles into a 600k-article ZIM. The UI gauge briefly dropped to 5% before the per-batch onProgress callback caught up to the true overall percentage, reading as a backward jump every time a new batch started. Compute initialPercent from batchOffset / totalArticles when available, falling back to 5 for single-batch files (uploaded PDFs, txts) where totalArticles isn't set. Capped at 99 to leave headroom for the 100% final-batch marker. Follow-up to PR #880 (which fixed the 0-100% scaling during a batch but still had the initial-frame regression). --- admin/app/jobs/embed_file_job.ts | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/admin/app/jobs/embed_file_job.ts b/admin/app/jobs/embed_file_job.ts index 4607a9a2..844af7d7 100644 --- a/admin/app/jobs/embed_file_job.ts +++ b/admin/app/jobs/embed_file_job.ts @@ -85,8 +85,16 @@ export class EmbedFileJob { logger.info(`[EmbedFileJob] Services ready. Processing file: ${fileName}`) - // Update progress starting - await this.safeUpdateProgress(job, 5) + // Anchor initial progress to where we are in the overall file. For a + // continuation batch midway through a multi-batch ZIM (e.g. offset 100k of + // 600k), the hardcoded 5 used to make the gauge briefly flash 0→5→real, + // which read as a backward jump. Fall back to 5 for single-batch files + // where totalArticles isn't set. + const initialPercent = + totalArticles && totalArticles > 0 + ? Math.min(99, Math.round(((batchOffset || 0) / totalArticles) * 100)) + : 5 + await this.safeUpdateProgress(job, initialPercent) await job.updateData({ ...job.data, status: 'processing', From c9ccd4a2026c9d3e6d1b1cac7f4a04aba9dbad78 Mon Sep 17 00:00:00 2001 From: Chris Sherwood Date: Thu, 14 May 2026 12:11:43 -0700 Subject: [PATCH 073/108] fix(AI): pre-cap embed input + log fallback reason (#881) The OpenAI-compatible /v1/embeddings fallback path can't pass `truncate:true` / `num_ctx:8192` to the model, so any chunk that exceeds the model's loaded context_length (often 2048 for nomic-embed-text:v1.5) returns a 400 BadRequestError and is silently dropped from Qdrant. Two CPU-only ingestion runs on NOMAD1 hit this on dense technical content (medlineplus, arduino.stackexchange) even after PR #763's num_ctx fix on the native path. Pre-cap each input string at 4000 chars before either backend call. That's ~1000-2000 tokens depending on density, comfortably under the model's 2048 default. The chunker in RagService is sized for MAX_SAFE_TOKENS=1600 (3200 chars at its conservative 2 chars/token estimate), so well-formed inputs are never touched; this is purely a runtime safety net for the edge cases that slip through. Also stop swallowing the original error in the catch. The bare `} catch {}` here has masked recurring "input length exceeds context length" failures for months (#369, #670, #881). Capture and warn-log the message so future investigations see why we fell back. Same root cause as #369 and #670 which were closed without an actual fix to the fallback path. --- admin/app/services/ollama_service.ts | 43 ++++++++++++++++++++++++---- 1 file changed, 38 insertions(+), 5 deletions(-) diff --git a/admin/app/services/ollama_service.ts b/admin/app/services/ollama_service.ts index 78fbf69c..d5a4d7e8 100644 --- a/admin/app/services/ollama_service.ts +++ b/admin/app/services/ollama_service.ts @@ -469,6 +469,18 @@ export class OllamaService { } } + /** + * Hard char cap per embed input, applied as a runtime safety net regardless of + * which backend path runs. The chunker in RagService caps at MAX_SAFE_TOKENS=1600 + * (3200 chars at the conservative 2 chars/token estimate), but dense technical + * content has been observed to slip past on multi-batch ZIM ingestion (#881). + * + * 4000 chars ≈ 1000–2000 tokens depending on density, which keeps us comfortably + * under nomic-embed-text:v1.5's default 2048-token context even on the OpenAI-compat + * fallback path (which can't pass `truncate:true`/`num_ctx` to the model). + */ + public static readonly EMBED_MAX_INPUT_CHARS = 4000 + /** * Generate embeddings for the given input strings. * Tries the Ollama native /api/embed endpoint first, falls back to /v1/embeddings. @@ -479,6 +491,16 @@ export class OllamaService { throw new Error('AI service is not initialized.') } + // Runtime safety net (#881). The OpenAI-compat fallback has no equivalent of + // truncate:true, so a chunk that exceeds the model's loaded context_length + // (often 2048 for nomic-embed-text:v1.5) returns 400 and the chunk is silently + // dropped from Qdrant. Pre-capping at the input layer protects both paths. + const safeInput = input.map((s) => + s.length > OllamaService.EMBED_MAX_INPUT_CHARS + ? s.slice(0, OllamaService.EMBED_MAX_INPUT_CHARS) + : s + ) + try { // Prefer Ollama native endpoint (supports batch input natively). // Pass num_ctx explicitly so we don't depend on the embedding model's @@ -491,7 +513,7 @@ export class OllamaService { `${this.baseUrl}/api/embed`, { model, - input, + input: safeInput, truncate: true, options: { num_ctx: 8192 }, }, @@ -503,12 +525,23 @@ export class OllamaService { throw new Error('Invalid /api/embed response — missing embeddings array') } return { embeddings: response.data.embeddings } - } catch { - // Fall back to OpenAI-compatible /v1/embeddings + } catch (err) { + // Capture the original error so we know *why* we fell back. Earlier bare + // catches here masked recurring "input length exceeds context length" + // failures for months (#369, #670, #881) — without this log we have no + // signal that /api/embed is the broken path vs the fallback. + logger.warn( + '[OllamaService] /api/embed failed, falling back to /v1/embeddings: %s', + err instanceof Error ? err.message : String(err) + ) + // Fall back to OpenAI-compatible /v1/embeddings. // Explicitly request float format — some backends (e.g. LM Studio) don't reliably // implement the base64 encoding the OpenAI SDK requests by default. - logger.info('[OllamaService] /api/embed unavailable, falling back to /v1/embeddings') - const results = await this.openai.embeddings.create({ model, input, encoding_format: 'float' }) + const results = await this.openai.embeddings.create({ + model, + input: safeInput, + encoding_format: 'float', + }) return { embeddings: results.data.map((e) => e.embedding as number[]) } } } From 8ce5790ab53c3360502cc8d440183088406fc480 Mon Sep 17 00:00:00 2001 From: Jake Turner Date: Sat, 16 May 2026 06:09:39 +0000 Subject: [PATCH 074/108] fix(AI): add truncation DEBUG log --- admin/app/services/ollama_service.ts | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/admin/app/services/ollama_service.ts b/admin/app/services/ollama_service.ts index d5a4d7e8..075fda41 100644 --- a/admin/app/services/ollama_service.ts +++ b/admin/app/services/ollama_service.ts @@ -500,6 +500,18 @@ export class OllamaService { ? s.slice(0, OllamaService.EMBED_MAX_INPUT_CHARS) : s ) + const truncatedCount = input.reduce( + (n, s) => (s.length > OllamaService.EMBED_MAX_INPUT_CHARS ? n + 1 : n), + 0 + ) + if (truncatedCount > 0) { + logger.debug( + '[OllamaService] embed: pre-capped %d/%d inputs at %d chars', + truncatedCount, + input.length, + OllamaService.EMBED_MAX_INPUT_CHARS + ) + } try { // Prefer Ollama native endpoint (supports batch input natively). From 68e1bd5ff257e0c3b8ad0d3a15fefe93090ef475 Mon Sep 17 00:00:00 2001 From: Chris Sherwood Date: Thu, 14 May 2026 12:15:46 -0700 Subject: [PATCH 075/108] feat(KB): ratio registry for disk + time estimates (Phase 1B of RFC #883) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Foundation for the cost estimates and partial-stall detection that Phase 2 will surface. No consumers yet — this PR just lays the table, the seed rows, and the lookup helper so subsequent UI work has estimates available without a per-ZIM benchmark. ## What lands - New table `kb_ratio_registry` (pattern, chunks_per_mb, sample_count, notes). Migration creates and seeds heuristic defaults from the RFC appendix: devdocs (1100/MB), Wikipedia variants (270/MB), iFixit (50/MB), Stack Exchange Q&A (200/MB), video-only ZIMs (0), plus a catch-all fallback at 100/MB. - `KbRatioRegistry` model with static `lookup()` and `estimateChunks()`. - Pure helper `kb_ratio_lookup.ts` doing longest-prefix-match — a specific entry (`wikipedia_en_simple_`) overrides a broader one (`wikipedia_en_`). 9 unit tests covering the lookup boundary. - `sample_count` starts at 0 (heuristic seed) and is reserved for Phase 4 self-calibration to increment as observed ZIMs update each row. ## Not in scope - Self-calibration on successful ingestion (Phase 4) - UI consumers — Warning B (partial-embed stall) and the storage budget meter / time estimates land in Phase 2. ## Tested - Type-check clean - 9 unit tests pass for `findChunksPerMb` and `estimateChunkCount` - Migration applied on NOMAD3 via hot-patch; 9 seed rows verified in DB --- admin/app/models/kb_ratio_registry.ts | 51 +++++++++++++++ admin/app/utils/kb_ratio_lookup.ts | 44 +++++++++++++ ...00000001_create_kb_ratio_registry_table.ts | 64 +++++++++++++++++++ admin/tests/unit/kb_ratio_lookup.spec.ts | 62 ++++++++++++++++++ 4 files changed, 221 insertions(+) create mode 100644 admin/app/models/kb_ratio_registry.ts create mode 100644 admin/app/utils/kb_ratio_lookup.ts create mode 100644 admin/database/migrations/1776100000001_create_kb_ratio_registry_table.ts create mode 100644 admin/tests/unit/kb_ratio_lookup.spec.ts diff --git a/admin/app/models/kb_ratio_registry.ts b/admin/app/models/kb_ratio_registry.ts new file mode 100644 index 00000000..97cd1f2f --- /dev/null +++ b/admin/app/models/kb_ratio_registry.ts @@ -0,0 +1,51 @@ +import { DateTime } from 'luxon' +import { BaseModel, column, SnakeCaseNamingStrategy } from '@adonisjs/lucid/orm' +import { findChunksPerMb, estimateChunkCount } from '../utils/kb_ratio_lookup.js' + +/** + * Self-calibrating registry of `{filename-prefix → chunks_per_mb}` ratios used + * for disk-footprint and time-to-embed estimates surfaced in the KB panel. + * + * Migration seeds the registry with heuristic defaults from the RFC #883 + * appendix; Phase 4 self-calibration will update rows in place as ZIMs finish + * ingesting and the real ratio becomes known. Lookup is longest-prefix-match + * (see `kb_ratio_lookup.ts`) so a specific entry (`wikipedia_en_simple_`) + * overrides a broader one (`wikipedia_en_`). + */ +export default class KbRatioRegistry extends BaseModel { + static table = 'kb_ratio_registry' + static namingStrategy = new SnakeCaseNamingStrategy() + + @column({ isPrimary: true }) + declare id: number + + @column() + declare pattern: string + + @column() + declare chunks_per_mb: number + + @column() + declare sample_count: number + + @column() + declare notes: string | null + + @column.dateTime({ autoCreate: true }) + declare created_at: DateTime + + @column.dateTime({ autoCreate: true, autoUpdate: true }) + declare updated_at: DateTime + + /** Look up chunks_per_mb for a filename by longest-prefix match. */ + static async lookup(filename: string): Promise { + const rows = await this.all() + return findChunksPerMb(filename, rows) + } + + /** Estimate total chunks for a file of the given size on disk. */ + static async estimateChunks(filename: string, fileSizeBytes: number): Promise { + const rows = await this.all() + return estimateChunkCount(filename, fileSizeBytes, rows) + } +} diff --git a/admin/app/utils/kb_ratio_lookup.ts b/admin/app/utils/kb_ratio_lookup.ts new file mode 100644 index 00000000..19b22b62 --- /dev/null +++ b/admin/app/utils/kb_ratio_lookup.ts @@ -0,0 +1,44 @@ +export interface RatioRow { + pattern: string + chunks_per_mb: number +} + +/** + * Pick the chunks_per_mb estimate for a filename by longest-prefix match. + * + * Patterns are filename prefixes (`devdocs_`, `wikipedia_en_simple_`, ...). + * The longest matching prefix wins, so a specific entry (`wikipedia_en_simple_`) + * overrides the broader fallback (`wikipedia_en_`). An empty-string pattern in + * the registry serves as a catch-all that matches every input. + * + * Returns `null` if no row matches and no empty-string fallback is present — + * caller decides whether to surface "unknown" or use its own default. + */ +export function findChunksPerMb(filename: string, rows: RatioRow[]): number | null { + let best: RatioRow | null = null + for (const row of rows) { + if (!filename.startsWith(row.pattern)) continue + if (best === null || row.pattern.length > best.pattern.length) { + best = row + } + } + return best === null ? null : best.chunks_per_mb +} + +/** + * Estimate the number of embedding chunks a ZIM-style file will produce given + * its size on disk in bytes. Returns `null` when the registry has nothing to + * match against. Caller is responsible for converting the estimate into either + * a disk-footprint estimate (chunks × bytes-per-chunk in Qdrant) or a time + * estimate (chunks ÷ chunks-per-minute-on-this-hardware). + */ +export function estimateChunkCount( + filename: string, + fileSizeBytes: number, + rows: RatioRow[] +): number | null { + const ratio = findChunksPerMb(filename, rows) + if (ratio === null) return null + const megabytes = fileSizeBytes / (1024 * 1024) + return Math.round(ratio * megabytes) +} diff --git a/admin/database/migrations/1776100000001_create_kb_ratio_registry_table.ts b/admin/database/migrations/1776100000001_create_kb_ratio_registry_table.ts new file mode 100644 index 00000000..fb0e38c4 --- /dev/null +++ b/admin/database/migrations/1776100000001_create_kb_ratio_registry_table.ts @@ -0,0 +1,64 @@ +import { BaseSchema } from '@adonisjs/lucid/schema' +import { DateTime } from 'luxon' + +const SEED_ROWS: Array<{ pattern: string; chunks_per_mb: number; notes: string }> = [ + // Dense technical reference — every paragraph carries content + { pattern: 'devdocs_', chunks_per_mb: 1100, notes: 'Heuristic seed: dense API references' }, + // Encyclopedia prose — Simple English & general Wikipedia variants + { + pattern: 'wikipedia_en_simple_', + chunks_per_mb: 270, + notes: 'Heuristic seed: Simple English Wikipedia', + }, + { + pattern: 'wikipedia_en_', + chunks_per_mb: 270, + notes: 'Heuristic seed: general Wikipedia variants', + }, + // Sparse text, image-heavy + { pattern: 'ifixit_', chunks_per_mb: 50, notes: 'Heuristic seed: image-heavy repair guides' }, + // Q&A pages — moderate density, mostly short answers + { + pattern: 'cooking.stackexchange.com_', + chunks_per_mb: 200, + notes: 'Heuristic seed: Stack Exchange Q&A', + }, + // Video-only ZIMs produce zero text chunks. Listing these explicitly keeps + // the cost estimator from spinning up "indexing in progress" UI for content + // that has no embeddable text whatsoever. + { pattern: 'lrnselfreliance_', chunks_per_mb: 0, notes: 'Heuristic seed: video-only ZIM' }, + { pattern: 'ted_', chunks_per_mb: 0, notes: 'Heuristic seed: video-only ZIM' }, + { pattern: 'freedom-of-religion_', chunks_per_mb: 0, notes: 'Heuristic seed: video-only ZIM' }, + // Empty-pattern fallback — every filename startsWith('') is true. The lookup + // picks the longest matching pattern, so this only fires for ZIMs that match + // none of the above (medium prose density). + { pattern: '', chunks_per_mb: 100, notes: 'Heuristic fallback' }, +] + +export default class extends BaseSchema { + protected tableName = 'kb_ratio_registry' + + async up() { + this.schema.createTable(this.tableName, (table) => { + table.increments('id').primary() + table.string('pattern', 255).notNullable().unique() + table.decimal('chunks_per_mb', 10, 2).notNullable() + // 0 = heuristic seed, >0 = number of observed ZIMs that have updated this entry. + // Phase 4 self-calibration increments this on each successful ingestion. + table.integer('sample_count').notNullable().defaultTo(0) + table.text('notes').nullable() + table.timestamp('created_at').notNullable() + table.timestamp('updated_at').notNullable() + }) + + const now = DateTime.utc().toSQL({ includeOffset: false }) as string + const rows = SEED_ROWS.map((row) => ({ ...row, created_at: now, updated_at: now })) + this.defer(async (db) => { + await db.table(this.tableName).multiInsert(rows) + }) + } + + async down() { + this.schema.dropTable(this.tableName) + } +} diff --git a/admin/tests/unit/kb_ratio_lookup.spec.ts b/admin/tests/unit/kb_ratio_lookup.spec.ts new file mode 100644 index 00000000..08c33504 --- /dev/null +++ b/admin/tests/unit/kb_ratio_lookup.spec.ts @@ -0,0 +1,62 @@ +import * as assert from 'node:assert/strict' +import { test } from 'node:test' + +import { estimateChunkCount, findChunksPerMb } from '../../app/utils/kb_ratio_lookup.js' + +const SEEDED_ROWS = [ + { pattern: 'devdocs_', chunks_per_mb: 1100 }, + { pattern: 'wikipedia_en_simple_', chunks_per_mb: 270 }, + { pattern: 'wikipedia_en_', chunks_per_mb: 250 }, + { pattern: 'ifixit_', chunks_per_mb: 50 }, + { pattern: 'lrnselfreliance_', chunks_per_mb: 0 }, + { pattern: '', chunks_per_mb: 100 }, +] + +test('exact prefix match', () => { + assert.equal(findChunksPerMb('devdocs_en_python_2026-02.zim', SEEDED_ROWS), 1100) +}) + +test('longest-prefix wins over broader sibling', () => { + // wikipedia_en_simple_* should pick 270, not the 250 from wikipedia_en_ + assert.equal( + findChunksPerMb('wikipedia_en_simple_all_nopic_2026-02.zim', SEEDED_ROWS), + 270 + ) +}) + +test('broader prefix used when no specific match', () => { + // wikipedia_en_medicine_* is not seeded; falls through to wikipedia_en_ at 250 + assert.equal(findChunksPerMb('wikipedia_en_medicine_nopic_2026-04.zim', SEEDED_ROWS), 250) +}) + +test('empty-string fallback catches unmatched filenames', () => { + assert.equal(findChunksPerMb('something_unknown_2026-02.zim', SEEDED_ROWS), 100) +}) + +test('returns null when no row matches and no fallback is registered', () => { + const rowsWithoutFallback = SEEDED_ROWS.filter((r) => r.pattern !== '') + assert.equal(findChunksPerMb('something_unknown_2026-02.zim', rowsWithoutFallback), null) +}) + +test('zero-ratio entry returns 0, not null (video-only ZIMs)', () => { + assert.equal(findChunksPerMb('lrnselfreliance_en_all_2025-12.zim', SEEDED_ROWS), 0) +}) + +test('estimateChunkCount scales by file size in MB', () => { + // 100 MB * 1100 chunks/MB ≈ 110,000 chunks for devdocs + const bytes = 100 * 1024 * 1024 + assert.equal(estimateChunkCount('devdocs_en_python_2026-02.zim', bytes, SEEDED_ROWS), 110000) +}) + +test('estimateChunkCount returns 0 for video-only ZIM regardless of size', () => { + const bytes = 5 * 1024 * 1024 * 1024 // 5 GB + assert.equal(estimateChunkCount('lrnselfreliance_en_all_2025-12.zim', bytes, SEEDED_ROWS), 0) +}) + +test('estimateChunkCount returns null when no match and no fallback', () => { + const rowsWithoutFallback = SEEDED_ROWS.filter((r) => r.pattern !== '') + assert.equal( + estimateChunkCount('something_unknown_2026-02.zim', 50 * 1024 * 1024, rowsWithoutFallback), + null + ) +}) From 460065ae853303babc0f33f3dfd63f14cddb913d Mon Sep 17 00:00:00 2001 From: Jake Turner Date: Sun, 17 May 2026 03:18:41 +0000 Subject: [PATCH 076/108] fix(KB): align chunks_per_mb column type with TS contract MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Switch kb_ratio_registry.chunks_per_mb from DECIMAL(10,2) to UNSIGNED INTEGER so the value mysql2 returns matches the `number` type declared on the model. DECIMAL columns deserialize as strings by default, which would break `=== 0` checks for video-only ZIMs and silently coerce through arithmetic in Phase 2 consumers. All seeds are whole numbers and the heuristic's real-world variance (~±50%) makes sub-integer precision meaningless. --- .../migrations/1776100000001_create_kb_ratio_registry_table.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/admin/database/migrations/1776100000001_create_kb_ratio_registry_table.ts b/admin/database/migrations/1776100000001_create_kb_ratio_registry_table.ts index fb0e38c4..604e6e8b 100644 --- a/admin/database/migrations/1776100000001_create_kb_ratio_registry_table.ts +++ b/admin/database/migrations/1776100000001_create_kb_ratio_registry_table.ts @@ -42,7 +42,7 @@ export default class extends BaseSchema { this.schema.createTable(this.tableName, (table) => { table.increments('id').primary() table.string('pattern', 255).notNullable().unique() - table.decimal('chunks_per_mb', 10, 2).notNullable() + table.integer('chunks_per_mb').unsigned().notNullable() // 0 = heuristic seed, >0 = number of observed ZIMs that have updated this entry. // Phase 4 self-calibration increments this on each successful ingestion. table.integer('sample_count').notNullable().defaultTo(0) From 7d7459bc141fe3a3253bd8a3b2d7caf2611f81e0 Mon Sep 17 00:00:00 2001 From: Chris Sherwood Date: Thu, 14 May 2026 12:26:58 -0700 Subject: [PATCH 077/108] =?UTF-8?q?feat(KB):=20group=20admin=20docs=20into?= =?UTF-8?q?=20single=20row=20in=20Stored=20Files=20(RFC=20#883=20=C2=A79)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Project NOMAD's bundled docs (`/app/docs/*.md` and `README.md`) each embed as their own KB source — currently rendering as 12+ individual rows that swamp user-uploaded content in the Stored Files table. Collapse them into one informational row: > Project NOMAD documentation · 12 files · Managed by NOMAD The admin-docs row hides the Delete button (those files would be re-embedded on the next sync anyway, so deleting is a footgun). User uploads and ZIMs keep their existing per-row Delete UX. Also adds deterministic sort: ZIMs → user uploads → admin docs → other, alphabetical within each bucket. Pure frontend change — `/api/rag/files` response shape unchanged. Decision logic extracted to `kb_file_grouping.ts` with 9 unit tests covering bucket classification, sort order, count noun pluralization, and empty-input handling. --- .../components/chat/KnowledgeBaseModal.tsx | 30 ++++-- admin/inertia/lib/kb_file_grouping.ts | 100 ++++++++++++++++++ admin/tests/unit/kb_file_grouping.spec.ts | 100 ++++++++++++++++++ 3 files changed, 222 insertions(+), 8 deletions(-) create mode 100644 admin/inertia/lib/kb_file_grouping.ts create mode 100644 admin/tests/unit/kb_file_grouping.spec.ts diff --git a/admin/inertia/components/chat/KnowledgeBaseModal.tsx b/admin/inertia/components/chat/KnowledgeBaseModal.tsx index d3e39984..5ca7cc50 100644 --- a/admin/inertia/components/chat/KnowledgeBaseModal.tsx +++ b/admin/inertia/components/chat/KnowledgeBaseModal.tsx @@ -6,6 +6,10 @@ import StyledSectionHeader from '~/components/StyledSectionHeader' import StyledTable from '~/components/StyledTable' import { useNotifications } from '~/context/NotificationContext' import api from '~/lib/api' +import { + groupAndSortKbFiles, + type KbFileGroup, +} from '~/lib/kb_file_grouping' import { IconX } from '@tabler/icons-react' import { useModals } from '~/context/ModalContext' import StyledModal from '../StyledModal' @@ -17,11 +21,6 @@ interface KnowledgeBaseModalProps { onClose: () => void } -function sourceToDisplayName(source: string): string { - const parts = source.split(/[/\\]/) - return parts[parts.length - 1] -} - export default function KnowledgeBaseModal({ aiAssistantName = "AI Assistant", onClose }: KnowledgeBaseModalProps) { const { addNotification } = useNotifications() const [files, setFiles] = useState([]) @@ -362,7 +361,7 @@ export default function KnowledgeBaseModal({ aiAssistantName = "AI Assistant", o
- + className="font-semibold" rowLines={true} columns={[ @@ -370,13 +369,28 @@ export default function KnowledgeBaseModal({ aiAssistantName = "AI Assistant", o accessor: 'source', title: 'File Name', render(record) { - return {sourceToDisplayName(record.source)} + return ( + {record.displayName} + ) }, }, { accessor: 'source', title: '', render(record) { + // Admin docs are auto-discovered and managed by NOMAD itself — + // deleting one would just be re-embedded on the next sync, so + // we surface them as informational only and hide Delete. + if (record.bucket === 'admin_docs') { + return ( +
+ + Managed by NOMAD + +
+ ) + } + const isConfirming = confirmDeleteSource === record.source const isDeleting = deleteMutation.isPending && confirmDeleteSource === record.source if (isConfirming) { @@ -417,7 +431,7 @@ export default function KnowledgeBaseModal({ aiAssistantName = "AI Assistant", o }, }, ]} - data={storedFiles.map((source) => ({ source }))} + data={groupAndSortKbFiles(storedFiles)} loading={isLoadingFiles} />
diff --git a/admin/inertia/lib/kb_file_grouping.ts b/admin/inertia/lib/kb_file_grouping.ts new file mode 100644 index 00000000..9f474c6d --- /dev/null +++ b/admin/inertia/lib/kb_file_grouping.ts @@ -0,0 +1,100 @@ +/** + * Knowledge-base files come back as a flat list of source paths from + * `/api/rag/files`. The UI groups them so the user sees the categories that + * matter to them — ZIMs, uploaded documents, and a single rolled-up entry for + * Project NOMAD's bundled docs (rather than the 12+ individual markdown files + * those break into). + * + * Bucket assignment is purely by path prefix; matching is done on `/` so the + * server-emitted absolute paths work regardless of which Linux mount the admin + * container uses. + */ +export type KbFileBucket = 'zim' | 'upload' | 'admin_docs' | 'other' + +const ADMIN_DOCS_PREFIXES = ['/app/docs/', '/app/README.md'] +const ZIM_PREFIX = '/app/storage/zim/' +const UPLOADS_PREFIX = '/app/storage/kb_uploads/' + +export function classifyKbFile(source: string): KbFileBucket { + if ( + ADMIN_DOCS_PREFIXES.some((p) => + p.endsWith('/') ? source.startsWith(p) : source === p + ) + ) { + return 'admin_docs' + } + if (source.startsWith(ZIM_PREFIX)) return 'zim' + if (source.startsWith(UPLOADS_PREFIX)) return 'upload' + return 'other' +} + +export function sourceToDisplayName(source: string): string { + const parts = source.split(/[/\\]/) + return parts[parts.length - 1] || source +} + +export interface KbFileGroup { + bucket: KbFileBucket + /** Source path used as the row's stable React key. For collapsed admin docs + * this is a synthetic marker; individual file paths live in `members`. */ + source: string + displayName: string + /** Number of underlying files this row represents (1 for non-collapsed). */ + count: number + /** All member source paths — populated for collapsed groups, empty otherwise. */ + members: string[] +} + +const BUCKET_SORT_ORDER: KbFileBucket[] = ['zim', 'upload', 'admin_docs', 'other'] + +/** + * Group raw source paths into rows for the Stored Files table. + * + * - Admin docs (`/app/docs/*`, README) collapse into a single + * "Project NOMAD documentation · N files" row. + * - ZIMs, uploads, and others stay as individual rows, sorted by bucket then + * alphabetically by filename so related items cluster naturally. + */ +export function groupAndSortKbFiles(sources: string[]): KbFileGroup[] { + const buckets: Record = { + zim: [], + upload: [], + admin_docs: [], + other: [], + } + for (const source of sources) { + buckets[classifyKbFile(source)].push(source) + } + + const groups: KbFileGroup[] = [] + + for (const bucket of BUCKET_SORT_ORDER) { + const members = buckets[bucket] + if (members.length === 0) continue + + if (bucket === 'admin_docs') { + groups.push({ + bucket, + source: '__admin_docs_group__', + displayName: `Project NOMAD documentation · ${members.length} file${members.length === 1 ? '' : 's'}`, + count: members.length, + members, + }) + continue + } + + for (const source of members.sort((a, b) => + sourceToDisplayName(a).localeCompare(sourceToDisplayName(b)) + )) { + groups.push({ + bucket, + source, + displayName: sourceToDisplayName(source), + count: 1, + members: [], + }) + } + } + + return groups +} diff --git a/admin/tests/unit/kb_file_grouping.spec.ts b/admin/tests/unit/kb_file_grouping.spec.ts new file mode 100644 index 00000000..caa1da7a --- /dev/null +++ b/admin/tests/unit/kb_file_grouping.spec.ts @@ -0,0 +1,100 @@ +import * as assert from 'node:assert/strict' +import { test } from 'node:test' + +import { + classifyKbFile, + groupAndSortKbFiles, + sourceToDisplayName, +} from '../../inertia/lib/kb_file_grouping.js' + +test('classifyKbFile distinguishes ZIM, upload, admin_docs, and other', () => { + assert.equal( + classifyKbFile('/app/storage/zim/devdocs_en_python_2026-02.zim'), + 'zim' + ) + assert.equal( + classifyKbFile('/app/storage/kb_uploads/federalist.txt-8cc4ec95aa8f.txt'), + 'upload' + ) + assert.equal(classifyKbFile('/app/docs/release-notes.md'), 'admin_docs') + assert.equal(classifyKbFile('/app/README.md'), 'admin_docs') + assert.equal(classifyKbFile('/unexpected/path/file.txt'), 'other') +}) + +test('classifyKbFile does not match /app/READMEs that are not the bundled one', () => { + assert.equal(classifyKbFile('/app/README.md.bak'), 'other') +}) + +test('sourceToDisplayName returns the basename', () => { + assert.equal( + sourceToDisplayName('/app/storage/zim/devdocs_en_python_2026-02.zim'), + 'devdocs_en_python_2026-02.zim' + ) + assert.equal(sourceToDisplayName('/app/docs/release-notes.md'), 'release-notes.md') +}) + +test('groupAndSortKbFiles collapses all admin docs into a single row', () => { + const groups = groupAndSortKbFiles([ + '/app/docs/release-notes.md', + '/app/docs/getting-started.md', + '/app/docs/maps.md', + '/app/README.md', + ]) + + assert.equal(groups.length, 1) + assert.equal(groups[0].bucket, 'admin_docs') + assert.equal(groups[0].count, 4) + assert.equal(groups[0].displayName, 'Project NOMAD documentation · 4 files') + assert.deepEqual(groups[0].members.sort(), [ + '/app/README.md', + '/app/docs/getting-started.md', + '/app/docs/maps.md', + '/app/docs/release-notes.md', + ]) +}) + +test('groupAndSortKbFiles orders buckets ZIM → upload → admin_docs → other', () => { + const groups = groupAndSortKbFiles([ + '/app/docs/release-notes.md', + '/unexpected/foo.txt', + '/app/storage/kb_uploads/upload.pdf', + '/app/storage/zim/devdocs.zim', + ]) + + assert.deepEqual( + groups.map((g) => g.bucket), + ['zim', 'upload', 'admin_docs', 'other'] + ) +}) + +test('groupAndSortKbFiles alphabetizes within a bucket', () => { + const groups = groupAndSortKbFiles([ + '/app/storage/zim/wikipedia.zim', + '/app/storage/zim/devdocs.zim', + '/app/storage/zim/ifixit.zim', + ]) + + assert.deepEqual( + groups.map((g) => g.displayName), + ['devdocs.zim', 'ifixit.zim', 'wikipedia.zim'] + ) +}) + +test('groupAndSortKbFiles uses singular noun when only one admin doc exists', () => { + const groups = groupAndSortKbFiles(['/app/docs/release-notes.md']) + assert.equal(groups[0].displayName, 'Project NOMAD documentation · 1 file') +}) + +test('groupAndSortKbFiles handles empty input', () => { + assert.deepEqual(groupAndSortKbFiles([]), []) +}) + +test('groupAndSortKbFiles preserves a stable synthetic key for the admin docs group', () => { + const groups = groupAndSortKbFiles([ + '/app/docs/release-notes.md', + '/app/docs/maps.md', + ]) + // The admin-docs row uses a synthetic source key (not a real path) so it + // can be used as a React key without colliding with any real file row. + assert.equal(groups[0].source, '__admin_docs_group__') +}) From ca5569c8eaab254b698ded25f253020a9b30f227 Mon Sep 17 00:00:00 2001 From: Chris Sherwood Date: Thu, 14 May 2026 12:31:13 -0700 Subject: [PATCH 078/108] =?UTF-8?q?feat(KB):=20status=20pill=20+=20last-ac?= =?UTF-8?q?tivity=20timestamp=20on=20Processing=20Queue=20(RFC=20#883=20?= =?UTF-8?q?=C2=A75/=C2=A710)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Each in-flight (or stuck) embedding job gets a colored health pill, relative-activity timestamp, and chunk counter so users can tell at a glance whether ingestion is making progress. ## Health states - **🟢 Active** — last batch < 2 min ago - **🟡 Slow** — last batch 2-5 min ago (CPU-paced multi-batch ingestion lives here naturally; not always a problem) - **🔴 Stalled** — last batch > 5 min ago (likely real problem) - **⚪ Waiting** — queued, no batch started yet - **🔴 Failed** — job recorded failed status ## What lands - New backend util `kb_job_health.ts` with pure `computeJobHealth(input)` decision function. Time-based thresholds (2 min / 5 min) inlined as constants. 9 unit tests pin the boundaries. - `EmbedJobWithProgress` gains `lastBatchAt`, `startedAt`, `chunks` — already set by `EmbedFileJob.handle` on every batch transition, just not previously surfaced through `listActiveJobs`. - Frontend `kb_job_health_display.ts` maps each status to a Tailwind dot color, label, and aria-label so backend and UI stay in sync. - `ActiveEmbedJobs.tsx` renders the pill, "last activity Xs ago", and chunk counter above each progress bar. Adds a manual Refresh button and "Last updated Xs ago" line — the existing 2s/30s auto-poll cadence in `useEmbedJobs` is left intact. - Live tick at 5s keeps the relative timestamps current without re-fetching from the API. ## Not in scope - Per-card Cancel / Retry / Un-index — separate Phase 2 PR - Conditional warnings A/B/C — separate Phase 2 PR - Computing throughput rate (chunks/min) — needs ratio registry consumer (Phase 2 follow-up); for now the pill answers the "is it stuck?" question directly without a rate estimate. --- admin/app/jobs/embed_file_job.ts | 25 +++-- admin/app/utils/kb_job_health.ts | 50 ++++++++++ admin/inertia/components/ActiveEmbedJobs.tsx | 100 +++++++++++++++---- admin/inertia/lib/kb_job_health_display.ts | 63 ++++++++++++ admin/tests/unit/kb_job_health.spec.ts | 100 +++++++++++++++++++ admin/types/rag.ts | 6 ++ 6 files changed, 318 insertions(+), 26 deletions(-) create mode 100644 admin/app/utils/kb_job_health.ts create mode 100644 admin/inertia/lib/kb_job_health_display.ts create mode 100644 admin/tests/unit/kb_job_health.spec.ts diff --git a/admin/app/jobs/embed_file_job.ts b/admin/app/jobs/embed_file_job.ts index 844af7d7..90ce6775 100644 --- a/admin/app/jobs/embed_file_job.ts +++ b/admin/app/jobs/embed_file_job.ts @@ -262,13 +262,24 @@ export class EmbedFileJob { const queue = queueService.getQueue(this.queue) const jobs = await queue.getJobs(['waiting', 'active', 'delayed']) - return jobs.map((job) => ({ - jobId: job.id!.toString(), - fileName: (job.data as EmbedFileJobParams).fileName, - filePath: (job.data as EmbedFileJobParams).filePath, - progress: typeof job.progress === 'number' ? job.progress : 0, - status: ((job.data as any).status as string) ?? 'waiting', - })) + return jobs.map((job) => { + const data = job.data as EmbedFileJobParams & { + status?: string + lastBatchAt?: number + startedAt?: number + chunks?: number + } + return { + jobId: job.id!.toString(), + fileName: data.fileName, + filePath: data.filePath, + progress: typeof job.progress === 'number' ? job.progress : 0, + status: data.status ?? 'waiting', + lastBatchAt: data.lastBatchAt, + startedAt: data.startedAt, + chunks: data.chunks, + } + }) } static async getByFilePath(filePath: string): Promise { diff --git a/admin/app/utils/kb_job_health.ts b/admin/app/utils/kb_job_health.ts new file mode 100644 index 00000000..d5e154f2 --- /dev/null +++ b/admin/app/utils/kb_job_health.ts @@ -0,0 +1,50 @@ +/** + * Visual status assigned to an in-flight (or stuck) embedding job, used to + * pick the colored status pill in the KB Processing Queue. See RFC #883 §5. + * + * - `waiting` — queued, no batch has started yet + * - `healthy` — last batch < 2 minutes ago + * - `slow` — last batch 2-5 minutes ago (CPU-paced multi-batch ingestion + * falls into this band; not necessarily a problem) + * - `stalled` — last batch > 5 minutes ago (likely a real problem) + * - `failed` — job recorded a failed status + */ +export type JobHealthStatus = 'waiting' | 'healthy' | 'slow' | 'stalled' | 'failed' + +export interface JobHealthInput { + /** BullMQ job.data.status — set by EmbedFileJob.handle on transitions. */ + status: string + /** 0-100. 0 means no work observed yet on this job-row. */ + progress: number + /** ms epoch of the last completed batch. Multi-batch ZIMs update this on + * every continuation; single-batch jobs leave it unset until completion. */ + lastBatchAt?: number + /** ms epoch of the first batch start. Used as a fallback "last activity" + * signal for jobs that haven't yet completed their first batch. */ + startedAt?: number + /** Current ms epoch. Injected for testability. */ + now: number +} + +const SLOW_THRESHOLD_MS = 2 * 60 * 1000 +const STALLED_THRESHOLD_MS = 5 * 60 * 1000 + +export function computeJobHealth(input: JobHealthInput): JobHealthStatus { + if (input.status === 'failed') return 'failed' + + // No progress recorded and no activity timestamps — job is still queued. + if ( + input.progress === 0 && + input.lastBatchAt === undefined && + input.startedAt === undefined + ) { + return 'waiting' + } + + const lastActivity = input.lastBatchAt ?? input.startedAt ?? input.now + const stalenessMs = input.now - lastActivity + + if (stalenessMs > STALLED_THRESHOLD_MS) return 'stalled' + if (stalenessMs > SLOW_THRESHOLD_MS) return 'slow' + return 'healthy' +} diff --git a/admin/inertia/components/ActiveEmbedJobs.tsx b/admin/inertia/components/ActiveEmbedJobs.tsx index 9da78bc5..754f4e52 100644 --- a/admin/inertia/components/ActiveEmbedJobs.tsx +++ b/admin/inertia/components/ActiveEmbedJobs.tsx @@ -1,39 +1,101 @@ +import { useEffect, useState } from 'react' import useEmbedJobs from '~/hooks/useEmbedJobs' import HorizontalBarChart from './HorizontalBarChart' +import StyledButton from './StyledButton' import StyledSectionHeader from './StyledSectionHeader' +import { + JOB_HEALTH_DISPLAY, + computeJobHealth, + formatTimeAgo, +} from '~/lib/kb_job_health_display' interface ActiveEmbedJobsProps { withHeader?: boolean } const ActiveEmbedJobs = ({ withHeader = false }: ActiveEmbedJobsProps) => { - const { data: jobs } = useEmbedJobs() + const { data: jobs, invalidate, dataUpdatedAt } = useEmbedJobs() + + // Live "last refreshed Xs ago" tick. We re-render every 5s purely to keep + // the relative timestamp current, without touching React Query state. + const [tick, setTick] = useState(() => Date.now()) + useEffect(() => { + const id = setInterval(() => setTick(Date.now()), 5000) + return () => clearInterval(id) + }, []) return ( <> {withHeader && ( )} + + {/* Refresh row — only shown when at least one job exists so the empty + state stays clean. */} + {jobs && jobs.length > 0 && ( +
+ + {dataUpdatedAt > 0 + ? `Last updated ${formatTimeAgo(dataUpdatedAt, tick)}` + : 'Loading…'} + + + Refresh + +
+ )} +
{jobs && jobs.length > 0 ? ( - jobs.map((job) => ( -
- -
- )) + jobs.map((job) => { + const health = computeJobHealth({ + status: job.status, + progress: job.progress, + lastBatchAt: job.lastBatchAt, + startedAt: job.startedAt, + now: tick, + }) + const display = JOB_HEALTH_DISPLAY[health] + const lastActivityMs = job.lastBatchAt ?? job.startedAt + return ( +
+
+ + + {display.label} + + {lastActivityMs !== undefined && ( + + · last activity {formatTimeAgo(lastActivityMs, tick)} + + )} + {typeof job.chunks === 'number' && job.chunks > 0 && ( + + · {job.chunks.toLocaleString()} chunks + + )} +
+ +
+ ) + }) ) : (

No files are currently being processed

)} diff --git a/admin/inertia/lib/kb_job_health_display.ts b/admin/inertia/lib/kb_job_health_display.ts new file mode 100644 index 00000000..4860a239 --- /dev/null +++ b/admin/inertia/lib/kb_job_health_display.ts @@ -0,0 +1,63 @@ +import { computeJobHealth, type JobHealthStatus } from '../../app/utils/kb_job_health.js' + +export { computeJobHealth, type JobHealthStatus } from '../../app/utils/kb_job_health.js' + +/** + * Visual presentation for each health status — pill color, dot color, and the + * short label rendered alongside the dot. Kept in one place so backend health + * decisions (`computeJobHealth`) and frontend rendering stay in sync. + */ +export const JOB_HEALTH_DISPLAY: Record< + JobHealthStatus, + { dot: string; label: string; ariaLabel: string } +> = { + waiting: { + dot: 'bg-gray-400 dark:bg-gray-500', + label: 'Waiting', + ariaLabel: 'Job is queued and waiting to start', + }, + healthy: { + dot: 'bg-green-500', + label: 'Active', + ariaLabel: 'Job is embedding at a normal rate', + }, + slow: { + dot: 'bg-yellow-500', + label: 'Slow', + ariaLabel: 'Job has not made progress for at least 2 minutes', + }, + stalled: { + dot: 'bg-red-500', + label: 'Stalled', + ariaLabel: 'Job has not made progress for at least 5 minutes', + }, + failed: { + dot: 'bg-red-700', + label: 'Failed', + ariaLabel: 'Job failed', + }, +} + +/** + * Format a relative timestamp as "Xs ago", "Xm ago", "Xh ago" with sensible + * thresholds for the KB Processing Queue's "Last activity" line. + */ +export function formatTimeAgo(timestampMs: number, now: number): string { + const seconds = Math.max(0, Math.floor((now - timestampMs) / 1000)) + if (seconds < 5) return 'just now' + if (seconds < 60) return `${seconds}s ago` + const minutes = Math.floor(seconds / 60) + if (minutes < 60) return `${minutes}m ago` + const hours = Math.floor(minutes / 60) + return `${hours}h ago` +} + +/** + * Convenience wrapper that resolves a job's health status without the caller + * having to remember to pass `now`. Mostly for ergonomic frontend use. + */ +export function computeJobHealthNow( + input: Omit[0], 'now'> +): JobHealthStatus { + return computeJobHealth({ ...input, now: Date.now() }) +} diff --git a/admin/tests/unit/kb_job_health.spec.ts b/admin/tests/unit/kb_job_health.spec.ts new file mode 100644 index 00000000..2b22501a --- /dev/null +++ b/admin/tests/unit/kb_job_health.spec.ts @@ -0,0 +1,100 @@ +import * as assert from 'node:assert/strict' +import { test } from 'node:test' + +import { computeJobHealth } from '../../app/utils/kb_job_health.js' + +const MIN = 60 * 1000 +const NOW = 1_700_000_000_000 // arbitrary fixed epoch for deterministic tests + +test('failed status takes precedence over any timing', () => { + assert.equal( + computeJobHealth({ status: 'failed', progress: 42, lastBatchAt: NOW, now: NOW }), + 'failed' + ) +}) + +test('no progress + no activity timestamps → waiting', () => { + assert.equal( + computeJobHealth({ status: 'waiting', progress: 0, now: NOW }), + 'waiting' + ) +}) + +test('progress > 0 but no lastBatchAt yet → healthy (first batch just started)', () => { + assert.equal( + computeJobHealth({ status: 'processing', progress: 5, startedAt: NOW, now: NOW }), + 'healthy' + ) +}) + +test('lastBatchAt 30s ago → healthy', () => { + assert.equal( + computeJobHealth({ + status: 'batch_completed', + progress: 50, + lastBatchAt: NOW - 30 * 1000, + now: NOW, + }), + 'healthy' + ) +}) + +test('lastBatchAt 90s ago → still healthy (under 2 min threshold)', () => { + assert.equal( + computeJobHealth({ + status: 'batch_completed', + progress: 50, + lastBatchAt: NOW - 90 * 1000, + now: NOW, + }), + 'healthy' + ) +}) + +test('lastBatchAt 3 min ago → slow (CPU-paced ingestion lives here)', () => { + assert.equal( + computeJobHealth({ + status: 'batch_completed', + progress: 50, + lastBatchAt: NOW - 3 * MIN, + now: NOW, + }), + 'slow' + ) +}) + +test('lastBatchAt 4:30 ago → still slow (under 5 min stalled threshold)', () => { + assert.equal( + computeJobHealth({ + status: 'batch_completed', + progress: 50, + lastBatchAt: NOW - 4.5 * MIN, + now: NOW, + }), + 'slow' + ) +}) + +test('lastBatchAt 5:01 ago → stalled', () => { + assert.equal( + computeJobHealth({ + status: 'batch_completed', + progress: 50, + lastBatchAt: NOW - (5 * MIN + 1000), + now: NOW, + }), + 'stalled' + ) +}) + +test('lastBatchAt missing but startedAt 10 min ago → stalled (first-batch-never-finished case)', () => { + assert.equal( + computeJobHealth({ + status: 'processing', + progress: 5, + startedAt: NOW - 10 * MIN, + now: NOW, + }), + 'stalled' + ) +}) diff --git a/admin/types/rag.ts b/admin/types/rag.ts index e84f3498..a50cb4cb 100644 --- a/admin/types/rag.ts +++ b/admin/types/rag.ts @@ -5,6 +5,12 @@ export type EmbedJobWithProgress = { progress: number status: string error?: string + /** ms epoch of last completed batch; multi-batch ZIMs update this each batch. */ + lastBatchAt?: number + /** ms epoch of first batch start; used as a fallback when lastBatchAt unset. */ + startedAt?: number + /** Total chunks embedded across this job's batches so far. */ + chunks?: number } export type ProcessAndEmbedFileResponse = { From 603a7070e8927713646e8ea55860a3cacb59ec72 Mon Sep 17 00:00:00 2001 From: chriscrosstalk <49691103+chriscrosstalk@users.noreply.github.com> Date: Sat, 16 May 2026 21:00:11 -0700 Subject: [PATCH 079/108] =?UTF-8?q?feat(KB):=20Always/Manual=20ingest=20po?= =?UTF-8?q?licy=20toggle=20(RFC=20#883=20=C2=A71/=C2=A74)=20(#894)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat(KB): per-file ingest state machine (Phase 1 of RFC #883) Adds a persistent state machine for AI knowledge-base ingestion so the scanner can distinguish "fully indexed", "user opted out", "failed", and "stalled" from each other — none of which were derivable from the prior binary "any chunks in Qdrant ⇒ embedded" check. ## What lands - New table `kb_ingest_state` keyed by `file_path` with enum state column (`pending_decision | indexed | browse_only | failed | stalled`). Independent of `installed_resources` so it covers both curated downloads and manually-uploaded KB files. - New KV key `rag.defaultIngestPolicy` (string: `Always | Manual`). Registered now but not consumed yet — JIT prompt + wizard step land in Phase 3 of the RFC. - `EmbedFileJob.handle` writes state on terminal outcomes: - Success (final batch) → `indexed` + chunks count - `UnrecoverableError` → `failed` + error message - Retryable errors are left to BullMQ's existing retry path - `scanAndSyncStorage` swaps the binary qdrant check for a state-aware decision tree (see `decideScanAction`). Existing installs auto-backfill on first scan: files with chunks in Qdrant but no state row become `indexed`; new files start as `pending_decision`. - `deleteFileBySource` drops the state row last, so removed files disappear entirely instead of leaving an orphan that the next scan would re-dispatch into nothing. ## What does NOT land here - Ratio registry (separate PR) — needed for partial-stall detection and cost estimates, but a separable concern. - #880 follow-up initial-progress anchor (separate tiny PR). - Phase 2 UI (status pill, per-card actions, conditional warnings). - Phase 3 policy surfaces (wizard step, JIT prompt, guardrail modal). - PR #886's bulk-action hookup — `_deletePointsBySource` / Re-embed All / Reset & Rebuild would also want to set state, but #886 isn't merged yet; that wiring goes in a follow-up once #886 lands. ## Target This is forward work for v1.40.0 (RFC #883). Branching off `rc` because that's the current latest base and post-GA Jake will sync rc→dev; a retarget at PR-open time is a fast-forward if requested. ## Tests - 9 new unit tests for `decideScanAction` covering all five states plus the no-row / chunks-present / chunks-missing combinations - Type-check clean - Smoke-tested end-to-end on NOMAD3 via hot-patch: - Backfill: 5 ZIMs + 2 KB uploads with existing chunks in Qdrant all came back `indexed` on first scan - Pending dispatch: a video-only ZIM with no chunks (`lrnselfreliance`) came back `pending_decision` and was correctly re-dispatched (Bull deduped to its historical `:completed` jobId — bgauger's #886 fix drains that) - Delete hook: deleting a KB upload via `DELETE /api/rag/files` removed both the disk file and the state row * feat(KB): Always/Manual ingest policy toggle (RFC #883 §1/§4) Activates the `rag.defaultIngestPolicy` KV registered in Phase 1 (#888) so users on a fresh install (or anyone who picks Manual mode) no longer get every new ZIM auto-dispatched to the embed pipeline. ## Stacks on #888 This PR's base is `feat/kb-ingest-state-machine` (#888). The state machine has to be in place for the decision function to be policy-aware; GitHub will fast-forward the base to `rc` once #888 merges. ## Backend changes - `decideScanAction` now takes a `policy: 'Always' | 'Manual'` argument (defaults to `Always` for backward compatibility). - New `ScanAction` kind: `create_pending`. Manual mode records that the scanner has seen a new file (so the UI can surface a per-card Index affordance later) without dispatching an EmbedFileJob. - `scanAndSyncStorage` reads the KV and passes it through. The scan-result log line now includes the active policy and a `waiting on user` count for Manual-mode hits. - `rag.defaultIngestPolicy` added to `SETTINGS_KEYS` so it's reachable through the existing `GET/PATCH /api/system/settings` surface — no new endpoint. ## Frontend changes - New section in the KB panel between "Why upload" and "Processing Queue": "Auto-index new content for AI? [Always | Manual]" — segmented radio with copy explaining the 5-10× disk multiplier. Default Always. - `useQuery('ingestPolicy')` reads the current value; clicking the inactive option mutates and shows a notification confirming the new behavior. ## Tests - 14 unit tests on `decideScanAction` (was 9) — split into Always-mode cases (preserves Phase 1's contract) and Manual-mode cases (`create_pending`, `pending_decision → skip`, etc.). - Type-check clean. - Hot-patch + browser verification deferred until #888 lands; the state machine smoke-tested cleanly on NOMAD3 in #888's PR, and this PR's decision-tree changes are exhaustively unit-tested. ## RFC open question §3 — policy-change re-trigger Switching Manual → Always doesn't auto-dispatch existing `pending_decision` rows immediately. The next scan re-evaluates and dispatches them under the new policy. This matches the RFC's "treat the switch as I've- thought-about-it" instinct for the guardrail; full guardrail implementation lands in Phase 3 task 14. --------- Co-authored-by: Jake Turner <52841588+jakeaturner@users.noreply.github.com> --- admin/app/services/rag_service.ts | 23 +++++- admin/app/utils/kb_ingest_decision.ts | 27 +++++-- admin/constants/kv_store.ts | 2 +- .../components/chat/KnowledgeBaseModal.tsx | 73 +++++++++++++++++++ admin/tests/unit/kb_ingest_decision.spec.ts | 72 +++++++++++++----- 5 files changed, 170 insertions(+), 27 deletions(-) diff --git a/admin/app/services/rag_service.ts b/admin/app/services/rag_service.ts index a18b4614..37758183 100644 --- a/admin/app/services/rag_service.ts +++ b/admin/app/services/rag_service.ts @@ -17,7 +17,7 @@ import { randomUUID } from 'node:crypto' import { join, resolve, sep } from 'node:path' import KVStore from '#models/kv_store' import KbIngestState from '#models/kb_ingest_state' -import { decideScanAction } from '../utils/kb_ingest_decision.js' +import { decideScanAction, type IngestPolicy } from '../utils/kb_ingest_decision.js' import { ZIMExtractionService } from './zim_extraction_service.js' import { ZIM_BATCH_SIZE } from '../../constants/zim_extraction.js' import { ProcessAndEmbedFileResponse, ProcessZIMFileResponse, RAGResult, RerankedRAGResult } from '../../types/rag.js' @@ -1353,14 +1353,21 @@ export class RagService { (filePath) => determineFileType(filePath) !== 'unknown' ) + // Read the global ingest policy. Unset is treated as 'Always' so legacy + // installs keep their current behavior until the user explicitly opts + // into Manual mode from the KB panel. + const policyRaw = await KVStore.getValue('rag.defaultIngestPolicy') + const policy: IngestPolicy = policyRaw === 'Manual' ? 'Manual' : 'Always' + const filesToEmbed: string[] = [] let backfilled = 0 let createdRows = 0 + let createdPending = 0 let skipped = 0 for (const filePath of embeddableFiles) { const stateRow = stateByPath.get(filePath) ?? null - const action = decideScanAction(stateRow, sourcesInQdrant.has(filePath)) + const action = decideScanAction(stateRow, sourcesInQdrant.has(filePath), policy) switch (action.kind) { case 'skip': @@ -1378,6 +1385,16 @@ export class RagService { }) backfilled++ break + case 'create_pending': + // Manual mode: record that we've seen the file but don't dispatch. + // The KB panel surfaces a per-card "Index" affordance for these. + await KbIngestState.create({ + file_path: filePath, + state: 'pending_decision', + chunks_embedded: 0, + }) + createdPending++ + break case 'dispatch': if (action.createStateRow) { await KbIngestState.create({ @@ -1393,7 +1410,7 @@ export class RagService { } logger.info( - `[RAG] Scan results: ${filesToEmbed.length} to embed, ${backfilled} backfilled, ${createdRows} new pending, ${skipped} skipped` + `[RAG] Scan results (policy=${policy}): ${filesToEmbed.length} to embed, ${backfilled} backfilled, ${createdRows} new pending, ${createdPending} waiting on user, ${skipped} skipped` ) if (filesToEmbed.length === 0) { diff --git a/admin/app/utils/kb_ingest_decision.ts b/admin/app/utils/kb_ingest_decision.ts index f9417c3d..72794d57 100644 --- a/admin/app/utils/kb_ingest_decision.ts +++ b/admin/app/utils/kb_ingest_decision.ts @@ -2,8 +2,8 @@ import type { KbIngestStateValue } from '../../types/kb_ingest_state.js' /** * Decision returned by `decideScanAction` describing what scanAndSyncStorage - * should do for one file given its current state row (if any) and whether - * Qdrant already has chunks for it. + * should do for one file given its current state row (if any), whether Qdrant + * already has chunks for it, and the global ingest policy. * * - `skip` — file is in a settled state (already indexed, deliberately not * indexed, or in a manual-recovery state); no auto-dispatch. @@ -13,16 +13,26 @@ import type { KbIngestStateValue } from '../../types/kb_ingest_state.js' * - `backfill_indexed` — Qdrant has chunks but no state row exists yet * (pre-RFC install, or new admin instance pointed at an existing Qdrant * volume). Create a row in `indexed` state without re-embedding. + * - `create_pending` — Manual mode: record that we've seen the file but + * don't dispatch. Frontend surfaces a per-card "Index" affordance. */ export type ScanAction = | { kind: 'skip' } | { kind: 'dispatch'; createStateRow: boolean } | { kind: 'backfill_indexed' } + | { kind: 'create_pending' } export interface KbIngestStateRow { state: KbIngestStateValue } +/** + * Global auto-index policy stored at KV `rag.defaultIngestPolicy`. Unset is + * treated as `Always` so existing installs keep their current behavior until + * the user opts into Manual mode through the KB panel. + */ +export type IngestPolicy = 'Always' | 'Manual' + /** * Decide what scanAndSyncStorage should do for a single embeddable file. * @@ -33,18 +43,25 @@ export interface KbIngestStateRow { */ export function decideScanAction( stateRow: KbIngestStateRow | null, - hasChunksInQdrant: boolean + hasChunksInQdrant: boolean, + policy: IngestPolicy = 'Always' ): ScanAction { if (!stateRow) { if (hasChunksInQdrant) return { kind: 'backfill_indexed' } - return { kind: 'dispatch', createStateRow: true } + return policy === 'Always' + ? { kind: 'dispatch', createStateRow: true } + : { kind: 'create_pending' } } switch (stateRow.state) { case 'indexed': return hasChunksInQdrant ? { kind: 'skip' } : { kind: 'dispatch', createStateRow: false } case 'pending_decision': - return { kind: 'dispatch', createStateRow: false } + // Manual mode: file is waiting for the user to opt in via per-card Index. + // Always mode: treat as "user-equivalent of auto-index" and dispatch. + return policy === 'Always' + ? { kind: 'dispatch', createStateRow: false } + : { kind: 'skip' } case 'browse_only': case 'failed': case 'stalled': diff --git a/admin/constants/kv_store.ts b/admin/constants/kv_store.ts index c49416cd..6caeb72b 100644 --- a/admin/constants/kv_store.ts +++ b/admin/constants/kv_store.ts @@ -1,3 +1,3 @@ import { KVStoreKey } from "../types/kv_store.js"; -export const SETTINGS_KEYS: KVStoreKey[] = ['chat.suggestionsEnabled', 'chat.lastModel', 'ui.hasVisitedEasySetup', 'ui.theme', 'system.earlyAccess', 'ai.assistantCustomName', 'ai.remoteOllamaUrl', 'ai.ollamaFlashAttention']; \ No newline at end of file +export const SETTINGS_KEYS: KVStoreKey[] = ['chat.suggestionsEnabled', 'chat.lastModel', 'ui.hasVisitedEasySetup', 'ui.theme', 'system.earlyAccess', 'ai.assistantCustomName', 'ai.remoteOllamaUrl', 'ai.ollamaFlashAttention', 'rag.defaultIngestPolicy']; \ No newline at end of file diff --git a/admin/inertia/components/chat/KnowledgeBaseModal.tsx b/admin/inertia/components/chat/KnowledgeBaseModal.tsx index 5ca7cc50..8b41524a 100644 --- a/admin/inertia/components/chat/KnowledgeBaseModal.tsx +++ b/admin/inertia/components/chat/KnowledgeBaseModal.tsx @@ -51,6 +51,37 @@ export default function KnowledgeBaseModal({ aiAssistantName = "AI Assistant", o select: (data) => data || [], }) + // Global auto-index policy. KVStore returns `null` for an unset key, which + // we treat as 'Always' for backward compatibility with installs that predate + // this UI. The user can opt into Manual mode from the toggle below. + const { data: ingestPolicySetting } = useQuery({ + queryKey: ['ingestPolicy'], + queryFn: () => api.getSetting('rag.defaultIngestPolicy'), + }) + const ingestPolicy: 'Always' | 'Manual' = + ingestPolicySetting?.value === 'Manual' ? 'Manual' : 'Always' + + const updateIngestPolicyMutation = useMutation({ + mutationFn: (policy: 'Always' | 'Manual') => + api.updateSetting('rag.defaultIngestPolicy', policy), + onSuccess: (_data, policy) => { + queryClient.invalidateQueries({ queryKey: ['ingestPolicy'] }) + addNotification({ + type: 'success', + message: + policy === 'Always' + ? 'New content will be auto-indexed for AI.' + : 'New content will wait for you to opt in.', + }) + }, + onError: (error: any) => { + addNotification({ + type: 'error', + message: error?.message || 'Failed to update indexing policy.', + }) + }, + }) + const uploadMutation = useMutation({ mutationFn: (file: File) => api.uploadDocument(file), }) @@ -307,6 +338,48 @@ export default function KnowledgeBaseModal({ aiAssistantName = "AI Assistant", o
+
+
+
+

+ Auto-index new content for AI? +

+

+ Indexed content typically uses 5–10× the original file size on disk. + Changes apply to new content added after this setting changes. +

+
+
+ {(['Always', 'Manual'] as const).map((option) => { + const isActive = ingestPolicy === option + return ( + + ) + })} +
+
+
+
diff --git a/admin/tests/unit/kb_ingest_decision.spec.ts b/admin/tests/unit/kb_ingest_decision.spec.ts index 1f9336d5..531d4b4b 100644 --- a/admin/tests/unit/kb_ingest_decision.spec.ts +++ b/admin/tests/unit/kb_ingest_decision.spec.ts @@ -3,44 +3,80 @@ import { test } from 'node:test' import { decideScanAction } from '../../app/utils/kb_ingest_decision.js' -test('no state row, no chunks → dispatch and create row (new file)', () => { - assert.deepEqual(decideScanAction(null, false), { kind: 'dispatch', createStateRow: true }) +// ---------- Always-policy cases (default behavior; preserves pre-policy install) ---------- + +test('Always: no state row, no chunks → dispatch and create row (new file)', () => { + assert.deepEqual(decideScanAction(null, false, 'Always'), { + kind: 'dispatch', + createStateRow: true, + }) }) -test('no state row, chunks present → backfill_indexed (pre-RFC install, existing Qdrant volume)', () => { - assert.deepEqual(decideScanAction(null, true), { kind: 'backfill_indexed' }) +test('Always: no state row, chunks present → backfill_indexed (pre-RFC install, existing Qdrant volume)', () => { + assert.deepEqual(decideScanAction(null, true, 'Always'), { kind: 'backfill_indexed' }) }) -test('indexed + chunks present → skip', () => { - assert.deepEqual(decideScanAction({ state: 'indexed' }, true), { kind: 'skip' }) +test('Always: indexed + chunks present → skip', () => { + assert.deepEqual(decideScanAction({ state: 'indexed' }, true, 'Always'), { kind: 'skip' }) }) -test('indexed + chunks missing → re-dispatch (state stale, Qdrant collection rebuilt or chunks deleted)', () => { - assert.deepEqual(decideScanAction({ state: 'indexed' }, false), { +test('Always: indexed + chunks missing → re-dispatch (Qdrant collection rebuilt or chunks deleted)', () => { + assert.deepEqual(decideScanAction({ state: 'indexed' }, false, 'Always'), { kind: 'dispatch', createStateRow: false, }) }) -test('pending_decision → dispatch (preserves current Always behavior until policy is consumed)', () => { - assert.deepEqual(decideScanAction({ state: 'pending_decision' }, false), { +test('Always: pending_decision → dispatch', () => { + assert.deepEqual(decideScanAction({ state: 'pending_decision' }, false, 'Always'), { kind: 'dispatch', createStateRow: false, }) }) -test('browse_only → skip (user opted out of indexing)', () => { - assert.deepEqual(decideScanAction({ state: 'browse_only' }, false), { kind: 'skip' }) +test('Always: browse_only → skip (user opted out of indexing)', () => { + assert.deepEqual(decideScanAction({ state: 'browse_only' }, false, 'Always'), { kind: 'skip' }) }) -test('browse_only + chunks present → skip (do not silently re-index after un-index)', () => { - assert.deepEqual(decideScanAction({ state: 'browse_only' }, true), { kind: 'skip' }) +test('Always: failed → skip (manual retry needed, do not auto-redispatch)', () => { + assert.deepEqual(decideScanAction({ state: 'failed' }, false, 'Always'), { kind: 'skip' }) }) -test('failed → skip (manual retry needed, do not auto-redispatch)', () => { - assert.deepEqual(decideScanAction({ state: 'failed' }, false), { kind: 'skip' }) +test('Always: stalled → skip (manual retry needed)', () => { + assert.deepEqual(decideScanAction({ state: 'stalled' }, false, 'Always'), { kind: 'skip' }) }) -test('stalled → skip (manual retry needed)', () => { - assert.deepEqual(decideScanAction({ state: 'stalled' }, false), { kind: 'skip' }) +// ---------- Manual-policy cases ---------- + +test('Manual: no state row, no chunks → create_pending (do not auto-dispatch new content)', () => { + assert.deepEqual(decideScanAction(null, false, 'Manual'), { kind: 'create_pending' }) +}) + +test('Manual: no state row, chunks present → backfill_indexed (same as Always — Qdrant is authoritative)', () => { + assert.deepEqual(decideScanAction(null, true, 'Manual'), { kind: 'backfill_indexed' }) +}) + +test('Manual: pending_decision → skip (waiting for user to opt in via Index button)', () => { + assert.deepEqual(decideScanAction({ state: 'pending_decision' }, false, 'Manual'), { + kind: 'skip', + }) +}) + +test('Manual: indexed + chunks missing → re-dispatch (user has already opted in for this file)', () => { + // Policy switch from Always→Manual must not break in-flight or partially-deleted indexes + // for files the user previously chose to index. + assert.deepEqual(decideScanAction({ state: 'indexed' }, false, 'Manual'), { + kind: 'dispatch', + createStateRow: false, + }) +}) + +test('Manual: browse_only → skip (same as Always)', () => { + assert.deepEqual(decideScanAction({ state: 'browse_only' }, false, 'Manual'), { kind: 'skip' }) +}) + +// ---------- Policy default ---------- + +test('omitted policy defaults to Always (unset KV preserves legacy behavior)', () => { + assert.deepEqual(decideScanAction(null, false), { kind: 'dispatch', createStateRow: true }) }) From ab8281d08bf7077e8ddfab63a344c1537c54f8cb Mon Sep 17 00:00:00 2001 From: Chris Sherwood Date: Thu, 14 May 2026 15:01:24 -0700 Subject: [PATCH 080/108] =?UTF-8?q?feat(KB):=20surface=20embedding-disk=20?= =?UTF-8?q?estimate=20in=20curated=20tier-change=20modal=20(RFC=20#883=20?= =?UTF-8?q?=C2=A71)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When a user picks a tier in TierSelectionModal, show how much additional disk space the AI Assistant will need if the new ZIMs are indexed, plus a policy-aware footer explaining whether they'll auto-index (Always) or wait for opt-in (Manual). Estimates consume #891's KbRatioRegistry via a new POST /api/rag/estimate-batch endpoint. Backend - New POST /api/rag/estimate-batch route + RagController.estimateBatch - VineJS schema accepting array of {filename, sizeBytes}, capped at 500 - KbRatioRegistry.estimateBatch aggregates via the existing prefix-match lookup, returns {totalChunks, totalBytes, hasUnknown} - New BYTES_PER_CHUNK_ON_DISK constant (~8 KB: 3 KB vector + ~3 KB chunk text + ~2 KB payload/index overhead). Tunable; will be replaced by Phase 4 self-calibration once we have real measurements. - Controller normalizes incoming filenames via path.basename so callers that send full paths or URLs still match registry prefixes correctly. Frontend - api.estimateEmbeddingBatch() client method - TierSelectionModal: when localSelectedSlug is set, resolve the tier's resources (incl. inherited tiers), POST to /estimate-batch, and render a new info block with the +~X GB figure + ingest-policy copy. Also fetches rag.defaultIngestPolicy so the same block surfaces whether indexing will fire automatically or wait for the user. - resourceFilename() helper extracts the basename from the resource URL so the registry lookup hits the right prefix regardless of mirror. Tests - 4 new cases in tests/unit/kb_ratio_lookup.spec.ts covering the estimateBatch aggregator: standard sum, unknown-flagging, video-only ZIM (0 chunks but known, hasUnknown stays false), empty input. Stacks on feat/kb-ratio-registry (#891) — consumes the registry table seeded by that PR. Once #891 merges to rc, this PR auto-rebases. Out of scope for this PR (deferred to follow-ups): - Per-batch opt-in checkbox (RFC §1's '☑ Also index these for AI') needs a per-batch policy override path and is a separate PR - Guardrail modal at 50 GB / 10% free / 6 hr thresholds (RFC §7) is also separate; this PR is informational, not gating - Time-to-embed estimate awaits a chunks-per-second metric per host --- admin/app/controllers/rag_controller.ts | 17 +++- admin/app/models/kb_ratio_registry.ts | 18 +++- admin/app/utils/kb_ratio_lookup.ts | 53 +++++++++++ admin/app/validators/rag.ts | 14 +++ .../inertia/components/TierSelectionModal.tsx | 89 +++++++++++++++++-- admin/inertia/lib/api.ts | 11 +++ admin/start/routes.ts | 1 + admin/tests/unit/kb_ratio_lookup.spec.ts | 51 ++++++++++- 8 files changed, 246 insertions(+), 8 deletions(-) diff --git a/admin/app/controllers/rag_controller.ts b/admin/app/controllers/rag_controller.ts index 205fb699..3d01ca42 100644 --- a/admin/app/controllers/rag_controller.ts +++ b/admin/app/controllers/rag_controller.ts @@ -1,11 +1,13 @@ import { RagService } from '#services/rag_service' import { EmbedFileJob } from '#jobs/embed_file_job' +import KbRatioRegistry from '#models/kb_ratio_registry' import { inject } from '@adonisjs/core' import type { HttpContext } from '@adonisjs/core/http' import app from '@adonisjs/core/services/app' import { randomBytes } from 'node:crypto' import { sanitizeFilename } from '../utils/fs.js' -import { deleteFileSchema, getJobStatusSchema } from '#validators/rag' +import { basename } from 'node:path' +import { deleteFileSchema, estimateBatchSchema, getJobStatusSchema } from '#validators/rag' import logger from '@adonisjs/core/services/logger' @inject() @@ -122,4 +124,17 @@ export default class RagController { const result = await this.ragService.checkQdrantHealth() return response.status(200).json(result) } + + public async estimateBatch({ request, response }: HttpContext) { + const { files } = await request.validateUsing(estimateBatchSchema) + // The registry matches on basename prefixes; if a caller passes a full path + // (e.g. /app/storage/zim/wikipedia_en_simple_…), strip directories first so + // patterns like `wikipedia_en_simple_` still match. + const normalized = files.map((f) => ({ + filename: basename(f.filename), + sizeBytes: f.sizeBytes, + })) + const result = await KbRatioRegistry.estimateBatch(normalized) + return response.status(200).json(result) + } } diff --git a/admin/app/models/kb_ratio_registry.ts b/admin/app/models/kb_ratio_registry.ts index 97cd1f2f..13731b80 100644 --- a/admin/app/models/kb_ratio_registry.ts +++ b/admin/app/models/kb_ratio_registry.ts @@ -1,6 +1,12 @@ import { DateTime } from 'luxon' import { BaseModel, column, SnakeCaseNamingStrategy } from '@adonisjs/lucid/orm' -import { findChunksPerMb, estimateChunkCount } from '../utils/kb_ratio_lookup.js' +import { + findChunksPerMb, + estimateChunkCount, + estimateBatch, + type BatchEstimate, + type BatchEstimateInput, +} from '../utils/kb_ratio_lookup.js' /** * Self-calibrating registry of `{filename-prefix → chunks_per_mb}` ratios used @@ -48,4 +54,14 @@ export default class KbRatioRegistry extends BaseModel { const rows = await this.all() return estimateChunkCount(filename, fileSizeBytes, rows) } + + /** + * Aggregate an embedding-disk-cost estimate across a batch of files. Used by + * the curated-tier-change UI to show "you're about to add ~X GB of + * embeddings on top of the ZIM downloads" before the user commits. + */ + static async estimateBatch(files: BatchEstimateInput[]): Promise { + const rows = await this.all() + return estimateBatch(files, rows) + } } diff --git a/admin/app/utils/kb_ratio_lookup.ts b/admin/app/utils/kb_ratio_lookup.ts index 19b22b62..1abd5c6f 100644 --- a/admin/app/utils/kb_ratio_lookup.ts +++ b/admin/app/utils/kb_ratio_lookup.ts @@ -3,6 +3,59 @@ export interface RatioRow { chunks_per_mb: number } +/** + * Bytes of on-disk storage one embedded chunk consumes inside Qdrant. + * + * Rough composition for our pipeline: + * - vector: 768 dims × float32 = 3,072 B + * - chunk text payload: ~3,000 B (target 1,500 tokens × 2 chars/token) + * - source/metadata payload + Qdrant indexes: ~2,000 B + * + * Used for surfacing pre-ingest disk-cost estimates; the actual figure + * varies with collection params and will be replaced by self-calibration + * (RFC #883 Phase 4) once we have real measurements. + */ +export const BYTES_PER_CHUNK_ON_DISK = 8_000 + +export interface BatchEstimateInput { + filename: string + sizeBytes: number +} + +export interface BatchEstimate { + totalChunks: number + totalBytes: number + hasUnknown: boolean +} + +/** + * Aggregate an embedding-disk-cost estimate across a batch of files (curated + * tier add, multi-upload, sync preview, etc). `hasUnknown` is true when at + * least one file did not match any registry row — the totals only include + * matched files, so callers should annotate "estimate excludes unknown files" + * when surfacing the figure. + */ +export function estimateBatch( + files: BatchEstimateInput[], + rows: RatioRow[] +): BatchEstimate { + let totalChunks = 0 + let hasUnknown = false + for (const f of files) { + const chunks = estimateChunkCount(f.filename, f.sizeBytes, rows) + if (chunks === null) { + hasUnknown = true + } else { + totalChunks += chunks + } + } + return { + totalChunks, + totalBytes: totalChunks * BYTES_PER_CHUNK_ON_DISK, + hasUnknown, + } +} + /** * Pick the chunks_per_mb estimate for a filename by longest-prefix match. * diff --git a/admin/app/validators/rag.ts b/admin/app/validators/rag.ts index a9124b4c..8326f44c 100644 --- a/admin/app/validators/rag.ts +++ b/admin/app/validators/rag.ts @@ -11,3 +11,17 @@ export const deleteFileSchema = vine.compile( source: vine.string(), }) ) + +export const estimateBatchSchema = vine.compile( + vine.object({ + files: vine + .array( + vine.object({ + filename: vine.string().minLength(1).maxLength(255), + sizeBytes: vine.number().min(0), + }) + ) + .minLength(1) + .maxLength(500), + }) +) diff --git a/admin/inertia/components/TierSelectionModal.tsx b/admin/inertia/components/TierSelectionModal.tsx index 0a67169e..a55a2443 100644 --- a/admin/inertia/components/TierSelectionModal.tsx +++ b/admin/inertia/components/TierSelectionModal.tsx @@ -1,13 +1,25 @@ -import { Fragment, useState, useEffect } from 'react' +import { Fragment, useState, useEffect, useMemo } from 'react' import { Dialog, Transition } from '@headlessui/react' import { IconX, IconCheck, IconInfoCircle } from '@tabler/icons-react' +import { useQuery } from '@tanstack/react-query' import type { CategoryWithStatus, SpecTier, SpecResource } from '../../types/collections' import { resolveTierResources } from '~/lib/collections' import { formatBytes } from '~/lib/util' +import api from '~/lib/api' import classNames from 'classnames' import DynamicIcon, { DynamicIconName } from './DynamicIcon' import StyledButton from './StyledButton' +/** + * Filename for the embed-estimate registry lookup. Strips the URL path so + * patterns like `wikipedia_en_simple_` continue to match upstream filenames + * regardless of mirror domain. + */ +function resourceFilename(resource: SpecResource): string { + const last = resource.url.split('/').pop() + return last && last.length > 0 ? last : resource.id +} + interface TierSelectionModalProps { isOpen: boolean onClose: () => void @@ -33,13 +45,47 @@ const TierSelectionModal: React.FC = ({ } }, [isOpen, category, selectedTierSlug]) - if (!category) return null - - // Get all resources for a tier (including inherited resources) + // Get all resources for a tier (including inherited resources). Defined as a + // hook-safe closure (always callable, returns [] when no category) so the + // memo below can depend on `category` without breaking hook order. const getAllResourcesForTier = (tier: SpecTier): SpecResource[] => { + if (!category) return [] return resolveTierResources(tier, category.tiers) } + // Pre-compute the selected tier's resources outside the JSX so hooks below + // don't re-run on every render. Empty array when no selection. + const selectedTierResources = useMemo(() => { + if (!category || !localSelectedSlug) return [] + const tier = category.tiers.find((t) => t.slug === localSelectedSlug) + return tier ? resolveTierResources(tier, category.tiers) : [] + }, [category, localSelectedSlug]) + + const embedEstimateRequest = useMemo( + () => + selectedTierResources.map((r) => ({ + filename: resourceFilename(r), + sizeBytes: Math.round(r.size_mb * 1024 * 1024), + })), + [selectedTierResources] + ) + + const { data: embedEstimate } = useQuery({ + queryKey: ['embedEstimateBatch', embedEstimateRequest], + queryFn: () => api.estimateEmbeddingBatch(embedEstimateRequest), + enabled: embedEstimateRequest.length > 0, + staleTime: 5 * 60_000, + }) + + const { data: ingestPolicySetting } = useQuery({ + queryKey: ['ingestPolicy'], + queryFn: () => api.getSetting('rag.defaultIngestPolicy'), + }) + const ingestPolicy: 'Always' | 'Manual' = + ingestPolicySetting?.value === 'Manual' ? 'Manual' : 'Always' + + if (!category) return null + const getTierTotalSize = (tier: SpecTier): number => { return getAllResourcesForTier(tier).reduce((acc, r) => acc + r.size_mb * 1024 * 1024, 0) } @@ -203,8 +249,41 @@ const TierSelectionModal: React.FC = ({ })}
+ {/* Embedding-cost preview — visible whenever a tier is + selected. The estimate uses #891's ratio registry to + project how much extra disk space the AI Assistant will + need for these files on top of the raw downloads. */} + {localSelectedSlug && embedEstimate && embedEstimate.totalBytes > 0 && ( +
+
+ +
+

+ +~{formatBytes(embedEstimate.totalBytes, 1)} + {' '}of additional storage if these are indexed for the AI Assistant + {embedEstimate.hasUnknown && ( + (estimate excludes some files we have no prior data for) + )} + . +

+

+ {ingestPolicy === 'Always' ? ( + <> + Your Auto-index setting is Always, so these files will be indexed automatically once downloaded. You can change this in the Knowledge Base settings. + + ) : ( + <> + Your Auto-index setting is Manual, so these files will sit unindexed until you opt in from the Knowledge Base settings. + + )} +

+
+
+
+ )} + {/* Info note */} -
+

You can change your selection at any time. Click Submit to confirm your choice. diff --git a/admin/inertia/lib/api.ts b/admin/inertia/lib/api.ts index a859cf5d..0b3359f6 100644 --- a/admin/inertia/lib/api.ts +++ b/admin/inertia/lib/api.ts @@ -835,6 +835,17 @@ class API { })() } + async estimateEmbeddingBatch(files: { filename: string; sizeBytes: number }[]) { + return catchInternal(async () => { + const response = await this.client.post<{ + totalChunks: number + totalBytes: number + hasUnknown: boolean + }>('/rag/estimate-batch', { files }) + return response.data + })() + } + // Wikipedia selector methods async getWikipediaState(): Promise { diff --git a/admin/start/routes.ts b/admin/start/routes.ts index 4d214f3e..2038cb7c 100644 --- a/admin/start/routes.ts +++ b/admin/start/routes.ts @@ -149,6 +149,7 @@ router router.post('/sync', [RagController, 'scanAndSync']) router.post('/re-embed-all', [RagController, 'reembedAll']) router.post('/reset-and-rebuild', [RagController, 'resetAndRebuild']) + router.post('/estimate-batch', [RagController, 'estimateBatch']) router.get('/health', [RagController, 'health']) }) .prefix('/api/rag') diff --git a/admin/tests/unit/kb_ratio_lookup.spec.ts b/admin/tests/unit/kb_ratio_lookup.spec.ts index 08c33504..330e2a3a 100644 --- a/admin/tests/unit/kb_ratio_lookup.spec.ts +++ b/admin/tests/unit/kb_ratio_lookup.spec.ts @@ -1,7 +1,12 @@ import * as assert from 'node:assert/strict' import { test } from 'node:test' -import { estimateChunkCount, findChunksPerMb } from '../../app/utils/kb_ratio_lookup.js' +import { + BYTES_PER_CHUNK_ON_DISK, + estimateBatch, + estimateChunkCount, + findChunksPerMb, +} from '../../app/utils/kb_ratio_lookup.js' const SEEDED_ROWS = [ { pattern: 'devdocs_', chunks_per_mb: 1100 }, @@ -60,3 +65,47 @@ test('estimateChunkCount returns null when no match and no fallback', () => { null ) }) + +test('estimateBatch sums chunks and bytes for matched files', () => { + const files = [ + // 100 MB devdocs -> 110,000 chunks + { filename: 'devdocs_en_python_2026-02.zim', sizeBytes: 100 * 1024 * 1024 }, + // 500 MB wikipedia_en_simple -> 135,000 chunks + { filename: 'wikipedia_en_simple_all_nopic_2026-02.zim', sizeBytes: 500 * 1024 * 1024 }, + ] + const out = estimateBatch(files, SEEDED_ROWS) + assert.equal(out.totalChunks, 110_000 + 135_000) + assert.equal(out.totalBytes, (110_000 + 135_000) * BYTES_PER_CHUNK_ON_DISK) + assert.equal(out.hasUnknown, false) +}) + +test('estimateBatch sets hasUnknown when a file has no registry match', () => { + // Drop the empty-string fallback so the unknown file truly has no match + const rows = SEEDED_ROWS.filter((r) => r.pattern !== '') + const files = [ + { filename: 'devdocs_en_python_2026-02.zim', sizeBytes: 100 * 1024 * 1024 }, + { filename: 'something_unknown_2026-02.zim', sizeBytes: 50 * 1024 * 1024 }, + ] + const out = estimateBatch(files, rows) + assert.equal(out.totalChunks, 110_000) // only the matched file + assert.equal(out.hasUnknown, true) +}) + +test('estimateBatch handles video-only ZIMs (0 chunks/MB) without flagging hasUnknown', () => { + // A 5 GB video ZIM matches the registry with 0 chunks/MB; that is a + // *known* value, not an unknown — totals should be 0 and hasUnknown false. + const files = [ + { filename: 'lrnselfreliance_en_all_2025-12.zim', sizeBytes: 5 * 1024 * 1024 * 1024 }, + ] + const out = estimateBatch(files, SEEDED_ROWS) + assert.equal(out.totalChunks, 0) + assert.equal(out.totalBytes, 0) + assert.equal(out.hasUnknown, false) +}) + +test('estimateBatch on empty input returns zeros', () => { + const out = estimateBatch([], SEEDED_ROWS) + assert.equal(out.totalChunks, 0) + assert.equal(out.totalBytes, 0) + assert.equal(out.hasUnknown, false) +}) From 7c2282acf1abc4ca14519686415dc332dbdb860d Mon Sep 17 00:00:00 2001 From: Chris Sherwood Date: Thu, 14 May 2026 12:47:25 -0700 Subject: [PATCH 081/108] =?UTF-8?q?feat(KB):=20conditional=20warnings=20A?= =?UTF-8?q?=20+=20B=20on=20Stored=20Files=20(RFC=20#883=20=C2=A76)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Surfaces two silent failure modes that the prior binary "any-chunks-in-Qdrant ⇒ embedded" check could not distinguish from healthy ingestion: - **Warning A — Zero-chunk file** (file_size > 100 MB, chunks = 0) Fires on video-only / image-only ZIMs (`lrnselfreliance_en_all`, TED talks, etc.) that the pipeline completes "successfully" with no extractable text. AI Assistant literally cannot reference these. - **Warning B — Partial-embed stall** (chunks < 50% of expected from the ratio registry). Surfaces the simple_wiki "266 of 600,000 chunks" case observed during NOMAD1 ingestion testing — previously these looked identical to fully-completed embeds in the UI. Both warnings render only when their condition is met (silent by default; noisy only on real problems). Base is `feat/kb-ratio-registry` (#891) because Warning B's "expected chunks" estimate comes from `KbRatioRegistry.estimateChunks()`. GitHub fast-forwards to `rc` once #891 merges. - `app/utils/kb_warning_decision.ts` — pure `decideWarnings(inputs)` with thresholds (`100 MB`, `0.5×`) as exported constants. 10 unit tests cover the healthy case, both warnings, the under/at/over boundary, the registry-miss suppression, and the video-only registry case (`expectedChunks: 0` correctly skips Warning B). - `RagService.computeFileWarnings()` — single Qdrant scroll tallies chunks per source, filesystem walk fills in zero-chunk files, ratio registry estimates the expectation, decision function emits. - New endpoint `GET /api/rag/file-warnings` returns `Record` (sources with no warnings are omitted, so the frontend can `warnings[source] ?? []` for clean defaults). - KB modal: warnings render inline under the file name as amber-tinted pills. Polled every 30s alongside the existing health check. - Warning C — chunks skipped due to length. PR #890 (#881 fix) prevents the silent drop at the embed boundary, so the underlying condition shouldn't fire anymore. If we still want to surface "we truncated N chunks to fit", that needs separate `skipped_count` tracking in EmbedFileJob — a Phase 2 follow-up. - Suppressing Warning B during active mid-ingestion. The user can cross- reference the Processing Queue to know it's in-flight; suppressing warnings while a job runs would mask real stalls where the job died mid-batch. Will revisit when per-card status is wired through. - Use of `kb_ingest_state.chunks_embedded` (#888) as the chunk count source. This PR uses Qdrant scroll directly so it can land independently of #888. - 10 new unit tests on `decideWarnings`, all pass - Type-check clean - Hot-patch + browser smoke test deferred until #891 lands (the ratio registry needs to exist in the DB for `estimateChunks()` to return non-null estimates — without it, only Warning A fires which is still useful but Warning B stays dormant) --- admin/app/controllers/rag_controller.ts | 5 + admin/app/services/rag_service.ts | 86 ++++++++++++ admin/app/utils/kb_warning_decision.ts | 70 ++++++++++ .../components/chat/KnowledgeBaseModal.tsx | 38 +++++- admin/inertia/lib/api.ts | 10 ++ admin/start/routes.ts | 1 + admin/tests/unit/kb_warning_decision.spec.ts | 125 ++++++++++++++++++ 7 files changed, 334 insertions(+), 1 deletion(-) create mode 100644 admin/app/utils/kb_warning_decision.ts create mode 100644 admin/tests/unit/kb_warning_decision.spec.ts diff --git a/admin/app/controllers/rag_controller.ts b/admin/app/controllers/rag_controller.ts index 3d01ca42..01479f48 100644 --- a/admin/app/controllers/rag_controller.ts +++ b/admin/app/controllers/rag_controller.ts @@ -68,6 +68,11 @@ export default class RagController { return response.status(200).json({ files }) } + public async getFileWarnings({ response }: HttpContext) { + const warnings = await this.ragService.computeFileWarnings() + return response.status(200).json({ warnings }) + } + public async deleteFile({ request, response }: HttpContext) { const { source } = await request.validateUsing(deleteFileSchema) const result = await this.ragService.deleteFileBySource(source) diff --git a/admin/app/services/rag_service.ts b/admin/app/services/rag_service.ts index 37758183..91f1658d 100644 --- a/admin/app/services/rag_service.ts +++ b/admin/app/services/rag_service.ts @@ -18,6 +18,8 @@ import { join, resolve, sep } from 'node:path' import KVStore from '#models/kv_store' import KbIngestState from '#models/kb_ingest_state' import { decideScanAction, type IngestPolicy } from '../utils/kb_ingest_decision.js' +import KbRatioRegistry from '#models/kb_ratio_registry' +import { decideWarnings, type FileWarning } from '../utils/kb_warning_decision.js' import { ZIMExtractionService } from './zim_extraction_service.js' import { ZIM_BATCH_SIZE } from '../../constants/zim_extraction.js' import { ProcessAndEmbedFileResponse, ProcessZIMFileResponse, RAGResult, RerankedRAGResult } from '../../types/rag.js' @@ -1086,6 +1088,90 @@ export class RagService { } } + /** + * Compute conditional warnings (RFC #883 §6) for every source the scanner + * sees on disk. Returns a map from source path → list of warnings, with + * sources that have no warnings omitted entirely (so the frontend can + * `warningsBySource[source] ?? []` for clean defaults). + * + * Per-source chunk counts come from a single Qdrant scroll over the + * collection's points; expected-chunk estimates come from the ratio + * registry. Files in the scanner's directories that have no qdrant points + * at all show up with `chunksInQdrant: 0` so Warning A can fire. + */ + public async computeFileWarnings(): Promise> { + try { + await this._ensureCollection( + RagService.CONTENT_COLLECTION_NAME, + RagService.EMBEDDING_DIMENSION + ) + + // Per-source chunk count from a single scroll. We deliberately don't + // assume `kb_ingest_state.chunks_embedded` here so this PR stays + // independent of the state-machine PR (#888) — but a future cleanup can + // read from there for efficiency once both have landed. + const chunksBySource = new Map() + let offset: string | number | null | Record = null + const batchSize = 100 + do { + const scrollResult = await this.qdrant!.scroll(RagService.CONTENT_COLLECTION_NAME, { + limit: batchSize, + offset, + with_payload: ['source'], + with_vector: false, + }) + for (const point of scrollResult.points) { + const source = point.payload?.source + if (source && typeof source === 'string') { + chunksBySource.set(source, (chunksBySource.get(source) ?? 0) + 1) + } + } + offset = scrollResult.next_page_offset || null + } while (offset !== null) + + // Scan the filesystem the same way scanAndSyncStorage does so Warning A + // can fire on files with zero qdrant points (the headline "video-only + // ZIM" case). + const KB_UPLOADS_PATH = join(process.cwd(), RagService.UPLOADS_STORAGE_PATH) + const ZIM_PATH = join(process.cwd(), ZIM_STORAGE_PATH) + const allSources = new Set(chunksBySource.keys()) + const sizeByPath = new Map() + + for (const dir of [KB_UPLOADS_PATH, ZIM_PATH]) { + try { + const entries = await listDirectoryContentsRecursive(dir) + for (const entry of entries) { + if (entry.type !== 'file') continue + allSources.add(entry.key) + const stat = await getFileStatsIfExists(entry.key) + if (stat) sizeByPath.set(entry.key, Number(stat.size)) + } + } catch (error: any) { + if (error?.code !== 'ENOENT') throw error + } + } + + const out: Record = {} + for (const source of allSources) { + const fileSizeBytes = sizeByPath.get(source) ?? 0 + const chunksInQdrant = chunksBySource.get(source) ?? 0 + const fileName = source.split(/[/\\]/).pop() ?? source + const expectedChunks = + fileSizeBytes > 0 + ? await KbRatioRegistry.estimateChunks(fileName, fileSizeBytes) + : null + + const warnings = decideWarnings({ fileSizeBytes, chunksInQdrant, expectedChunks }) + if (warnings.length > 0) out[source] = warnings + } + + return out + } catch (error) { + logger.error('[RAG] Error computing file warnings:', error) + return {} + } + } + /** * Delete all Qdrant points associated with a given source path and remove * the corresponding file from disk if it lives under the uploads directory. diff --git a/admin/app/utils/kb_warning_decision.ts b/admin/app/utils/kb_warning_decision.ts new file mode 100644 index 00000000..cec59fa7 --- /dev/null +++ b/admin/app/utils/kb_warning_decision.ts @@ -0,0 +1,70 @@ +/** + * Conditional warnings surfaced on Stored Files rows in the KB panel. + * See RFC #883 §6 — these warnings appear ONLY when their triggering condition + * is met, never on healthy files, to keep the panel silent in the common case. + * + * - `zero_chunks` — a non-trivial file produced 0 embedding chunks. Common + * cause: video-only or image-only ZIMs that the pipeline + * completes "successfully" with no extractable text. + * AI Assistant cannot reference this content. + * - `partial_stall` — the file has embedded chunks but well below the count + * expected from the ratio registry. Likely a mid-batch + * stall (which the binary "any chunks ⇒ embedded" check + * used to mask). Surfaces a Retry affordance. + */ +export type FileWarning = + | { kind: 'zero_chunks'; fileSizeBytes: number } + | { kind: 'partial_stall'; chunksEmbedded: number; chunksExpected: number } + +/** Files smaller than this are too small to flag as suspicious zero-chunk + * cases — a 5 KB upload that produces 0 chunks is much more likely to be a + * legitimate edge case (placeholder file) than the gigabyte-scale video ZIM + * problem this warning targets. */ +export const ZERO_CHUNKS_MIN_SIZE_BYTES = 100 * 1024 * 1024 // 100 MB + +/** Fraction of expected chunks below which we consider a file partially + * stalled. 0.5 (50%) matches the threshold described in RFC #883 §6 Warning B. */ +export const PARTIAL_STALL_RATIO_THRESHOLD = 0.5 + +export interface WarningInputs { + /** Source file size on disk in bytes. */ + fileSizeBytes: number + /** Distinct chunks present in Qdrant for this source. */ + chunksInQdrant: number + /** Best estimate of chunks the file should produce, from the ratio + * registry. `null` when no registry pattern matches and no fallback is + * configured — Warning B is suppressed in that case (we'd rather be silent + * than wrong). */ + expectedChunks: number | null +} + +export function decideWarnings(inputs: WarningInputs): FileWarning[] { + const warnings: FileWarning[] = [] + + // Warning A: file is large but produced nothing. Almost always a video-only + // or image-only ZIM; AI Assistant literally cannot reference this content. + if ( + inputs.chunksInQdrant === 0 && + inputs.fileSizeBytes > ZERO_CHUNKS_MIN_SIZE_BYTES + ) { + warnings.push({ kind: 'zero_chunks', fileSizeBytes: inputs.fileSizeBytes }) + } + + // Warning B: chunks present but far below expectation. Suppresses when we + // have no expectation (registry miss) since the comparison would be + // meaningless and we'd rather under-warn than mislead. + if ( + inputs.expectedChunks !== null && + inputs.expectedChunks > 0 && + inputs.chunksInQdrant > 0 && + inputs.chunksInQdrant < inputs.expectedChunks * PARTIAL_STALL_RATIO_THRESHOLD + ) { + warnings.push({ + kind: 'partial_stall', + chunksEmbedded: inputs.chunksInQdrant, + chunksExpected: inputs.expectedChunks, + }) + } + + return warnings +} diff --git a/admin/inertia/components/chat/KnowledgeBaseModal.tsx b/admin/inertia/components/chat/KnowledgeBaseModal.tsx index 8b41524a..9311718a 100644 --- a/admin/inertia/components/chat/KnowledgeBaseModal.tsx +++ b/admin/inertia/components/chat/KnowledgeBaseModal.tsx @@ -51,6 +51,16 @@ export default function KnowledgeBaseModal({ aiAssistantName = "AI Assistant", o select: (data) => data || [], }) + // Per-file conditional warnings (RFC #883 §6). Only sources with at least + // one triggered warning are returned, so an empty map means everything is + // healthy. Polled at the same idle cadence as health for low overhead. + const { data: fileWarnings = {} } = useQuery({ + queryKey: ['kbFileWarnings'], + queryFn: () => api.getKbFileWarnings(), + select: (data) => data ?? {}, + refetchInterval: 30_000, + }) + // Global auto-index policy. KVStore returns `null` for an unset key, which // we treat as 'Always' for backward compatibility with installs that predate // this UI. The user can opt into Manual mode from the toggle below. @@ -442,8 +452,34 @@ export default function KnowledgeBaseModal({ aiAssistantName = "AI Assistant", o accessor: 'source', title: 'File Name', render(record) { + const warnings = fileWarnings[record.source] ?? [] return ( - {record.displayName} +

+ + {sourceToDisplayName(record.source)} + + {warnings.map((w, i) => ( + + + {w.kind === 'zero_chunks' && ( + + Embedded 0 chunks — this file has no text content. + AI Assistant cannot reference it. + + )} + {w.kind === 'partial_stall' && ( + + Only {w.chunksEmbedded.toLocaleString()} of est.{' '} + {w.chunksExpected.toLocaleString()} chunks embedded — + ingestion may have stalled. + + )} + + ))} +
) }, }, diff --git a/admin/inertia/lib/api.ts b/admin/inertia/lib/api.ts index 0b3359f6..628f4a04 100644 --- a/admin/inertia/lib/api.ts +++ b/admin/inertia/lib/api.ts @@ -6,6 +6,7 @@ import { CheckLatestVersionResult, SystemInformationResponse, SystemUpdateStatus import { DownloadJobWithProgress, WikipediaState } from '../../types/downloads' import type { Country, CountryCode, CountryGroup, MapExtractPreflight } from '../../types/maps' import { EmbedJobWithProgress } from '../../types/rag' +import type { FileWarning } from '../../app/utils/kb_warning_decision.js' import type { CategoryWithStatus, CollectionWithStatus, ContentUpdateCheckResult, ResourceUpdateInfo } from '../../types/collections' import { catchInternal } from './util' import { NomadChatResponse, NomadInstalledModel, NomadOllamaModel, OllamaChatRequest } from '../../types/ollama' @@ -475,6 +476,15 @@ class API { })() } + async getKbFileWarnings() { + return catchInternal(async () => { + const response = await this.client.get<{ warnings: Record }>( + '/rag/file-warnings' + ) + return response.data.warnings + })() + } + async deleteRAGFile(source: string) { return catchInternal(async () => { const response = await this.client.delete<{ message: string }>('/rag/files', { data: { source } }) diff --git a/admin/start/routes.ts b/admin/start/routes.ts index 2038cb7c..d30dc61d 100644 --- a/admin/start/routes.ts +++ b/admin/start/routes.ts @@ -141,6 +141,7 @@ router .group(() => { router.post('/upload', [RagController, 'upload']) router.get('/files', [RagController, 'getStoredFiles']) + router.get('/file-warnings', [RagController, 'getFileWarnings']) router.delete('/files', [RagController, 'deleteFile']) router.get('/active-jobs', [RagController, 'getActiveJobs']) router.get('/failed-jobs', [RagController, 'getFailedJobs']) diff --git a/admin/tests/unit/kb_warning_decision.spec.ts b/admin/tests/unit/kb_warning_decision.spec.ts new file mode 100644 index 00000000..4ca6a003 --- /dev/null +++ b/admin/tests/unit/kb_warning_decision.spec.ts @@ -0,0 +1,125 @@ +import * as assert from 'node:assert/strict' +import { test } from 'node:test' + +import { decideWarnings } from '../../app/utils/kb_warning_decision.js' + +const MB = 1024 * 1024 + +test('healthy file: chunks present and on-target → no warnings', () => { + assert.deepEqual( + decideWarnings({ + fileSizeBytes: 100 * MB, + chunksInQdrant: 11_000, + expectedChunks: 11_000, + }), + [] + ) +}) + +test('healthy file: chunks slightly above expectation → no warnings', () => { + assert.deepEqual( + decideWarnings({ + fileSizeBytes: 100 * MB, + chunksInQdrant: 12_000, + expectedChunks: 11_000, + }), + [] + ) +}) + +test('Warning A: large file with 0 chunks (video-only ZIM)', () => { + assert.deepEqual( + decideWarnings({ + fileSizeBytes: 5 * 1024 * MB, + chunksInQdrant: 0, + expectedChunks: 0, + }), + [{ kind: 'zero_chunks', fileSizeBytes: 5 * 1024 * MB }] + ) +}) + +test('Warning A: small empty file is silently ignored (under 100 MB threshold)', () => { + // A user uploads a 5 KB placeholder.txt that produces nothing → not worth a banner + assert.deepEqual( + decideWarnings({ + fileSizeBytes: 5 * 1024, // 5 KB + chunksInQdrant: 0, + expectedChunks: null, + }), + [] + ) +}) + +test('Warning B: partial stall — chunks well below expectation', () => { + assert.deepEqual( + decideWarnings({ + fileSizeBytes: 1000 * MB, + chunksInQdrant: 266, + expectedChunks: 600_000, + }), + [{ kind: 'partial_stall', chunksEmbedded: 266, chunksExpected: 600_000 }] + ) +}) + +test('Warning B: chunks just under 50% of expected → triggers', () => { + assert.deepEqual( + decideWarnings({ + fileSizeBytes: 100 * MB, + chunksInQdrant: 4_999, + expectedChunks: 10_000, + }), + [{ kind: 'partial_stall', chunksEmbedded: 4_999, chunksExpected: 10_000 }] + ) +}) + +test('Warning B: chunks at exactly 50% of expected → does NOT trigger', () => { + // Strict less-than threshold leaves room for the boundary + assert.deepEqual( + decideWarnings({ + fileSizeBytes: 100 * MB, + chunksInQdrant: 5_000, + expectedChunks: 10_000, + }), + [] + ) +}) + +test('Warning B suppressed when expectedChunks is null (registry miss)', () => { + // Better to be silent than show a meaningless "266 of unknown" comparison + assert.deepEqual( + decideWarnings({ + fileSizeBytes: 100 * MB, + chunksInQdrant: 266, + expectedChunks: null, + }), + [] + ) +}) + +test('Warning B suppressed when expectedChunks is 0 (video-only registry entry)', () => { + // A `lrnselfreliance_` row in the registry says "expect 0 chunks". A real + // file matching it correctly producing 0 chunks must not trigger Warning B. + assert.deepEqual( + decideWarnings({ + fileSizeBytes: 500 * MB, + chunksInQdrant: 0, + expectedChunks: 0, + }), + // Note: Warning A triggers here because file > 100 MB and chunks = 0 + [{ kind: 'zero_chunks', fileSizeBytes: 500 * MB }] + ) +}) + +test('Both warnings can fire on the same file in principle', () => { + // Edge case: huge file, 0 chunks, but ratio registry expected 100k. + // Warning A fires (large + zero), Warning B suppressed (chunksInQdrant must be > 0). + // This documents the chunksInQdrant > 0 guard on Warning B. + assert.deepEqual( + decideWarnings({ + fileSizeBytes: 1000 * MB, + chunksInQdrant: 0, + expectedChunks: 100_000, + }), + [{ kind: 'zero_chunks', fileSizeBytes: 1000 * MB }] + ) +}) From cbd86b7af94ba4b4270091830e255a98f75e1cc7 Mon Sep 17 00:00:00 2001 From: Jake Turner Date: Sun, 17 May 2026 04:11:17 +0000 Subject: [PATCH 082/108] refactor(KB): move FileWarning to shared types/rag following existing convention --- admin/app/utils/kb_warning_decision.ts | 6 +++--- admin/inertia/lib/api.ts | 3 +-- admin/types/rag.ts | 6 +++++- 3 files changed, 9 insertions(+), 6 deletions(-) diff --git a/admin/app/utils/kb_warning_decision.ts b/admin/app/utils/kb_warning_decision.ts index cec59fa7..4551566f 100644 --- a/admin/app/utils/kb_warning_decision.ts +++ b/admin/app/utils/kb_warning_decision.ts @@ -12,9 +12,9 @@ * stall (which the binary "any chunks ⇒ embedded" check * used to mask). Surfaces a Retry affordance. */ -export type FileWarning = - | { kind: 'zero_chunks'; fileSizeBytes: number } - | { kind: 'partial_stall'; chunksEmbedded: number; chunksExpected: number } +import type { FileWarning } from '../../types/rag.js' + +export type { FileWarning } /** Files smaller than this are too small to flag as suspicious zero-chunk * cases — a 5 KB upload that produces 0 chunks is much more likely to be a diff --git a/admin/inertia/lib/api.ts b/admin/inertia/lib/api.ts index 628f4a04..950f5938 100644 --- a/admin/inertia/lib/api.ts +++ b/admin/inertia/lib/api.ts @@ -5,8 +5,7 @@ import { FileEntry } from '../../types/files' import { CheckLatestVersionResult, SystemInformationResponse, SystemUpdateStatus } from '../../types/system' import { DownloadJobWithProgress, WikipediaState } from '../../types/downloads' import type { Country, CountryCode, CountryGroup, MapExtractPreflight } from '../../types/maps' -import { EmbedJobWithProgress } from '../../types/rag' -import type { FileWarning } from '../../app/utils/kb_warning_decision.js' +import { EmbedJobWithProgress, FileWarning } from '../../types/rag' import type { CategoryWithStatus, CollectionWithStatus, ContentUpdateCheckResult, ResourceUpdateInfo } from '../../types/collections' import { catchInternal } from './util' import { NomadChatResponse, NomadInstalledModel, NomadOllamaModel, OllamaChatRequest } from '../../types/ollama' diff --git a/admin/types/rag.ts b/admin/types/rag.ts index a50cb4cb..8aaa0f6e 100644 --- a/admin/types/rag.ts +++ b/admin/types/rag.ts @@ -40,4 +40,8 @@ export type RAGResult = { export type RerankedRAGResult = Omit & { finalScore: number -} \ No newline at end of file +} + +export type FileWarning = + | { kind: 'zero_chunks'; fileSizeBytes: number } + | { kind: 'partial_stall'; chunksEmbedded: number; chunksExpected: number } \ No newline at end of file From cbae48a3c8282900fb4e523de64985fb7afd1b45 Mon Sep 17 00:00:00 2001 From: Jake Turner Date: Sun, 17 May 2026 04:23:28 +0000 Subject: [PATCH 083/108] fix(KB): surface file-warning compute failures instead of masking as healthy (PR #895 review) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `computeFileWarnings()` previously caught all errors and returned an empty map, which the frontend rendered as "every file is healthy" — reintroducing exactly the silent-failure mode this surface exists to expose. Return `{ ok, warnings }`; flip `ok: false` from the catch. KB modal renders an inline amber notice under the Stored Files header when `ok === false`, leaving per-row warning rendering untouched. Transient failures self-heal on the next 30s poll; no toast spam. --- admin/app/controllers/rag_controller.ts | 4 ++-- admin/app/services/rag_service.ts | 17 ++++++++------ .../components/chat/KnowledgeBaseModal.tsx | 22 ++++++++++++++----- admin/inertia/lib/api.ts | 8 +++---- admin/types/rag.ts | 14 +++++++++++- 5 files changed, 44 insertions(+), 21 deletions(-) diff --git a/admin/app/controllers/rag_controller.ts b/admin/app/controllers/rag_controller.ts index 01479f48..de84e115 100644 --- a/admin/app/controllers/rag_controller.ts +++ b/admin/app/controllers/rag_controller.ts @@ -69,8 +69,8 @@ export default class RagController { } public async getFileWarnings({ response }: HttpContext) { - const warnings = await this.ragService.computeFileWarnings() - return response.status(200).json({ warnings }) + const result = await this.ragService.computeFileWarnings() + return response.status(200).json(result) } public async deleteFile({ request, response }: HttpContext) { diff --git a/admin/app/services/rag_service.ts b/admin/app/services/rag_service.ts index 91f1658d..7c8f9432 100644 --- a/admin/app/services/rag_service.ts +++ b/admin/app/services/rag_service.ts @@ -19,7 +19,8 @@ import KVStore from '#models/kv_store' import KbIngestState from '#models/kb_ingest_state' import { decideScanAction, type IngestPolicy } from '../utils/kb_ingest_decision.js' import KbRatioRegistry from '#models/kb_ratio_registry' -import { decideWarnings, type FileWarning } from '../utils/kb_warning_decision.js' +import { decideWarnings } from '../utils/kb_warning_decision.js' +import type { FileWarning, FileWarningsResult } from '../../types/rag.js' import { ZIMExtractionService } from './zim_extraction_service.js' import { ZIM_BATCH_SIZE } from '../../constants/zim_extraction.js' import { ProcessAndEmbedFileResponse, ProcessZIMFileResponse, RAGResult, RerankedRAGResult } from '../../types/rag.js' @@ -1090,16 +1091,18 @@ export class RagService { /** * Compute conditional warnings (RFC #883 §6) for every source the scanner - * sees on disk. Returns a map from source path → list of warnings, with - * sources that have no warnings omitted entirely (so the frontend can - * `warningsBySource[source] ?? []` for clean defaults). + * sees on disk. Returns `{ ok, warnings }` — `ok: false` distinguishes a + * computation failure (Qdrant unreachable, DB outage, FS error) from the + * healthy-but-empty case, which is critical because the whole point of this + * surface is to expose silent failures; reporting "everything healthy" when + * we couldn't actually check would reintroduce the bug we set out to fix. * * Per-source chunk counts come from a single Qdrant scroll over the * collection's points; expected-chunk estimates come from the ratio * registry. Files in the scanner's directories that have no qdrant points * at all show up with `chunksInQdrant: 0` so Warning A can fire. */ - public async computeFileWarnings(): Promise> { + public async computeFileWarnings(): Promise { try { await this._ensureCollection( RagService.CONTENT_COLLECTION_NAME, @@ -1165,10 +1168,10 @@ export class RagService { if (warnings.length > 0) out[source] = warnings } - return out + return { ok: true, warnings: out } } catch (error) { logger.error('[RAG] Error computing file warnings:', error) - return {} + return { ok: false, warnings: {} } } } diff --git a/admin/inertia/components/chat/KnowledgeBaseModal.tsx b/admin/inertia/components/chat/KnowledgeBaseModal.tsx index 9311718a..0667d885 100644 --- a/admin/inertia/components/chat/KnowledgeBaseModal.tsx +++ b/admin/inertia/components/chat/KnowledgeBaseModal.tsx @@ -51,15 +51,17 @@ export default function KnowledgeBaseModal({ aiAssistantName = "AI Assistant", o select: (data) => data || [], }) - // Per-file conditional warnings (RFC #883 §6). Only sources with at least - // one triggered warning are returned, so an empty map means everything is - // healthy. Polled at the same idle cadence as health for low overhead. - const { data: fileWarnings = {} } = useQuery({ + // Per-file conditional warnings (RFC #883 §6). `ok: false` means the + // computation itself failed (Qdrant/DB/FS) — distinct from `ok: true` with + // an empty map, which means everything is healthy. We surface the failure + // explicitly so a silent backend failure doesn't masquerade as health. + const { data: warningsResult } = useQuery({ queryKey: ['kbFileWarnings'], queryFn: () => api.getKbFileWarnings(), - select: (data) => data ?? {}, refetchInterval: 30_000, }) + const fileWarnings = warningsResult?.warnings ?? {} + const warningsUnavailable = warningsResult !== undefined && warningsResult.ok === false // Global auto-index policy. KVStore returns `null` for an unset key, which // we treat as 'Always' for backward compatibility with installs that predate @@ -444,7 +446,15 @@ export default function KnowledgeBaseModal({ aiAssistantName = "AI Assistant", o
- + {warningsUnavailable && ( +
+ + + File warnings unavailable — couldn't read storage state. Retrying… + +
+ )} + className="font-semibold" rowLines={true} columns={[ diff --git a/admin/inertia/lib/api.ts b/admin/inertia/lib/api.ts index 950f5938..f16f730a 100644 --- a/admin/inertia/lib/api.ts +++ b/admin/inertia/lib/api.ts @@ -5,7 +5,7 @@ import { FileEntry } from '../../types/files' import { CheckLatestVersionResult, SystemInformationResponse, SystemUpdateStatus } from '../../types/system' import { DownloadJobWithProgress, WikipediaState } from '../../types/downloads' import type { Country, CountryCode, CountryGroup, MapExtractPreflight } from '../../types/maps' -import { EmbedJobWithProgress, FileWarning } from '../../types/rag' +import { EmbedJobWithProgress, FileWarningsResult } from '../../types/rag' import type { CategoryWithStatus, CollectionWithStatus, ContentUpdateCheckResult, ResourceUpdateInfo } from '../../types/collections' import { catchInternal } from './util' import { NomadChatResponse, NomadInstalledModel, NomadOllamaModel, OllamaChatRequest } from '../../types/ollama' @@ -477,10 +477,8 @@ class API { async getKbFileWarnings() { return catchInternal(async () => { - const response = await this.client.get<{ warnings: Record }>( - '/rag/file-warnings' - ) - return response.data.warnings + const response = await this.client.get('/rag/file-warnings') + return response.data })() } diff --git a/admin/types/rag.ts b/admin/types/rag.ts index 8aaa0f6e..aa14127c 100644 --- a/admin/types/rag.ts +++ b/admin/types/rag.ts @@ -44,4 +44,16 @@ export type RerankedRAGResult = Omit & { export type FileWarning = | { kind: 'zero_chunks'; fileSizeBytes: number } - | { kind: 'partial_stall'; chunksEmbedded: number; chunksExpected: number } \ No newline at end of file + | { kind: 'partial_stall'; chunksEmbedded: number; chunksExpected: number } + +/** + * Result of computing per-file warnings. `ok: false` means the computation + * itself failed (Qdrant unreachable, DB outage, FS read error) — distinct from + * `ok: true` with an empty map, which means every scanned file is healthy. + * The frontend should surface a neutral "warnings unavailable" indicator on + * `!ok` rather than implying everything is fine. + */ +export type FileWarningsResult = { + ok: boolean + warnings: Record +} \ No newline at end of file From 5fcbbc5333d4e32e331107749d43ee1bd85af8bf Mon Sep 17 00:00:00 2001 From: Chris Sherwood Date: Thu, 14 May 2026 14:52:17 -0700 Subject: [PATCH 084/108] fix(KB): remove redundant Refresh button from Processing Queue MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit useEmbedJobs already polls every 2s while jobs are active (and 30s when idle) and auto-invalidates Stored Files when the queue drains. The manual Refresh button was a no-op signal — it just confuses users who click it and see no change. Per-job 'last activity Xs ago' lines remain as the live-recency indicator. Stacks on feat/kb-job-status-pill (#893) since the Refresh button only exists in that branch. --- admin/inertia/components/ActiveEmbedJobs.tsx | 21 ++------------------ 1 file changed, 2 insertions(+), 19 deletions(-) diff --git a/admin/inertia/components/ActiveEmbedJobs.tsx b/admin/inertia/components/ActiveEmbedJobs.tsx index 754f4e52..bb8fbd1a 100644 --- a/admin/inertia/components/ActiveEmbedJobs.tsx +++ b/admin/inertia/components/ActiveEmbedJobs.tsx @@ -1,7 +1,6 @@ import { useEffect, useState } from 'react' import useEmbedJobs from '~/hooks/useEmbedJobs' import HorizontalBarChart from './HorizontalBarChart' -import StyledButton from './StyledButton' import StyledSectionHeader from './StyledSectionHeader' import { JOB_HEALTH_DISPLAY, @@ -14,10 +13,9 @@ interface ActiveEmbedJobsProps { } const ActiveEmbedJobs = ({ withHeader = false }: ActiveEmbedJobsProps) => { - const { data: jobs, invalidate, dataUpdatedAt } = useEmbedJobs() + const { data: jobs } = useEmbedJobs() - // Live "last refreshed Xs ago" tick. We re-render every 5s purely to keep - // the relative timestamp current, without touching React Query state. + // Re-render every 5s to keep per-job "last activity Xs ago" timestamps fresh. const [tick, setTick] = useState(() => Date.now()) useEffect(() => { const id = setInterval(() => setTick(Date.now()), 5000) @@ -30,21 +28,6 @@ const ActiveEmbedJobs = ({ withHeader = false }: ActiveEmbedJobsProps) => { )} - {/* Refresh row — only shown when at least one job exists so the empty - state stays clean. */} - {jobs && jobs.length > 0 && ( -
- - {dataUpdatedAt > 0 - ? `Last updated ${formatTimeAgo(dataUpdatedAt, tick)}` - : 'Loading…'} - - - Refresh - -
- )} -
{jobs && jobs.length > 0 ? ( jobs.map((job) => { From 4479919632b983f134462bb01a91830ffa3ac50d Mon Sep 17 00:00:00 2001 From: chriscrosstalk <49691103+chriscrosstalk@users.noreply.github.com> Date: Sat, 16 May 2026 22:44:30 -0700 Subject: [PATCH 085/108] fix(KB): union Stored Files list with state-machine file paths (#898) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Closes the 'zero_chunks warning has no row to attach to' gap surfaced by the 2026-05-14 integration UAT. Before this fix RagService.getStoredFiles returned only file paths that appeared in Qdrant's payload.source — so files with 0 embedded chunks (video-only ZIMs, browse_only opt-outs, ingestions that failed before producing any chunks) silently disappeared from the KB panel's Stored Files table. The fix unions the Qdrant scroll result with the disk-backed file paths recorded in kb_ingest_state. Effect: - lrnselfreliance_en_all_2025-12.zim (3.97 GB video-only ZIM, 0 chunks) now appears in the table, picks up its zero_chunks warning chip - Files in pending_decision under Manual policy show up so the user can see what's waiting for opt-in - Files in browse_only / failed states have a row for future per-card Retry / Re-index actions (forthcoming, blocked on #886) The state-machine query is wrapped in its own try/catch so a transient DB error degrades to the Qdrant-only list rather than 500-ing the whole panel — same defensive posture as the outer try/catch. Stacks on feat/kb-ingest-state-machine (#888) because the union depends on the kb_ingest_state table that PR introduces. Will rebase to rc once #888 merges. Completes the second half of #895's warning surface; the first half (partial_stall) already worked because those files have at least some chunks in Qdrant. --- admin/app/services/rag_service.ts | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/admin/app/services/rag_service.ts b/admin/app/services/rag_service.ts index 7c8f9432..a0eb5be6 100644 --- a/admin/app/services/rag_service.ts +++ b/admin/app/services/rag_service.ts @@ -1082,6 +1082,28 @@ export class RagService { offset = scrollResult.next_page_offset || null } while (offset !== null) + // Union the Qdrant-derived list with the disk-backed file paths the + // state machine has tracked. Without this, files known to the scanner + // but with zero embedded chunks (video-only ZIMs, failed-before-first- + // chunk ingestions, browse_only opt-outs) never get a row in Stored + // Files — which means warnings keyed off those files (#895 zero_chunks + // in particular) have no row to attach to. The state machine is the + // authoritative "what's on disk?" view; Qdrant is "what made it into + // the vector store?". Both are needed to render the KB UI honestly. + try { + const stateRows = await KbIngestState.query().select('file_path') + for (const row of stateRows) { + sources.add(row.file_path) + } + } catch (error) { + // Non-fatal: if the state machine query fails for any reason we'd + // rather return the Qdrant-derived list than 500 the whole panel. + logger.warn( + { err: error }, + '[RagService.getStoredFiles] state-machine union skipped; returning Qdrant-only list' + ) + } + return Array.from(sources) } catch (error) { logger.error('Error retrieving stored files:', error) From 60e7d45908fc6d1388e1dbee2ad0b65d345e264e Mon Sep 17 00:00:00 2001 From: Chris Sherwood Date: Thu, 14 May 2026 16:16:28 -0700 Subject: [PATCH 086/108] feat(KB): first-chat JIT prompt for ingest policy (RFC #883 Phase 3 task 12) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When a user opens AI Chat with content available but no global ingest policy yet recorded, surface a one-time banner above the chat header asking how they want new content handled: - 'Index existing content' -> sets rag.defaultIngestPolicy=Always and triggers a sync so pending_decision files queue immediately - 'Maybe later' -> sets policy=Manual; existing and future content waits in pending_decision until the user opts in from the KB modal After either button is clicked the banner never reappears, because both write the policy KV (the same one #894 manages via the KB modal toggle). There is intentionally no 'dismiss without deciding' X — that would just re-show the banner forever. Backend - New GET /api/rag/policy-prompt-state returns {shouldPrompt, hasContent, totalFiles} - RagService.getPolicyPromptState() reads KVStore('rag.defaultIngestPolicy') and counts kb_ingest_state rows; shouldPrompt is true only when policy is null AND scanner has seen >=1 file (avoids prompting on empty NOMADs) Frontend - New KbPolicyPromptBanner component (~120 LOC) handles the two-button decision flow with optimistic loading state, success/error toasts, and invalidates kbPolicyPromptState + ingestPolicy + embed-jobs + storedFiles on success - Mounted in components/chat/index.tsx as the first child of the main content column so it sits above the chat title bar without taking space when shouldPrompt is false (renders nothing) - Reads aiAssistantName from Inertia page props so banner copy matches the user's chosen assistant name Stacks on feat/kb-policy-toggle (#894) because the policy KV mechanism it writes through is introduced there. Both can land in rc.5; this PR auto-rebases to rc once #894 merges. Existing users on first upgrade to v1.32.0 will see this banner on first chat visit post-upgrade — an explicit opt-in moment for content that was already on disk. New users see it the first time they have curated content downloaded. --- admin/app/controllers/rag_controller.ts | 5 + admin/app/services/rag_service.ts | 26 ++++ .../components/chat/KbPolicyPromptBanner.tsx | 119 ++++++++++++++++++ admin/inertia/components/chat/index.tsx | 2 + admin/inertia/lib/api.ts | 11 ++ admin/start/routes.ts | 1 + 6 files changed, 164 insertions(+) create mode 100644 admin/inertia/components/chat/KbPolicyPromptBanner.tsx diff --git a/admin/app/controllers/rag_controller.ts b/admin/app/controllers/rag_controller.ts index de84e115..7ca7b6de 100644 --- a/admin/app/controllers/rag_controller.ts +++ b/admin/app/controllers/rag_controller.ts @@ -95,6 +95,11 @@ export default class RagController { }) } + public async policyPromptState({ response }: HttpContext) { + const result = await this.ragService.getPolicyPromptState() + return response.status(200).json(result) + } + public async scanAndSync({ response }: HttpContext) { try { const syncResult = await this.ragService.scanAndSyncStorage() diff --git a/admin/app/services/rag_service.ts b/admin/app/services/rag_service.ts index a0eb5be6..21e302f9 100644 --- a/admin/app/services/rag_service.ts +++ b/admin/app/services/rag_service.ts @@ -1111,6 +1111,32 @@ export class RagService { } } + /** + * Compute whether the first-chat JIT prompt should fire and surface the file + * count the banner uses in its copy ("Index your N existing files?"). The + * banner appears when the user hasn't yet picked a global ingest policy + * (`rag.defaultIngestPolicy` unset) and the scanner has actually seen at + * least one embeddable file — i.e., the prompt is actionable, not theoretical + * on a freshly-installed empty NOMAD. + * + * Once the user picks a policy (Always or Manual) via the banner buttons or + * the KB modal toggle, `shouldPrompt` flips to false for good. + */ + public async getPolicyPromptState(): Promise<{ + shouldPrompt: boolean + hasContent: boolean + totalFiles: number + }> { + const policy = await KVStore.getValue('rag.defaultIngestPolicy') + const countRow = await KbIngestState.query().count('* as total').first() + const totalFiles = Number((countRow as any)?.$extras?.total ?? 0) + return { + shouldPrompt: policy === null && totalFiles > 0, + hasContent: totalFiles > 0, + totalFiles, + } + } + /** * Compute conditional warnings (RFC #883 §6) for every source the scanner * sees on disk. Returns `{ ok, warnings }` — `ok: false` distinguishes a diff --git a/admin/inertia/components/chat/KbPolicyPromptBanner.tsx b/admin/inertia/components/chat/KbPolicyPromptBanner.tsx new file mode 100644 index 00000000..3daa6545 --- /dev/null +++ b/admin/inertia/components/chat/KbPolicyPromptBanner.tsx @@ -0,0 +1,119 @@ +import { useMutation, useQuery, useQueryClient } from '@tanstack/react-query' +import { usePage } from '@inertiajs/react' +import { IconBrain } from '@tabler/icons-react' +import api from '~/lib/api' +import StyledButton from '~/components/StyledButton' +import { useNotifications } from '~/context/NotificationContext' + +/** + * First-chat onboarding banner (RFC #883 Phase 3 task 12). + * + * Renders above the chat header when the scanner has seen at least one + * embeddable file AND the user has not yet picked a global ingest policy + * (`rag.defaultIngestPolicy` unset). Two buttons let the user decide once, + * after which the prompt never returns: + * + * - "Index existing content" → sets policy=Always and dispatches a sync so + * anything already on disk + in `pending_decision` gets queued for embed. + * - "Maybe later" → sets policy=Manual. New content waits in + * `pending_decision` until the user opts in from the KB modal. + * + * The "dismiss without deciding" X is intentionally NOT here. Dismissing + * without setting policy would make the banner reappear on every visit until + * a choice is recorded — annoying. The two action buttons each set policy, + * and the user can change their mind any time via the Always/Manual radio in + * the KB modal. + */ +export default function KbPolicyPromptBanner() { + const queryClient = useQueryClient() + const { addNotification } = useNotifications() + // Inertia injects `aiAssistantName` as a shared page prop on chat-mounted + // pages so the banner pulls the user-set name when surfaced. Default to + // "AI Assistant" when accessed outside that context (no-op for chat pages, + // but keeps the component safe for future reuse elsewhere). + const aiAssistantName = + usePage<{ aiAssistantName?: string }>().props?.aiAssistantName || 'AI Assistant' + + const { data: promptState } = useQuery({ + queryKey: ['kbPolicyPromptState'], + queryFn: () => api.getKbPolicyPromptState(), + }) + + const indexNowMutation = useMutation({ + mutationFn: async () => { + await api.updateSetting('rag.defaultIngestPolicy', 'Always') + await api.syncRAGStorage() + }, + onSuccess: () => { + addNotification({ + type: 'success', + message: `${aiAssistantName} will index your existing content. You can track progress in the Knowledge Base panel.`, + }) + queryClient.invalidateQueries({ queryKey: ['kbPolicyPromptState'] }) + queryClient.invalidateQueries({ queryKey: ['ingestPolicy'] }) + queryClient.invalidateQueries({ queryKey: ['embed-jobs'] }) + queryClient.invalidateQueries({ queryKey: ['storedFiles'] }) + }, + onError: (error: any) => { + addNotification({ + type: 'error', + message: error?.message || 'Could not start indexing. Try again from the Knowledge Base panel.', + }) + }, + }) + + const maybeLaterMutation = useMutation({ + mutationFn: () => api.updateSetting('rag.defaultIngestPolicy', 'Manual'), + onSuccess: () => { + addNotification({ + type: 'success', + message: 'Your content stays unindexed for now. You can opt in any time from the Knowledge Base panel.', + }) + queryClient.invalidateQueries({ queryKey: ['kbPolicyPromptState'] }) + queryClient.invalidateQueries({ queryKey: ['ingestPolicy'] }) + }, + }) + + if (!promptState?.shouldPrompt) return null + + const fileCount = promptState.totalFiles + const isBusy = indexNowMutation.isPending || maybeLaterMutation.isPending + + return ( +
+
+ +
+

+ + {fileCount === 1 + ? `Index your existing file for ${aiAssistantName}?` + : `Index your ${fileCount.toLocaleString()} existing files for ${aiAssistantName}?`} + + {' '}When indexed, {aiAssistantName} can reference them while answering your questions. +

+
+
+ indexNowMutation.mutate()} + variant="primary" + size="sm" + disabled={isBusy} + loading={indexNowMutation.isPending} + > + Index existing content + + maybeLaterMutation.mutate()} + variant="ghost" + size="sm" + disabled={isBusy} + loading={maybeLaterMutation.isPending} + > + Maybe later + +
+
+
+ ) +} diff --git a/admin/inertia/components/chat/index.tsx b/admin/inertia/components/chat/index.tsx index 577579ab..04f7c159 100644 --- a/admin/inertia/components/chat/index.tsx +++ b/admin/inertia/components/chat/index.tsx @@ -2,6 +2,7 @@ import { useState, useCallback, useEffect, useRef, useMemo } from 'react' import { useQuery, useMutation, useQueryClient } from '@tanstack/react-query' import ChatSidebar from './ChatSidebar' import ChatInterface from './ChatInterface' +import KbPolicyPromptBanner from './KbPolicyPromptBanner' import StyledModal from '../StyledModal' import api from '~/lib/api' import { formatBytes } from '~/lib/util' @@ -366,6 +367,7 @@ export default function Chat({ isInModal={isInModal} />
+

{activeSession?.title || 'New Chat'} diff --git a/admin/inertia/lib/api.ts b/admin/inertia/lib/api.ts index f16f730a..3d929957 100644 --- a/admin/inertia/lib/api.ts +++ b/admin/inertia/lib/api.ts @@ -853,6 +853,17 @@ class API { })() } + async getKbPolicyPromptState() { + return catchInternal(async () => { + const response = await this.client.get<{ + shouldPrompt: boolean + hasContent: boolean + totalFiles: number + }>('/rag/policy-prompt-state') + return response.data + })() + } + // Wikipedia selector methods async getWikipediaState(): Promise { diff --git a/admin/start/routes.ts b/admin/start/routes.ts index d30dc61d..d117463d 100644 --- a/admin/start/routes.ts +++ b/admin/start/routes.ts @@ -151,6 +151,7 @@ router router.post('/re-embed-all', [RagController, 'reembedAll']) router.post('/reset-and-rebuild', [RagController, 'resetAndRebuild']) router.post('/estimate-batch', [RagController, 'estimateBatch']) + router.get('/policy-prompt-state', [RagController, 'policyPromptState']) router.get('/health', [RagController, 'health']) }) .prefix('/api/rag') From 857eb006bbc7e9481c29ba99a922a6a2082ed54d Mon Sep 17 00:00:00 2001 From: Jake Turner Date: Sun, 17 May 2026 06:03:30 +0000 Subject: [PATCH 087/108] fix(KB): silent maybe-later error + redundant prompt-state refetches (PR #899 review) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - KbPolicyPromptBanner: add onError toast to maybeLaterMutation so a failed policy save surfaces to the user instead of looking like a broken button (banner would otherwise reappear on next chat open with no explanation). - KbPolicyPromptBanner: set staleTime: Infinity on the prompt-state query. For users who already picked a policy (the vast majority), the result is effectively immutable per session — the mutations invalidate the key when it actually changes. --- admin/inertia/components/chat/KbPolicyPromptBanner.tsx | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/admin/inertia/components/chat/KbPolicyPromptBanner.tsx b/admin/inertia/components/chat/KbPolicyPromptBanner.tsx index 3daa6545..b2fd99b9 100644 --- a/admin/inertia/components/chat/KbPolicyPromptBanner.tsx +++ b/admin/inertia/components/chat/KbPolicyPromptBanner.tsx @@ -37,6 +37,7 @@ export default function KbPolicyPromptBanner() { const { data: promptState } = useQuery({ queryKey: ['kbPolicyPromptState'], queryFn: () => api.getKbPolicyPromptState(), + staleTime: Infinity, }) const indexNowMutation = useMutation({ @@ -72,6 +73,12 @@ export default function KbPolicyPromptBanner() { queryClient.invalidateQueries({ queryKey: ['kbPolicyPromptState'] }) queryClient.invalidateQueries({ queryKey: ['ingestPolicy'] }) }, + onError: (error: any) => { + addNotification({ + type: 'error', + message: error?.message || 'Could not save your choice. Try again.', + }) + }, }) if (!promptState?.shouldPrompt) return null From fa7cdbb3398d93c78adc225b775c0f1c4ee77467 Mon Sep 17 00:00:00 2001 From: Chris Sherwood Date: Thu, 14 May 2026 16:28:53 -0700 Subject: [PATCH 088/108] feat(KB): wizard AI policy step (RFC #883 Phase 3 task 13) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds an inline auto-index policy choice inside the Easy Setup wizard's existing AI section (Step 3 'Content', alongside AI model selection). The selection is persisted to KVStore['rag.defaultIngestPolicy'] on wizard submit — same key #894's KB modal toggle reads/writes — so a user who completes the wizard never sees the first-chat JIT prompt (#899); their decision is already recorded. Default is 'Always' so new users who keep the default get the 'just works' experience: content downloaded by the wizard becomes searchable as soon as it finishes embedding, without a follow-up step. Users who prefer the explicit-opt-in flow can flip to 'Manual' before submitting. Skipped when the user doesn't select the AI capability — the KV stays null and the JIT prompt handles the decision later if/when they enable AI from settings. UI placement - Step 3 'Content': new section below AI Models grid (only when AI is selected), two-button radio matching #894's KB-modal toggle pattern for visual consistency - Step 4 'Review': new 'Auto-index Setting' card summarizing the choice in plain English ('New content will be indexed automatically' vs 'New content will wait for you to opt in') so the user knows what they're agreeing to before clicking Complete Setup handleFinish - New api.updateSetting('rag.defaultIngestPolicy', ingestPolicy) call runs first, before service installs/downloads, so any content that finishes embedding during this same wizard run sees the right policy - Wrapped in its own try/catch so a transient KV write failure doesn't abort the rest of the wizard Stacks on feat/kb-policy-toggle (#894) — uses the policy KV mechanism that PR introduces. Auto-rebases to rc when #894 merges. Pairs with #899 (JIT prompt): wizard users decide here; non-wizard users decide at first chat. Together they cover every entry path to v1.32.0 without double-prompting. --- admin/inertia/pages/easy-setup/index.tsx | 84 ++++++++++++++++++++++++ 1 file changed, 84 insertions(+) diff --git a/admin/inertia/pages/easy-setup/index.tsx b/admin/inertia/pages/easy-setup/index.tsx index 2036faf7..f310db8e 100644 --- a/admin/inertia/pages/easy-setup/index.tsx +++ b/admin/inertia/pages/easy-setup/index.tsx @@ -122,6 +122,13 @@ export default function EasySetupWizard(props: { const [selectedServices, setSelectedServices] = useState([]) const [selectedMapCollections, setSelectedMapCollections] = useState([]) const [selectedAiModels, setSelectedAiModels] = useState([]) + // Auto-index policy for the AI Assistant Knowledge Base. Defaults to + // 'Always' so a new user who keeps the default behavior gets the "just + // works" experience — downloads become searchable automatically. Persisted + // to KVStore['rag.defaultIngestPolicy'] on wizard submit (same key #894's + // KB modal toggle reads/writes) so the JIT prompt at first chat sees a + // decided policy and doesn't ask again. + const [ingestPolicy, setIngestPolicy] = useState<'Always' | 'Manual'>('Always') const [isProcessing, setIsProcessing] = useState(false) const [showAdditionalTools, setShowAdditionalTools] = useState(false) const [remoteOllamaEnabled, setRemoteOllamaEnabled] = useState( @@ -338,6 +345,23 @@ export default function EasySetupWizard(props: { setIsProcessing(true) try { + // Persist the auto-index policy choice before kicking off downloads so + // any content that finishes during this same wizard run sees the right + // policy. Skipped when the user did not select the AI capability — the + // KV stays null and the first-chat JIT prompt (#899) handles the + // decision later if/when the user enables AI. + const aiSelected = + selectedServices.includes(SERVICE_NAMES.OLLAMA) || + installedServices.some((s) => s.service_name === SERVICE_NAMES.OLLAMA) + if (aiSelected) { + try { + await api.updateSetting('rag.defaultIngestPolicy', ingestPolicy) + } catch (err) { + // Non-fatal: the user can still set the policy from the KB modal. + console.warn('Could not persist ingest policy from wizard:', err) + } + } + // If using remote Ollama, configure it first before other installs if (remoteOllamaEnabled && remoteOllamaUrl) { const remoteResult = await api.configureRemoteOllama(remoteOllamaUrl) @@ -921,6 +945,47 @@ export default function EasySetupWizard(props: {

No recommended AI models available at this time.

)} + + {/* Auto-index policy — choose now so the JIT prompt at first chat + doesn't ask again (RFC #883 Phase 3 task 13). Persisted to + rag.defaultIngestPolicy on wizard submit. */} +
+

+ Auto-index new content for {aiAssistantName}? +

+

+ When you add new ZIMs, documents, or curated content, should {aiAssistantName} index them automatically so it can search them while answering your questions? +

+
+ + +
+

+ You can change this any time from the Knowledge Base panel inside AI Chat. +

+
)} @@ -1157,6 +1222,25 @@ export default function EasySetupWizard(props: {
)} + {(selectedAiModels.length > 0 || remoteOllamaEnabled) && ( +
+

+ Auto-index Setting +

+

+ {ingestPolicy === 'Always' ? ( + <> + New content will be indexed automatically as it arrives so {aiAssistantName} can search it. + + ) : ( + <> + New content will wait for you to opt in from the Knowledge Base panel before {aiAssistantName} indexes it. + + )} +

+
+ )} + Date: Thu, 14 May 2026 16:35:35 -0700 Subject: [PATCH 089/108] =?UTF-8?q?feat(KB):=20guardrail=20modal=20at=2050?= =?UTF-8?q?GB=20/=2010%-free=20thresholds=20(RFC=20#883=20=C2=A77)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit One-time confirmation step gating bulk indexing actions that would consume a substantial amount of disk for embedding storage. Fires only when the user has policy=Always (i.e., the system would auto-index) AND the estimate trips either: - GUARDRAIL_ABSOLUTE_BYTES = 50 GB embedding cost, OR - GUARDRAIL_FREE_DISK_RATIO = 10% of current free disk space Under policy=Manual the guardrail is silent because the user has already opted out of automatic ingestion — the files would just queue as pending_decision either way. Pieces - inertia/lib/kb_guardrail.ts: pure decision helper with two constants and an evaluateGuardrail() that returns a verdict + reasons. No I/O on the helper itself so the logic is trivially testable - inertia/components/KbGuardrailModal.tsx: confirmation dialog. Headless UI Transition + Dialog, amber 'large operation' header, plain-English estimate summary, [Cancel] / [Proceed anyway] footer. z-[60] so it layers above the tier modal underneath instead of replacing it - inertia/components/TierSelectionModal.tsx integration: handleSubmit now evaluates the guardrail when policy=Always and embedEstimate is available; if it trips, we stash the verdict in state and render the guardrail modal as an overlay. Confirm runs finalizeSubmit (which is the pre-existing onSelectTier + onClose path); Cancel just closes the guardrail and leaves the tier modal as-is so the user can change their tier choice or flip the policy The disk-free signal comes from the existing useSystemInfo hook + getPrimaryDiskInfo helper. Passing freeBytes=0 (unknown) skips the relative-disk check, so the modal still works on hosts whose disk introspection failed — just relies on the absolute 50 GB threshold Tests - 9 cases in tests/unit/kb_guardrail.spec.ts: standard small batch (no trip), exact absolute threshold trips, over-absolute trips, over 10% free trips, both-at-once trips with two reasons, freeBytes=0 skip, freeBytes=0 + over-absolute trip, exact-10% boundary trips, just- under-both safe. All green. Stacks on feat/kb-tier-estimate-on-disk (#897) — consumes that PR's estimate endpoint to compute the verdict input. Auto-rebases to rc when #897 merges. Pairs with #894 (policy toggle) and #899 (JIT prompt): together the three PRs cover the 'how do I avoid surprising the user with auto- indexing they didn't ask for?' arc. Out of scope (deferred) - 6 hr time threshold (RFC §7): needs a per-host chunks-per-second metric we don't capture yet; would be a follow-up after Phase 4 self-calibration (RFC §15) lands - Wider integration (KbPolicyPromptBanner 'Index now' button, manual KB-modal sync): TierSelectionModal is the dominant bulk-decision surface and the right place to land this first --- admin/inertia/components/KbGuardrailModal.tsx | 109 ++++++++++++++++++ .../inertia/components/TierSelectionModal.tsx | 65 ++++++++++- admin/inertia/lib/kb_guardrail.ts | 74 ++++++++++++ admin/tests/unit/kb_guardrail.spec.ts | 106 +++++++++++++++++ 4 files changed, 351 insertions(+), 3 deletions(-) create mode 100644 admin/inertia/components/KbGuardrailModal.tsx create mode 100644 admin/inertia/lib/kb_guardrail.ts create mode 100644 admin/tests/unit/kb_guardrail.spec.ts diff --git a/admin/inertia/components/KbGuardrailModal.tsx b/admin/inertia/components/KbGuardrailModal.tsx new file mode 100644 index 00000000..498cc4c5 --- /dev/null +++ b/admin/inertia/components/KbGuardrailModal.tsx @@ -0,0 +1,109 @@ +import { Fragment } from 'react' +import { Dialog, Transition } from '@headlessui/react' +import { IconAlertTriangle, IconX } from '@tabler/icons-react' +import { formatBytes } from '~/lib/util' +import StyledButton from './StyledButton' +import type { GuardrailVerdict } from '~/lib/kb_guardrail' + +/** + * One-time confirmation modal for bulk indexing actions that trip the + * disk-usage thresholds in `lib/kb_guardrail.ts`. The caller (e.g. + * TierSelectionModal) decides whether to show the modal by evaluating the + * guardrail BEFORE submit; this component just presents the verdict and + * passes the user's choice back via `onConfirm` / `onCancel`. + */ +interface KbGuardrailModalProps { + isOpen: boolean + verdict: GuardrailVerdict + onConfirm: () => void + onCancel: () => void +} + +export default function KbGuardrailModal({ + isOpen, + verdict, + onConfirm, + onCancel, +}: KbGuardrailModalProps) { + // The primary number to surface — every triggered reason carries the same + // estimateBytes, so just grab the first one. `0` is a defensive fallback + // for the (impossible-by-construction) "open with empty verdict" case. + const estimateBytes = verdict.reasons[0]?.estimateBytes ?? 0 + const freeReason = verdict.reasons.find((r) => r.kind === 'over_free_disk') + + return ( + + + +
+ + +
+
+ + +
+
+ + + Confirm large AI indexing operation + +
+ +
+ +
+

+ Indexing this batch for the AI Assistant will use approximately{' '} + {formatBytes(estimateBytes, 1)} of disk space for embeddings, on top of the raw downloads. +

+ + {freeReason && ( +

+ That's more than 10% of your remaining free disk space ({formatBytes(freeReason.freeBytes, 1)} free). Embedding can take several hours and is hard to interrupt cleanly once started. +

+ )} + +

+ If you'd rather review per-item before indexing, cancel here and switch your Auto-index setting to Manual from the Knowledge Base panel. +

+
+ +
+ + Cancel + + + Proceed anyway + +
+
+
+
+
+
+
+ ) +} diff --git a/admin/inertia/components/TierSelectionModal.tsx b/admin/inertia/components/TierSelectionModal.tsx index a55a2443..f135f3cc 100644 --- a/admin/inertia/components/TierSelectionModal.tsx +++ b/admin/inertia/components/TierSelectionModal.tsx @@ -9,6 +9,10 @@ import api from '~/lib/api' import classNames from 'classnames' import DynamicIcon, { DynamicIconName } from './DynamicIcon' import StyledButton from './StyledButton' +import KbGuardrailModal from './KbGuardrailModal' +import { evaluateGuardrail, type GuardrailVerdict } from '~/lib/kb_guardrail' +import { useSystemInfo } from '~/hooks/useSystemInfo' +import { getPrimaryDiskInfo } from '~/hooks/useDiskDisplayData' /** * Filename for the embed-estimate registry lookup. Strips the URL path so @@ -81,6 +85,17 @@ const TierSelectionModal: React.FC = ({ queryKey: ['ingestPolicy'], queryFn: () => api.getSetting('rag.defaultIngestPolicy'), }) + + // System info for the disk-free side of the guardrail. Shared queryKey with + // the home / easy-setup pages so we don't refetch when the user already has + // a fresh copy in cache from a sibling component. + const { data: systemInfo } = useSystemInfo({ enabled: true }) + + // Open state for the guardrail modal — separate from the tier modal so the + // user sees the warning as an overlay without losing their tier selection + // underneath. Cancel returns to the tier modal as-is; Proceed closes both + // and runs the original onSelectTier path. + const [guardrailVerdict, setGuardrailVerdict] = useState(null) const ingestPolicy: 'Always' | 'Manual' = ingestPolicySetting?.value === 'Manual' ? 'Manual' : 'Always' @@ -99,16 +114,49 @@ const TierSelectionModal: React.FC = ({ } } - const handleSubmit = () => { - if (!localSelectedSlug) return + // Compute disk-free bytes from system info; 0 means "unknown", which the + // guardrail helper treats as "skip the relative-disk check". + const freeBytes = useMemo(() => { + const primary = getPrimaryDiskInfo(systemInfo?.disk, systemInfo?.fsSize) + if (!primary) return 0 + return Math.max(0, primary.totalSize - primary.totalUsed) + }, [systemInfo]) - const selectedTier = category.tiers.find(t => t.slug === localSelectedSlug) + /** + * Runs the original onSelectTier-then-onClose flow. Pulled out of + * handleSubmit so the guardrail modal's confirm path can call it after + * the user has consented to the large operation. + */ + const finalizeSubmit = () => { + if (!localSelectedSlug || !category) return + const selectedTier = category.tiers.find((t) => t.slug === localSelectedSlug) if (selectedTier) { onSelectTier(category, selectedTier) } onClose() } + const handleSubmit = () => { + if (!localSelectedSlug || !category) return + + // Guardrail only runs when we have an estimate AND the global policy + // would auto-index this batch. Under Manual the user has already opted + // out of automatic ingestion, so the bulk-disk warning would be a false + // alarm — the files would just queue as pending_decision. + if (ingestPolicy === 'Always' && embedEstimate) { + const verdict = evaluateGuardrail({ + estimateBytes: embedEstimate.totalBytes, + freeBytes, + }) + if (verdict.trips) { + setGuardrailVerdict(verdict) + return + } + } + + finalizeSubmit() + } + return ( @@ -307,6 +355,17 @@ const TierSelectionModal: React.FC = ({
+ {guardrailVerdict && ( + { + setGuardrailVerdict(null) + finalizeSubmit() + }} + onCancel={() => setGuardrailVerdict(null)} + /> + )} ) } diff --git a/admin/inertia/lib/kb_guardrail.ts b/admin/inertia/lib/kb_guardrail.ts new file mode 100644 index 00000000..28ab1a3e --- /dev/null +++ b/admin/inertia/lib/kb_guardrail.ts @@ -0,0 +1,74 @@ +/** + * Auto-index guardrail thresholds and pure decision logic (RFC #883 §7). + * + * The guardrail fires when a user is about to commit to a bulk indexing + * action (curated tier change, large multi-file upload, etc.) that would + * use a substantial amount of disk for embedding storage. It's a one-time + * confirmation step at scary thresholds — it doesn't fire for ordinary + * everyday operations. After the user confirms once for a given batch + * the action proceeds as it would have without the guardrail. + * + * Thresholds are intentionally conservative to avoid surprise consumption + * of a user's storage. Tweak both constants if the field experience + * suggests we're nagging users too aggressively. + */ + +/** Absolute upper bound: estimates at or above this trip the guardrail. */ +export const GUARDRAIL_ABSOLUTE_BYTES = 50 * 1024 * 1024 * 1024 // 50 GB + +/** Relative-to-free-disk bound: estimates >= 10% of free disk trip too. */ +export const GUARDRAIL_FREE_DISK_RATIO = 0.1 + +export type GuardrailReason = + | { + kind: 'over_absolute' + estimateBytes: number + thresholdBytes: number + } + | { + kind: 'over_free_disk' + estimateBytes: number + freeBytes: number + thresholdBytes: number + } + +export type GuardrailVerdict = { + trips: boolean + reasons: GuardrailReason[] +} + +/** + * Decide whether a bulk indexing action should be gated behind the + * guardrail modal. Caller passes the precomputed embedding-storage + * estimate (from `KbRatioRegistry.estimateBatch` in #891 / #897) and + * the free-disk figure from system info. Pass `freeBytes = 0` to skip + * the relative-disk check when free space isn't known. + */ +export function evaluateGuardrail(input: { + estimateBytes: number + freeBytes: number +}): GuardrailVerdict { + const reasons: GuardrailReason[] = [] + + if (input.estimateBytes >= GUARDRAIL_ABSOLUTE_BYTES) { + reasons.push({ + kind: 'over_absolute', + estimateBytes: input.estimateBytes, + thresholdBytes: GUARDRAIL_ABSOLUTE_BYTES, + }) + } + + if (input.freeBytes > 0) { + const relativeThreshold = input.freeBytes * GUARDRAIL_FREE_DISK_RATIO + if (input.estimateBytes >= relativeThreshold) { + reasons.push({ + kind: 'over_free_disk', + estimateBytes: input.estimateBytes, + freeBytes: input.freeBytes, + thresholdBytes: relativeThreshold, + }) + } + } + + return { trips: reasons.length > 0, reasons } +} diff --git a/admin/tests/unit/kb_guardrail.spec.ts b/admin/tests/unit/kb_guardrail.spec.ts new file mode 100644 index 00000000..b300bd4f --- /dev/null +++ b/admin/tests/unit/kb_guardrail.spec.ts @@ -0,0 +1,106 @@ +import * as assert from 'node:assert/strict' +import { test } from 'node:test' + +import { + GUARDRAIL_ABSOLUTE_BYTES, + GUARDRAIL_FREE_DISK_RATIO, + evaluateGuardrail, +} from '../../inertia/lib/kb_guardrail.js' + +const GB = 1024 * 1024 * 1024 + +test('small batch does not trip the guardrail', () => { + const verdict = evaluateGuardrail({ + estimateBytes: 1 * GB, // 1 GB + freeBytes: 500 * GB, + }) + assert.equal(verdict.trips, false) + assert.deepEqual(verdict.reasons, []) +}) + +test('batch at exactly the absolute threshold trips', () => { + const verdict = evaluateGuardrail({ + estimateBytes: GUARDRAIL_ABSOLUTE_BYTES, + freeBytes: 1000 * GB, + }) + assert.equal(verdict.trips, true) + assert.equal(verdict.reasons.length, 1) + assert.equal(verdict.reasons[0].kind, 'over_absolute') +}) + +test('batch over the absolute threshold trips with over_absolute reason', () => { + const verdict = evaluateGuardrail({ + estimateBytes: 60 * GB, + freeBytes: 1000 * GB, + }) + const overAbsolute = verdict.reasons.find((r) => r.kind === 'over_absolute') + assert.ok(overAbsolute, 'should include over_absolute reason') + assert.equal(verdict.trips, true) +}) + +test('batch over 10% of free disk trips with over_free_disk reason', () => { + // 5 GB estimate against 40 GB free disk -> 5 > 4 (10% of 40) + const verdict = evaluateGuardrail({ + estimateBytes: 5 * GB, + freeBytes: 40 * GB, + }) + const overFree = verdict.reasons.find((r) => r.kind === 'over_free_disk') + assert.ok(overFree, 'should include over_free_disk reason') + assert.equal(verdict.trips, true) +}) + +test('batch can trip BOTH thresholds simultaneously', () => { + // 100 GB estimate, 200 GB free + // - over absolute (100 > 50) + // - over 10% of free (100 > 20) + const verdict = evaluateGuardrail({ + estimateBytes: 100 * GB, + freeBytes: 200 * GB, + }) + assert.equal(verdict.trips, true) + assert.equal(verdict.reasons.length, 2) + assert.ok(verdict.reasons.some((r) => r.kind === 'over_absolute')) + assert.ok(verdict.reasons.some((r) => r.kind === 'over_free_disk')) +}) + +test('freeBytes = 0 skips the relative-disk check', () => { + // 100 MB estimate, no free-disk signal: only the absolute check runs, + // and 100 MB is well below the 50 GB absolute threshold + const verdict = evaluateGuardrail({ + estimateBytes: 100 * 1024 * 1024, + freeBytes: 0, + }) + assert.equal(verdict.trips, false) +}) + +test('freeBytes = 0 still trips the absolute check at 50 GB', () => { + const verdict = evaluateGuardrail({ + estimateBytes: 100 * GB, + freeBytes: 0, + }) + assert.equal(verdict.trips, true) + assert.equal(verdict.reasons.length, 1) + assert.equal(verdict.reasons[0].kind, 'over_absolute') +}) + +test('relative-disk threshold computed from GUARDRAIL_FREE_DISK_RATIO constant', () => { + // Estimate exactly equal to 10% of free -> trips (>=) + const free = 100 * GB + const exactlyTenPercent = free * GUARDRAIL_FREE_DISK_RATIO + const verdict = evaluateGuardrail({ + estimateBytes: exactlyTenPercent, + freeBytes: free, + }) + const overFree = verdict.reasons.find((r) => r.kind === 'over_free_disk') + assert.ok(overFree, 'should trip at exactly the threshold') +}) + +test('batch just under both thresholds does not trip', () => { + // 4 GB estimate vs 50 GB free -> 10% of 50 = 5 GB, so 4 < 5 + // Also well below 50 GB absolute + const verdict = evaluateGuardrail({ + estimateBytes: 4 * GB, + freeBytes: 50 * GB, + }) + assert.equal(verdict.trips, false) +}) From 47e6c134e85109375073b9339afca889f7c58e4e Mon Sep 17 00:00:00 2001 From: Jake Turner Date: Sun, 17 May 2026 06:30:22 +0000 Subject: [PATCH 090/108] fix(KB): guardrail bypass during estimate load + Transition sibling (PR #901 review) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Disable TierSelectionModal Submit while the embed-estimate query is in flight, so a fast click can't slip past the guardrail with an undefined estimate. - Move KbGuardrailModal out of the outer and render it as a Fragment sibling — Headless UI's Transition expects Transition.Child descendants, not raw conditional siblings. --- .../inertia/components/TierSelectionModal.tsx | 28 ++++++++++--------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/admin/inertia/components/TierSelectionModal.tsx b/admin/inertia/components/TierSelectionModal.tsx index f135f3cc..3db50941 100644 --- a/admin/inertia/components/TierSelectionModal.tsx +++ b/admin/inertia/components/TierSelectionModal.tsx @@ -74,7 +74,7 @@ const TierSelectionModal: React.FC = ({ [selectedTierResources] ) - const { data: embedEstimate } = useQuery({ + const { data: embedEstimate, isLoading: isEstimating } = useQuery({ queryKey: ['embedEstimateBatch', embedEstimateRequest], queryFn: () => api.estimateEmbeddingBatch(embedEstimateRequest), enabled: embedEstimateRequest.length > 0, @@ -158,6 +158,7 @@ const TierSelectionModal: React.FC = ({ } return ( + <> = ({ variant='primary' size='lg' onClick={handleSubmit} - disabled={!localSelectedSlug} + disabled={!localSelectedSlug || (embedEstimateRequest.length > 0 && isEstimating)} > Submit @@ -355,18 +356,19 @@ const TierSelectionModal: React.FC = ({ - {guardrailVerdict && ( - { - setGuardrailVerdict(null) - finalizeSubmit() - }} - onCancel={() => setGuardrailVerdict(null)} - /> - )} + {guardrailVerdict && ( + { + setGuardrailVerdict(null) + finalizeSubmit() + }} + onCancel={() => setGuardrailVerdict(null)} + /> + )} + ) } From d4623c633afe664e1fa1ba79b6b4f743e0c2d4f3 Mon Sep 17 00:00:00 2001 From: Jake Turner Date: Sun, 17 May 2026 06:42:17 +0000 Subject: [PATCH 091/108] chore(release): 1.32.0-rc.5 [skip ci] --- package-lock.json | 4 ++-- package.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/package-lock.json b/package-lock.json index f12f0e58..184d63b7 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "project-nomad", - "version": "1.32.0-rc.4", + "version": "1.32.0-rc.5", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "project-nomad", - "version": "1.32.0-rc.4", + "version": "1.32.0-rc.5", "license": "ISC" } } diff --git a/package.json b/package.json index 475a818b..b16d4a7b 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "project-nomad", - "version": "1.32.0-rc.4", + "version": "1.32.0-rc.5", "description": "\"", "main": "index.js", "scripts": { From be91b55f16c3ab3d49d22a37359abf16eb2b62aa Mon Sep 17 00:00:00 2001 From: Chris Sherwood Date: Sun, 17 May 2026 11:10:16 -0700 Subject: [PATCH 092/108] fix(KB): blank-screen on panel open + tooltips on bulk-action buttons The Stored Knowledge Base Files render crashed on first open in v1.32.0-rc.5 with `ReferenceError: sourceToDisplayName is not defined`. The table column's render() called `sourceToDisplayName(record.source)` but the function was extracted to `lib/kb_file_grouping.ts` in PR #892 and never imported in KnowledgeBaseModal.tsx. The unhandled error unmounts the entire React tree, so users see a blank screen ~20s after opening the panel. Root cause: PR #895 (conditional warnings) rewrote the render() and used `sourceToDisplayName(record.source)` instead of `record.displayName`, which KbFileGroup already carries from groupAndSortKbFiles(). PR #895's review follow-up (cbae48a) compounded this by narrowing the StyledTable generic from `KbFileGroup` to `{source: string}`, hiding the type drift from tsc. This restores the post-#892 pattern: - StyledTable generic back to `KbFileGroup` - Render uses `record.displayName` (works for both per-file rows and the collapsed admin-docs row; calling sourceToDisplayName on the synthetic `__admin_docs_group__` would have rendered that literal as the row name). Also folds in tooltip copy on the three bulk-action buttons (Reset & Rebuild, Re-embed All, Sync Storage) so the difference in destructiveness is visible on hover. Uses native `title` attribute via StyledButton's prop pass-through; no new component dependency. Inertia tsconfig catches this regression cleanly (TS2304 + TS2339); the pre-push hook only runs the backend tsconfig which excludes inertia/**, so the bug shipped. Tracking the typecheck-coverage gap as a follow-up. Co-Authored-By: Claude Opus 4.7 (1M context) --- admin/inertia/components/chat/KnowledgeBaseModal.tsx | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/admin/inertia/components/chat/KnowledgeBaseModal.tsx b/admin/inertia/components/chat/KnowledgeBaseModal.tsx index 0667d885..dadfb3f0 100644 --- a/admin/inertia/components/chat/KnowledgeBaseModal.tsx +++ b/admin/inertia/components/chat/KnowledgeBaseModal.tsx @@ -420,6 +420,7 @@ export default function KnowledgeBaseModal({ aiAssistantName = "AI Assistant", o onClick={() => { setResetTyped(''); setBulkMode('reset') }} disabled={isUploading || qdrantOffline || bulkBusy} loading={resetMutation.isPending} + title="Drop the entire embeddings collection and re-embed everything from scratch. Permanently removes vectors for files no longer on disk. Destructive: requires typing RESET to confirm." > Reset & Rebuild @@ -430,6 +431,7 @@ export default function KnowledgeBaseModal({ aiAssistantName = "AI Assistant", o onClick={() => setBulkMode('reembed')} disabled={isUploading || qdrantOffline || bulkBusy || storedFiles.length === 0} loading={reembedMutation.isPending} + title="Re-embed every file on disk, replacing existing vectors file-by-file. Vectors for files no longer on disk are preserved. Use this if the chunker or embedding model has changed." > Re-embed All @@ -440,6 +442,7 @@ export default function KnowledgeBaseModal({ aiAssistantName = "AI Assistant", o onClick={handleConfirmSync} disabled={syncMutation.isPending || isUploading || qdrantOffline || bulkBusy} loading={syncMutation.isPending || isUploading} + title="Scan storage for new files and queue any that haven't been embedded yet. Safe to run anytime; won't touch already-embedded content." > Sync Storage @@ -454,7 +457,7 @@ export default function KnowledgeBaseModal({ aiAssistantName = "AI Assistant", o )} - + className="font-semibold" rowLines={true} columns={[ @@ -466,7 +469,7 @@ export default function KnowledgeBaseModal({ aiAssistantName = "AI Assistant", o return (
- {sourceToDisplayName(record.source)} + {record.displayName} {warnings.map((w, i) => ( Date: Sun, 17 May 2026 11:22:17 -0700 Subject: [PATCH 093/108] feat(easy-setup): split AI into its own conditional step (issue #905) Easy Setup wizard previously bundled AI model selection + the new ingest-policy radio into Step 3 alongside Wikipedia/ZIM tiers and curated content. Three problems with that: 1. Predicate divergence: "is AI selected?" was answered three different ways across Step 3 radio, Step 4 review card, and handleFinish persistence. Surfaced in @jakeaturner's review of PR #900. The three predicates disagree in real cases (e.g. Ollama already installed but user didn't re-select any models -- handleFinish writes the ingest KV while the review hides the AI summary). 2. Step 3 was overloaded -- ZIM tiers + curated content + AI models + ingest policy in one screen. 3. No way to opt out of seeing the AI policy radio when AI isn't part of the user's setup. This restructure makes step 4 a dedicated, conditional AI step: Step 1 (Apps) -- unchanged (services + remote Ollama toggle/URL) Step 2 (Maps) -- unchanged Step 3 (Content) -- Wikipedia + curated tiers only Step 4 (AI) -- NEW, conditional: model picker (or remote notice) + auto-index policy radio. Skipped entirely when AI isn't in the setup. Step 5 (Review) -- summary, reads back step 4's output via the same canonical predicate Decisions per issue #905 discussion: - Canonical predicate `isAiInSetup` as a useMemo. Single source consumed by step indicator, nav skip logic, review summary, and handleFinish. Both prior divergence cases collapse. - Step indicator renders dynamically: 4 dots when AI is off (positional display numbers 1..4), 5 dots when AI is on. WizardStep semantic values (1=Apps, 2=Maps, 3=Content, 4=AI, 5=Review) stay stable so nav handlers don't have to translate; the dot's `displayNumber` is decoupled from its `step` so users see sequential 1..N with no gap. - handleNext / handleBack are symmetric: 3 -> 5 forward, 5 -> 3 back, when !isAiInSetup. Same predicate gate. - Toggling AI capability off in Step 1 after AI step selections were made fires a confirm dialog ("Turning off AI will discard your AI model picks, indexing policy, and remote Ollama configuration") and clears selectedAiModels / ingestPolicy / remoteOllamaEnabled on confirm. Silent clear when nothing was set. - Remote Ollama toggle stays in Step 1 alongside the capability card. Don't fragment "am I using remote AI?" across two steps. The bundled review summary (renderStep5, was renderStep4) now uses `isAiInSetup` for the auto-index card visibility instead of the divergent `(selectedAiModels.length > 0 || remoteOllamaEnabled)`. Inertia tsconfig clean for this file (the only outstanding errors are the 3 KnowledgeBaseModal ones from issue tracked in PR #907 and the ~64 pre-existing errors elsewhere). Co-Authored-By: Claude Opus 4.7 (1M context) --- admin/inertia/pages/easy-setup/index.tsx | 447 +++++++++++++---------- 1 file changed, 251 insertions(+), 196 deletions(-) diff --git a/admin/inertia/pages/easy-setup/index.tsx b/admin/inertia/pages/easy-setup/index.tsx index f310db8e..c1fa42ef 100644 --- a/admin/inertia/pages/easy-setup/index.tsx +++ b/admin/inertia/pages/easy-setup/index.tsx @@ -106,7 +106,7 @@ const ADDITIONAL_TOOLS: Capability[] = [ }, ] -type WizardStep = 1 | 2 | 3 | 4 +type WizardStep = 1 | 2 | 3 | 4 | 5 const CURATED_MAP_COLLECTIONS_KEY = 'curated-map-collections' const CURATED_CATEGORIES_KEY = 'curated-categories' @@ -199,6 +199,19 @@ export default function EasySetupWizard(props: { // Services that are already installed const installedServices = props.system.services.filter((service) => service.installed) + // Canonical "is AI part of this user's setup?" predicate (RFC #883 / issue #905). + // Single source consumed by step-indicator render, navigation skip logic, the + // review summary, and handleFinish. The AI step renders if and only if this + // is true; if false, the wizard collapses to 4 steps and the AI step is + // skipped on both forward and back nav. + const isAiInSetup = useMemo( + () => + selectedServices.includes(SERVICE_NAMES.OLLAMA) || + installedServices.some((s) => s.service_name === SERVICE_NAMES.OLLAMA) || + remoteOllamaEnabled, + [selectedServices, installedServices, remoteOllamaEnabled] + ) + const toggleMapCollection = (slug: string) => { setSelectedMapCollections((prev) => prev.includes(slug) ? prev.filter((s) => s !== slug) : [...prev, slug] @@ -313,24 +326,29 @@ export default function EasySetupWizard(props: { // Get primary disk/filesystem info for storage projection const storageInfo = getPrimaryDiskInfo(systemInfo?.disk, systemInfo?.fsSize) + // Final step number (4 when AI is off, 5 when AI is on). Centralizing this + // here so canProceedToNextStep / handleNext / handleBack / the bottom-bar + // Next-vs-Finish switch all read the same value. + const finalStep: WizardStep = isAiInSetup ? 5 : 4 + const canProceedToNextStep = () => { if (!isOnline) return false // Must be online to proceed - if (currentStep === 1) return true // Can skip app installation - if (currentStep === 2) return true // Can skip map downloads - if (currentStep === 3) return true // Can skip ZIM downloads - return false + // Every step before the review is skippable; the review step shows Finish, not Next. + return currentStep < finalStep } const handleNext = () => { - if (currentStep < 4) { - setCurrentStep((prev) => (prev + 1) as WizardStep) - } + if (currentStep >= finalStep) return + // Skip the AI step (4) on forward nav when isAiInSetup is false. + const next = currentStep === 3 && !isAiInSetup ? 5 : currentStep + 1 + setCurrentStep(next as WizardStep) } const handleBack = () => { - if (currentStep > 1) { - setCurrentStep((prev) => (prev - 1) as WizardStep) - } + if (currentStep <= 1) return + // Skip the AI step (4) on back nav when isAiInSetup is false. + const prev = currentStep === 5 && !isAiInSetup ? 3 : currentStep - 1 + setCurrentStep(prev as WizardStep) } const handleFinish = async () => { @@ -347,13 +365,11 @@ export default function EasySetupWizard(props: { try { // Persist the auto-index policy choice before kicking off downloads so // any content that finishes during this same wizard run sees the right - // policy. Skipped when the user did not select the AI capability — the - // KV stays null and the first-chat JIT prompt (#899) handles the - // decision later if/when the user enables AI. - const aiSelected = - selectedServices.includes(SERVICE_NAMES.OLLAMA) || - installedServices.some((s) => s.service_name === SERVICE_NAMES.OLLAMA) - if (aiSelected) { + // policy. Skipped when AI is not in the user's setup; the KV stays null + // and the first-chat JIT prompt (#899) handles the decision later if/when + // the user enables AI. Uses the canonical isAiInSetup predicate so step + // 3 / step 4 / step 5 / handleFinish never disagree (issue #905). + if (isAiInSetup) { try { await api.updateSetting('rag.defaultIngestPolicy', ingestPolicy) } catch (err) { @@ -454,12 +470,25 @@ export default function EasySetupWizard(props: { }, []) const renderStepIndicator = () => { - const steps = [ - { number: 1, label: 'Apps' }, - { number: 2, label: 'Maps' }, - { number: 3, label: 'Content' }, - { number: 4, label: 'Review' }, - ] + // `step` is the stable WizardStep value (1=Apps, 2=Maps, 3=Content, + // 4=AI, 5=Review). `displayNumber` is the sequential position shown in + // the dot (always 1..N) so users see "1 2 3 4" when AI is off and + // "1 2 3 4 5" when AI is on, with no gap. + const baseSteps: Array<{ step: WizardStep; label: string }> = isAiInSetup + ? [ + { step: 1, label: 'Apps' }, + { step: 2, label: 'Maps' }, + { step: 3, label: 'Content' }, + { step: 4, label: 'AI' }, + { step: 5, label: 'Review' }, + ] + : [ + { step: 1, label: 'Apps' }, + { step: 2, label: 'Maps' }, + { step: 3, label: 'Content' }, + { step: 5, label: 'Review' }, + ] + const steps = baseSteps.map((s, idx) => ({ ...s, displayNumber: idx + 1 })) return (