Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
21 changes: 20 additions & 1 deletion admin/app/controllers/rag_controller.ts
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ import app from '@adonisjs/core/services/app'
import { randomBytes } from 'node:crypto'
import { sanitizeFilename } from '../utils/fs.js'
import { basename } from 'node:path'
import { deleteFileSchema, embedFileSchema, estimateBatchSchema, getJobStatusSchema } from '#validators/rag'
import { deleteFileSchema, embedFileSchema, estimateBatchSchema, fileSourceSchema, getJobStatusSchema } from '#validators/rag'
import logger from '@adonisjs/core/services/logger'

@inject()
Expand Down Expand Up @@ -162,4 +162,23 @@ export default class RagController {
const result = await KbRatioRegistry.estimateBatch(normalized)
return response.status(200).json(result)
}

public async getFileContent({ request, response }: HttpContext) {
const { source } = await request.validateUsing(fileSourceSchema)
const result = await this.ragService.readFileContent(source)
if (!result) {
return response.status(404).json({ error: 'File not found or not viewable' })
}
return response.status(200).json(result)
}

public async downloadFile({ request, response }: HttpContext) {
const { source } = await request.validateUsing(fileSourceSchema)
const filePath = await this.ragService.resolveDownloadPath(source)
if (!filePath) {
return response.status(404).json({ error: 'File not found' })
}
const fileName = filePath.split(/[/\\]/).at(-1) ?? 'download'
return response.attachment(filePath, fileName)
}
}
86 changes: 78 additions & 8 deletions admin/app/services/rag_service.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1120,20 +1120,90 @@ export class RagService {
)
}

return Array.from(sources).map((source) => {
const row = stateByPath.get(source)
return {
source,
state: row?.state ?? null,
chunksEmbedded: row?.chunks_embedded ?? 0,
}
})
const uploadsAbsPath = resolve(join(process.cwd(), RagService.UPLOADS_STORAGE_PATH))
return await Promise.all(
Array.from(sources).map(async (source) => {
const row = stateByPath.get(source)
const fileName = source.split(/[/\\]/).at(-1) ?? source
const isUserUpload = resolve(source).startsWith(uploadsAbsPath + sep)
const stats = await getFileStatsIfExists(source)
return {
source,
state: row?.state ?? null,
chunksEmbedded: row?.chunks_embedded ?? 0,
fileName,
size: stats?.size ?? null,
uploadedAt: stats?.modifiedTime.toISOString() ?? null,
isUserUpload,
}
})
)
} catch (error) {
logger.error('Error retrieving stored files:', error)
return []
}
}

/**
* Resolve a stored-file `source` to an absolute disk path, but only if the
* path lives under the uploads directory. Mirrors the docs_service traversal
* guard: resolve, then require the resolved path to be strictly inside the
* base + path separator (so siblings of `kb_uploads` can't slip through).
* Returns null for anything else — ZIMs, admin docs, README, or paths
* outside the app entirely. The viewer/download endpoints lean on this so
* they don't need to repeat the check.
*/
private resolveUploadPath(source: string): string | null {
const uploadsAbsPath = resolve(join(process.cwd(), RagService.UPLOADS_STORAGE_PATH))
const resolved = resolve(source)
if (!resolved.startsWith(uploadsAbsPath + sep)) return null
return resolved
}

private static readonly VIEWABLE_TEXT_EXTENSIONS: ReadonlySet<string> = new Set([
'md', 'txt', 'csv', 'json', 'yaml', 'yml', 'toml', 'xml', 'html',
])

/**
* Read the text content of a user-uploaded file for in-browser viewing.
* Returns null when the source is outside uploads, missing, or not a
* recognized text extension. The extension allowlist is intentionally narrow
* — PDFs/EPUBs/ZIMs round-trip through Download, not the viewer.
*/
public async readFileContent(
source: string
): Promise<{ content: string; extension: string; fileName: string } | null> {
const resolved = this.resolveUploadPath(source)
if (!resolved) return null

const extension = resolved.split('.').at(-1)?.toLowerCase() ?? ''
if (!RagService.VIEWABLE_TEXT_EXTENSIONS.has(extension)) return null

const stats = await getFileStatsIfExists(resolved)
if (!stats) return null

try {
const { readFile } = await import('node:fs/promises')
const content = await readFile(resolved, 'utf-8')
const fileName = resolved.split(/[/\\]/).at(-1) ?? resolved
return { content, extension, fileName }
} catch (error) {
logger.warn({ err: error, source }, '[RagService.readFileContent] read failed')
return null
}
}

/**
* Resolve a download-target path for a stored upload. Returns null if the
* source isn't an upload or the file is missing on disk.
*/
public async resolveDownloadPath(source: string): Promise<string | null> {
const resolved = this.resolveUploadPath(source)
if (!resolved) return null
const stats = await getFileStatsIfExists(resolved)
return stats ? resolved : null
}

/**
* Compute whether the first-chat JIT prompt should fire and surface the file
* count the banner uses in its copy ("Index your N existing files?"). The
Expand Down
6 changes: 6 additions & 0 deletions admin/app/validators/rag.ts
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,12 @@ export const embedFileSchema = vine.compile(
})
)

export const fileSourceSchema = vine.compile(
vine.object({
source: vine.string().minLength(1),
})
)

export const estimateBatchSchema = vine.compile(
vine.object({
files: vine
Expand Down
155 changes: 152 additions & 3 deletions admin/inertia/components/chat/KnowledgeBaseModal.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -10,9 +10,19 @@ import api from '~/lib/api'
import {
groupAndSortKbFiles,
type KbFileGroup,
type KbFileSort,
type KbFileSortKey,
} from '~/lib/kb_file_grouping'
import type { KbIngestStateValue } from '../../../types/kb_ingest_state'
import { IconX } from '@tabler/icons-react'
import { formatBytes } from '~/lib/util'
import {
IconArrowsSort,
IconDownload,
IconEye,
IconSortAscending,
IconSortDescending,
IconX,
} from '@tabler/icons-react'
import { useModals } from '~/context/ModalContext'
import StyledModal from '../StyledModal'
import ActiveEmbedJobs from '~/components/ActiveEmbedJobs'
Expand All @@ -23,6 +33,42 @@ interface KnowledgeBaseModalProps {
onClose: () => void
}

// File extensions the in-browser viewer can render. Must stay in sync with
// `RagService.VIEWABLE_TEXT_EXTENSIONS` — anything outside this set falls back
// to Download.
const VIEWABLE_EXTENSIONS = new Set(['md', 'txt', 'csv', 'json', 'yaml', 'yml', 'toml', 'xml', 'html'])

function isViewableExtension(filename: string): boolean {
const ext = filename.split('.').at(-1)?.toLowerCase() ?? ''
return VIEWABLE_EXTENSIONS.has(ext)
}

function renderSortHeader(
label: string,
key: KbFileSortKey,
sort: KbFileSort,
setSort: (s: KbFileSort) => void
): React.ReactNode {
const active = sort.key === key
const Icon = !active ? IconArrowsSort : sort.direction === 'asc' ? IconSortAscending : IconSortDescending
return (
<button
type="button"
className="inline-flex items-center gap-1 text-left hover:text-text-primary transition-colors"
onClick={() => {
if (!active) {
setSort({ key, direction: 'asc' })
} else {
setSort({ key, direction: sort.direction === 'asc' ? 'desc' : 'asc' })
}
}}
>
<span>{label}</span>
<Icon size={14} className={active ? 'text-text-primary' : 'text-text-muted'} aria-hidden="true" />
</button>
)
}

/**
* Compact label for the per-row ingestion state. Files that exist in Qdrant
* with no `kb_ingest_state` row (`state === null`) are legacy/pre-RFC-883
Expand Down Expand Up @@ -101,6 +147,8 @@ export default function KnowledgeBaseModal({ aiAssistantName = "AI Assistant", o
const [confirmReembed, setConfirmReembed] = useState<{ source: string; displayName: string } | null>(null)
const [bulkMode, setBulkMode] = useState<null | 'reembed' | 'reset'>(null)
const [resetTyped, setResetTyped] = useState('')
const [sort, setSort] = useState<KbFileSort>({ key: 'name', direction: 'asc' })
const [viewerSource, setViewerSource] = useState<string | null>(null)
const fileUploaderRef = useRef<React.ComponentRef<typeof FileUploader>>(null)
const { openModal, closeModal } = useModals()
const queryClient = useQueryClient()
Expand Down Expand Up @@ -555,7 +603,7 @@ export default function KnowledgeBaseModal({ aiAssistantName = "AI Assistant", o
columns={[
{
accessor: 'source',
title: 'File Name',
title: renderSortHeader('File Name', 'name', sort, setSort),
render(record) {
const warnings = fileWarnings[record.source] ?? []
const pill = renderStatePill(record)
Expand Down Expand Up @@ -594,6 +642,35 @@ export default function KnowledgeBaseModal({ aiAssistantName = "AI Assistant", o
)
},
},
{
accessor: 'size',
title: renderSortHeader('Size', 'size', sort, setSort),
className: 'whitespace-nowrap',
render(record) {
// The collapsed admin_docs group has no single size — leave blank
// rather than misleadingly summing across N files.
if (record.bucket === 'admin_docs' || record.size === null) {
return <span className="text-text-muted">—</span>
}
return <span className="text-text-secondary">{formatBytes(record.size)}</span>
},
},
{
accessor: 'uploadedAt',
title: renderSortHeader('Uploaded', 'uploadedAt', sort, setSort),
className: 'whitespace-nowrap',
render(record) {
if (record.bucket === 'admin_docs' || !record.uploadedAt) {
return <span className="text-text-muted">—</span>
}
const d = new Date(record.uploadedAt)
return (
<span className="text-text-secondary" title={d.toISOString()}>
{d.toLocaleDateString()}
</span>
)
},
},
{
accessor: 'source',
title: '',
Expand Down Expand Up @@ -642,6 +719,9 @@ export default function KnowledgeBaseModal({ aiAssistantName = "AI Assistant", o
const actionPendingForThisRow =
embedMutation.isPending && embedMutation.variables?.source === record.source

const canView = record.isUserUpload && isViewableExtension(record.displayName) && record.size !== null
const canDownload = record.isUserUpload && record.size !== null

return (
<div className="flex justify-end items-center gap-2">
{action && (
Expand All @@ -662,6 +742,24 @@ export default function KnowledgeBaseModal({ aiAssistantName = "AI Assistant", o
{action.label}
</StyledButton>
)}
{canView && (
<StyledButton
variant="ghost"
size="sm"
icon="IconEye"
onClick={() => setViewerSource(record.source)}
>View</StyledButton>
)}
{canDownload && (
<StyledButton
variant="ghost"
size="sm"
icon="IconDownload"
onClick={() => {
window.location.href = `/api/rag/files/download?source=${encodeURIComponent(record.source)}`
}}
>Download</StyledButton>
)}
<StyledButton
variant="danger"
size="sm"
Expand All @@ -675,7 +773,7 @@ export default function KnowledgeBaseModal({ aiAssistantName = "AI Assistant", o
},
},
]}
data={groupAndSortKbFiles(storedFiles)}
data={groupAndSortKbFiles(storedFiles, sort)}
loading={isLoadingFiles}
/>
</div>
Expand Down Expand Up @@ -807,6 +905,57 @@ export default function KnowledgeBaseModal({ aiAssistantName = "AI Assistant", o
</div>
</StyledModal>
)}

{viewerSource && (
<FileViewerModal
source={viewerSource}
onClose={() => setViewerSource(null)}
/>
)}
</div>
)
}

function FileViewerModal({ source, onClose }: { source: string; onClose: () => void }) {
const { data, isLoading, isFetched } = useQuery({
queryKey: ['rag', 'file-content', source],
queryFn: () => api.getFileContent(source),
staleTime: 60_000,
})

// Title falls back to the trailing path segment so the modal still has a
// useful header while the fetch is in-flight or if it failed.
const fallbackName = source.split(/[/\\]/).at(-1) ?? source
const title = data?.fileName ?? fallbackName
// `catchInternal` swallows errors and resolves to undefined, surfacing a
// toast — so the "couldn't load" branch is gated on a finished-but-empty
// fetch rather than on react-query's `isError`.
const showError = isFetched && !data

return (
<StyledModal
title={title}
open={true}
onClose={onClose}
onCancel={onClose}
cancelText="Close"
large
>
<div className="text-left text-sm">
{isLoading && (
<div className="text-text-secondary">Loading…</div>
)}
{showError && (
<div className="text-amber-700 dark:text-amber-300">
Couldn't load file. It may have been moved or its type isn't viewable.
</div>
)}
{data && (
<pre className="max-h-[60vh] overflow-auto whitespace-pre-wrap rounded border border-border-subtle bg-surface-secondary p-3 font-mono text-xs text-text-primary">
{data.content}
</pre>
)}
</div>
</StyledModal>
)
}
11 changes: 11 additions & 0 deletions admin/inertia/lib/api.ts
Original file line number Diff line number Diff line change
Expand Up @@ -514,6 +514,17 @@ class API {
})()
}

async getFileContent(source: string) {
return catchInternal(async () => {
const response = await this.client.get<{
content: string
extension: string
fileName: string
}>('/rag/files/content', { params: { source } })
return response.data
})()
}

async getSystemInfo() {
return catchInternal(async () => {
const response = await this.client.get<SystemInformationResponse>('/system/info')
Expand Down
Loading