From e955dc192c6c9c7705a38064a849005d6dccf195 Mon Sep 17 00:00:00 2001 From: Henry Estela Date: Sun, 19 Apr 2026 22:51:50 -0700 Subject: [PATCH] feat(zim): add zim uploader in content manager adds a collapsible file uploader to accept zim file uploads into kiwix. --- admin/app/controllers/zim_controller.ts | 86 ++++++++++++++ admin/app/services/zim_service.ts | 107 ++++++++++++++++++ admin/config/bodyparser.ts | 2 +- .../inertia/components/ZimUploader/index.tsx | 77 +++++++++++++ admin/inertia/pages/settings/zim/index.tsx | 48 ++++++-- .../pages/settings/zim/remote-explorer.tsx | 18 ++- admin/package-lock.json | 69 ++++++++++- admin/package.json | 1 + admin/start/routes.ts | 1 + 9 files changed, 397 insertions(+), 12 deletions(-) create mode 100644 admin/inertia/components/ZimUploader/index.tsx diff --git a/admin/app/controllers/zim_controller.ts b/admin/app/controllers/zim_controller.ts index 0087e76e..aed0136b 100644 --- a/admin/app/controllers/zim_controller.ts +++ b/admin/app/controllers/zim_controller.ts @@ -8,7 +8,12 @@ import { } from '#validators/common' import { addCustomLibraryValidator, browseLibraryValidator, idParamValidator, listRemoteZimValidator } from '#validators/zim' import { inject } from '@adonisjs/core' +import logger from '@adonisjs/core/services/logger' import type { HttpContext } from '@adonisjs/core/http' +import { createWriteStream } from 'fs' +import { rename } from 'fs/promises' +import { join, resolve, sep } from 'path' +import { ZIM_STORAGE_PATH, ensureDirectoryExists, sanitizeFilename } from '../utils/fs.js' @inject() export default class ZimController { @@ -83,6 +88,87 @@ export default class ZimController { } } + async upload({ request, response }: HttpContext) { + let filename: string | null = null + let tmpPath: string | null = null + let uploadError: string | null = null + + try { + const basePath = resolve(join(process.cwd(), ZIM_STORAGE_PATH)) + await ensureDirectoryExists(basePath) + + request.multipart.onFile('*', {}, async (part) => { + const clientName = part.filename || '' + if (!clientName.toLowerCase().endsWith('.zim')) { + part.resume() + uploadError = 'INVALID_TYPE' + return + } + + const sanitized = sanitizeFilename(clientName) + const finalPath = resolve(join(basePath, sanitized)) + + if (!finalPath.startsWith(basePath + sep)) { + part.resume() + uploadError = 'INVALID_FILENAME' + return + } + + const { access } = await import('fs/promises') + const exists = await access(finalPath).then(() => true).catch(() => false) + if (exists) { + part.resume() + uploadError = 'DUPLICATE_FILENAME' + return + } + + filename = sanitized + tmpPath = finalPath + '.tmp' + const ws = createWriteStream(tmpPath) + + await new Promise((res, rej) => { + ws.on('error', rej) + ws.on('finish', res) + part.on('error', rej) + part.pipe(ws) + }) + + await rename(tmpPath, finalPath) + tmpPath = null + }) + + await request.multipart.process() + + if (uploadError === 'INVALID_TYPE') { + return response.status(422).send({ message: 'Only .zim files are accepted' }) + } + if (uploadError === 'INVALID_FILENAME') { + return response.status(422).send({ message: 'Invalid filename' }) + } + if (uploadError === 'DUPLICATE_FILENAME') { + return response.status(409).send({ message: 'A ZIM file with that name already exists' }) + } + if (!filename) { + return response.status(400).send({ message: 'No file received' }) + } + + const { added } = await this.zimService.registerLocalUpload(filename) + + return response.status(201).send({ + message: 'ZIM file uploaded and registered successfully', + filename, + added, + }) + } catch (error) { + logger.error('[ZimController] Upload failed:', error) + if (tmpPath) { + const { unlink } = await import('fs/promises') + await unlink(tmpPath).catch(() => {}) + } + return response.status(500).send({ message: 'Upload failed' }) + } + } + // Wikipedia selector endpoints async getWikipediaState({}: HttpContext) { diff --git a/admin/app/services/zim_service.ts b/admin/app/services/zim_service.ts index 3b0469f5..5a362e07 100644 --- a/admin/app/services/zim_service.ts +++ b/admin/app/services/zim_service.ts @@ -25,6 +25,7 @@ import vine from '@vinejs/vine' import { wikipediaOptionsFileSchema } from '#validators/curated_collections' import WikipediaSelection from '#models/wikipedia_selection' import InstalledResource from '#models/installed_resource' +import CollectionManifest from '#models/collection_manifest' import { RunDownloadJob } from '#jobs/run_download_job' import { SERVICE_NAMES } from '../../constants/service_names.js' import { CollectionManifestService } from './collection_manifest_service.js' @@ -469,6 +470,97 @@ export class ZimService { return { before, after, added: Math.max(0, after - before) } } + async registerLocalUpload(filename: string): Promise<{ added: number }> { + let added = 0 + try { + const result = await this.rescanLibrary() + added = result.added + } catch (err) { + logger.error('[ZimService] Failed to rebuild kiwix library after local upload:', err) + } + + const parsed = CollectionManifestService.parseZimFilename(filename) + if (parsed) { + const filepath = join(process.cwd(), ZIM_STORAGE_PATH, filename) + const stats = await getFileStatsIfExists(filepath) + try { + const { DateTime } = await import('luxon') + await InstalledResource.updateOrCreate( + { resource_id: parsed.resource_id, resource_type: 'zim' }, + { + version: parsed.version, + url: `local-upload://${filename}`, + file_path: filepath, + file_size_bytes: stats ? Number(stats.size) : null, + installed_at: DateTime.now(), + } + ) + } catch (error) { + logger.error(`[ZimService] Failed to create InstalledResource for ${filename}:`, error) + } + } + + // If the uploaded file matches a known Wikipedia option, mark it as installed + try { + const manifest = await CollectionManifest.find('wikipedia') + if (manifest) { + const spec = manifest.spec_data as { options: Array<{ id: string; url: string | null }> } + const matchedOption = spec.options.find( + (opt) => opt.url && opt.url.split('/').pop() === filename + ) + if (matchedOption && matchedOption.url) { + const existing = await WikipediaSelection.query().first() + if (existing) { + existing.option_id = matchedOption.id + existing.url = matchedOption.url + existing.filename = filename + existing.status = 'installed' + await existing.save() + } else { + await WikipediaSelection.create({ + option_id: matchedOption.id, + url: matchedOption.url, + filename, + status: 'installed', + }) + } + logger.info(`[ZimService] Marked Wikipedia option '${matchedOption.id}' as installed from local upload`) + + // Remove any other wikipedia_en_*.zim files, same as the download flow + const allFiles = await this.list() + const staleWikipediaFiles = allFiles.files.filter( + (f) => f.name.startsWith('wikipedia_en_') && f.name !== filename + ) + for (const stale of staleWikipediaFiles) { + try { + await this.delete(stale.name) + logger.info(`[ZimService] Deleted stale Wikipedia file after upload: ${stale.name}`) + } catch (err) { + logger.warn(`[ZimService] Could not delete stale Wikipedia file: ${stale.name}`, err) + } + } + } + } + } catch (error) { + logger.error(`[ZimService] Failed to update WikipediaSelection for ${filename}:`, error) + } + + const ollamaUrl = await this.dockerService.getServiceURL('nomad_ollama') + if (ollamaUrl) { + try { + const { EmbedFileJob } = await import('#jobs/embed_file_job') + await EmbedFileJob.dispatch({ + fileName: filename, + filePath: join(process.cwd(), ZIM_STORAGE_PATH, filename), + }) + } catch (error) { + logger.error(`[ZimService] EmbedFileJob dispatch failed after local upload:`, error) + } + } + + return { added } + } + async delete(file: string): Promise { let fileName = file if (!fileName.endsWith('.zim')) { @@ -505,6 +597,21 @@ export class ZimService { .delete() logger.info(`[ZimService] Deleted InstalledResource entry for: ${parsed.resource_id}`) } + + // If this file was the active Wikipedia selection, clear the selection + try { + const selection = await WikipediaSelection.query().first() + if (selection && selection.filename === fileName) { + selection.option_id = 'none' + selection.status = 'none' + selection.filename = null + selection.url = null + await selection.save() + logger.info(`[ZimService] Cleared WikipediaSelection after deleting ${fileName}`) + } + } catch (error) { + logger.error(`[ZimService] Failed to clear WikipediaSelection after deleting ${fileName}:`, error) + } } // Wikipedia selector methods diff --git a/admin/config/bodyparser.ts b/admin/config/bodyparser.ts index c995755f..2c85bd95 100644 --- a/admin/config/bodyparser.ts +++ b/admin/config/bodyparser.ts @@ -41,7 +41,7 @@ const bodyParserConfig = defineConfig({ */ autoProcess: true, convertEmptyStringsToNull: true, - processManually: [], + processManually: ['/api/zim/upload'], /** * Maximum limit of data to parse including all files diff --git a/admin/inertia/components/ZimUploader/index.tsx b/admin/inertia/components/ZimUploader/index.tsx new file mode 100644 index 00000000..a734c0fb --- /dev/null +++ b/admin/inertia/components/ZimUploader/index.tsx @@ -0,0 +1,77 @@ +import Uppy from '@uppy/core' +import Dashboard from '@uppy/react/dashboard' +import XHRUpload from '@uppy/xhr-upload' +import '@uppy/core/css/style.min.css' +import '@uppy/dashboard/css/style.min.css' +import { useEffect, useRef, useState } from 'react' + +interface ZimUploaderProps { + onUploadComplete: (added: number) => void + existingFilenames: string[] +} + +export default function ZimUploader({ onUploadComplete, existingFilenames }: ZimUploaderProps) { + const existingFilenamesRef = useRef(existingFilenames) + useEffect(() => { + existingFilenamesRef.current = existingFilenames + }, [existingFilenames]) + + const [uppy] = useState(() => + new Uppy({ + restrictions: { + maxNumberOfFiles: 5, + allowedFileTypes: ['.zim'], + }, + autoProceed: false, + }).use(XHRUpload, { + endpoint: '/api/zim/upload', + fieldName: 'file', + getResponseError: (responseText) => { + try { + const body = JSON.parse(responseText) + if (body?.message) return new Error(body.message) + } catch {} + return new Error('Upload failed') + }, + }) + ) + + useEffect(() => { + const handleFileAdded = (file: { id: string; name: string }) => { + if (existingFilenamesRef.current.includes(file.name)) { + uppy.removeFile(file.id) + uppy.info('A ZIM file with that name already exists', 'error', 6000) + return + } + + const isWikipedia = (name: string) => name.startsWith('wikipedia_en_') + if (isWikipedia(file.name)) { + const alreadyQueued = uppy.getFiles().some((f) => f.id !== file.id && isWikipedia(f.name)) + if (alreadyQueued) { + uppy.removeFile(file.id) + uppy.info('Only one Wikipedia file can be uploaded at a time', 'error', 6000) + } + } + } + const handleComplete = (result: { successful: Array<{ response?: { body?: { added?: number } } }> }) => { + const added = result.successful.reduce((sum, f) => sum + (f.response?.body?.added ?? 0), 0) + onUploadComplete(added) + } + uppy.on('file-added', handleFileAdded) + uppy.on('complete', handleComplete) + return () => { + uppy.off('file-added', handleFileAdded) + uppy.off('complete', handleComplete) + uppy.destroy() + } + }, [uppy, onUploadComplete]) + + return ( + + ) +} diff --git a/admin/inertia/pages/settings/zim/index.tsx b/admin/inertia/pages/settings/zim/index.tsx index 2c727dc8..8dacf788 100644 --- a/admin/inertia/pages/settings/zim/index.tsx +++ b/admin/inertia/pages/settings/zim/index.tsx @@ -10,6 +10,7 @@ import StyledModal from '~/components/StyledModal' import useServiceInstalledStatus from '~/hooks/useServiceInstalledStatus' import Alert from '~/components/Alert' import { useNotifications } from '~/context/NotificationContext' +import ZimUploader from '~/components/ZimUploader' import { ZimFileWithMetadata } from '../../../../types/zim' import { SERVICE_NAMES } from '../../../../constants/service_names' import { formatBytes } from '~/lib/util' @@ -25,6 +26,7 @@ export default function ZimPage() { const { isInstalled } = useServiceInstalledStatus(SERVICE_NAMES.KIWIX) const [sortKey, setSortKey] = useState('size') const [sortDirection, setSortDirection] = useState('desc') + const [showUploader, setShowUploader] = useState(false) const { data, isLoading } = useQuery({ queryKey: ['zim-files'], queryFn: getFiles, @@ -135,18 +137,50 @@ export default function ZimPage() { Manage your stored content files.

- {isInstalled && ( +
rescanMutation.mutate()} + icon={showUploader ? 'IconX' : 'IconUpload'} + onClick={() => setShowUploader((v) => !v)} > - Rescan Library + {showUploader ? 'Hide Uploader' : 'Upload ZIM File'} - )} + {isInstalled && ( + rescanMutation.mutate()} + > + Rescan Library + + )} +
+ {showUploader && ( +
+

+ Upload a ZIM file from your browser. Files up to 20 GB are supported. For best results upload from the same machine or over a stable LAN connection. Larger files should be copied directly to the storage volume. +

+ f.name) ?? []} + onUploadComplete={(added) => { + queryClient.invalidateQueries({ queryKey: ['zim-files'] }) + queryClient.invalidateQueries({ queryKey: ['wikipedia-state'] }) + queryClient.invalidateQueries({ queryKey: ['curated-categories'] }) + setShowUploader(false) + addNotification({ + type: 'success', + message: + added > 0 + ? `Upload complete. ${added} new ${added === 1 ? 'book' : 'books'} added to the library.` + : 'Upload complete. Library is up to date.', + }) + }} + /> +
+ )} {!isInstalled && ( ({ + queryKey: [ZIM_FILES_KEY], + queryFn: async () => { + const res = await api.listZimFiles() + return res.data.files + }, + refetchOnWindowFocus: false, + }) + const { data: downloads, invalidate: invalidateDownloads } = useDownloads({ filetype: 'zim', enabled: true, @@ -152,15 +163,16 @@ export default function ZimRemoteExplorer() { const flatData = useMemo(() => { const mapped = data?.pages.flatMap((page) => page.items) || [] - // remove items that are currently downloading + const localNames = new Set(localFiles?.map((f) => f.name) ?? []) return mapped.filter((item) => { const isDownloading = downloads?.some((download) => { const filename = item.download_url.split('/').pop() return filename && download.filepath.endsWith(filename) }) - return !isDownloading + const isPresent = localNames.has(item.file_name) + return !isDownloading && !isPresent }) - }, [data, downloads]) + }, [data, downloads, localFiles]) const hasMore = useMemo(() => data?.pages[data.pages.length - 1]?.has_more || false, [data]) const fetchOnBottomReached = useCallback( diff --git a/admin/package-lock.json b/admin/package-lock.json index 6e3c1a19..c5f7f777 100644 --- a/admin/package-lock.json +++ b/admin/package-lock.json @@ -4588,6 +4588,7 @@ "cpu": [ "arm64" ], + "dev": true, "license": "Apache-2.0 AND MIT", "optional": true, "os": [ @@ -4604,6 +4605,7 @@ "cpu": [ "x64" ], + "dev": true, "license": "Apache-2.0 AND MIT", "optional": true, "os": [ @@ -4620,6 +4622,7 @@ "cpu": [ "arm" ], + "dev": true, "license": "Apache-2.0", "optional": true, "os": [ @@ -4712,6 +4715,7 @@ "cpu": [ "arm64" ], + "dev": true, "license": "Apache-2.0 AND MIT", "optional": true, "os": [ @@ -4728,6 +4732,7 @@ "cpu": [ "ia32" ], + "dev": true, "license": "Apache-2.0 AND MIT", "optional": true, "os": [ @@ -4744,6 +4749,7 @@ "cpu": [ "x64" ], + "dev": true, "license": "Apache-2.0 AND MIT", "optional": true, "os": [ @@ -5683,6 +5689,12 @@ "@types/react": "^19.2.0" } }, + "node_modules/@types/retry": { + "version": "0.12.2", + "resolved": "https://registry.npmjs.org/@types/retry/-/retry-0.12.2.tgz", + "integrity": "sha512-XISRgDJ2Tc5q4TRqvgJtzsRkFYNJzZrhTdtMoGVBttwzzQJkPnS3WWTFc7kuDRoPtPakl+T+OfdEUjYJj7Jbow==", + "license": "MIT" + }, "node_modules/@types/send": { "version": "1.2.1", "resolved": "https://registry.npmjs.org/@types/send/-/send-1.2.1.tgz", @@ -6029,6 +6041,20 @@ "integrity": "sha512-mUFwbeTqrVgDQxFveS+df2yfap6iuP20NAKAsBt5jDEoOTDew+zwLAOilHCeQJOVSvmgCX4ogqIrA0mnyr08yQ==", "license": "ISC" }, + "node_modules/@uppy/companion-client": { + "version": "5.1.1", + "resolved": "https://registry.npmjs.org/@uppy/companion-client/-/companion-client-5.1.1.tgz", + "integrity": "sha512-DzrOWTbIZHvtgAFXBMYHk2wD27NjpBSVhY2tEiEIUhPd2CxbFRZjHM/N3HOt3VwZEAP471QWFLlJRWPcIY3A2Q==", + "license": "MIT", + "dependencies": { + "@uppy/utils": "^7.1.1", + "namespace-emitter": "^2.0.1", + "p-retry": "^6.1.0" + }, + "peerDependencies": { + "@uppy/core": "^5.1.1" + } + }, "node_modules/@uppy/components": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/@uppy/components/-/components-1.2.0.tgz", @@ -6174,6 +6200,19 @@ "preact": "^10.26.10" } }, + "node_modules/@uppy/xhr-upload": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/@uppy/xhr-upload/-/xhr-upload-5.2.0.tgz", + "integrity": "sha512-3LV/X5Of6BINnKplP+CwUJ0a4/7cRFfzxwGyXnW+uCrNQHoo09dttcz3begWHejGvzenQHuUnMO3Fxyc71Pryg==", + "license": "MIT", + "dependencies": { + "@uppy/companion-client": "^5.1.1", + "@uppy/utils": "^7.2.0" + }, + "peerDependencies": { + "@uppy/core": "^5.2.0" + } + }, "node_modules/@vavite/multibuild": { "version": "5.1.0", "resolved": "https://registry.npmjs.org/@vavite/multibuild/-/multibuild-5.1.0.tgz", @@ -10513,6 +10552,18 @@ "url": "https://github.com/sponsors/wooorm" } }, + "node_modules/is-network-error": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/is-network-error/-/is-network-error-1.3.1.tgz", + "integrity": "sha512-6QCxa49rQbmUWLfk0nuGqzql9U8uaV2H6279bRErPBHe/109hCzsLUBUHfbEtvLIHBd6hyXbgedBSHevm43Edw==", + "license": "MIT", + "engines": { + "node": ">=16" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/is-number": { "version": "7.0.0", "resolved": "https://registry.npmjs.org/is-number/-/is-number-7.0.0.tgz", @@ -13343,6 +13394,23 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/p-retry": { + "version": "6.2.1", + "resolved": "https://registry.npmjs.org/p-retry/-/p-retry-6.2.1.tgz", + "integrity": "sha512-hEt02O4hUct5wtwg4H4KcWgDdm+l1bOaEy/hWzd8xtXB9BqxTWBBhb+2ImAtH4Cv4rPjV76xN3Zumqk3k3AhhQ==", + "license": "MIT", + "dependencies": { + "@types/retry": "0.12.2", + "is-network-error": "^1.0.0", + "retry": "^0.13.1" + }, + "engines": { + "node": ">=16.17" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/p-timeout": { "version": "6.1.4", "resolved": "https://registry.npmjs.org/p-timeout/-/p-timeout-6.1.4.tgz", @@ -14826,7 +14894,6 @@ "version": "0.13.1", "resolved": "https://registry.npmjs.org/retry/-/retry-0.13.1.tgz", "integrity": "sha512-XQBQ3I8W1Cge0Seh+6gjj03LbmRFWuoszgK9ooCpwYIrhhoO80pfq4cUkU5DkknwfOfFteRwlZ56PYOGYyFWdg==", - "devOptional": true, "license": "MIT", "engines": { "node": ">= 4" diff --git a/admin/package.json b/admin/package.json index bb0146be..49b4a5ce 100644 --- a/admin/package.json +++ b/admin/package.json @@ -88,6 +88,7 @@ "@uppy/core": "5.2.0", "@uppy/dashboard": "5.1.0", "@uppy/react": "5.1.1", + "@uppy/xhr-upload": "5.2.0", "@vinejs/vine": "3.0.1", "@vitejs/plugin-react": "4.7.0", "autoprefixer": "10.5.0", diff --git a/admin/start/routes.ts b/admin/start/routes.ts index 5553cbc2..f3c52dec 100644 --- a/admin/start/routes.ts +++ b/admin/start/routes.ts @@ -205,6 +205,7 @@ router router.post('/download-remote', [ZimController, 'downloadRemote']) router.post('/download-category-tier', [ZimController, 'downloadCategoryTier']) + router.post('/upload', [ZimController, 'upload']) router.get('/wikipedia', [ZimController, 'getWikipediaState']) router.post('/wikipedia/select', [ZimController, 'selectWikipedia'])