feat(zim): add zim uploader in content manager

admin/app/controllers/zim_controller.ts

@@ -8,7 +8,12 @@ import {
 } from '#validators/common'
 import { addCustomLibraryValidator, browseLibraryValidator, idParamValidator, listRemoteZimValidator } from '#validators/zim'
 import { inject } from '@adonisjs/core'
+import logger from '@adonisjs/core/services/logger'
 import type { HttpContext } from '@adonisjs/core/http'
+import { createWriteStream } from 'fs'
+import { rename } from 'fs/promises'
+import { join, resolve, sep } from 'path'
+import { ZIM_STORAGE_PATH, ensureDirectoryExists, sanitizeFilename } from '../utils/fs.js'
 
 @inject()
 export default class ZimController {
@@ -83,6 +88,87 @@ export default class ZimController {
     }
   }
 
+  async upload({ request, response }: HttpContext) {
+    let filename: string | null = null
+    let tmpPath: string | null = null
+    let uploadError: string | null = null
+
+    try {
+      const basePath = resolve(join(process.cwd(), ZIM_STORAGE_PATH))
+      await ensureDirectoryExists(basePath)
+
+      request.multipart.onFile('*', {}, async (part) => {
+        const clientName = part.filename || ''
+        if (!clientName.toLowerCase().endsWith('.zim')) {
+          part.resume()
+          uploadError = 'INVALID_TYPE'
+          return
+        }
+
+        const sanitized = sanitizeFilename(clientName)
+        const finalPath = resolve(join(basePath, sanitized))
+
+        if (!finalPath.startsWith(basePath + sep)) {
+          part.resume()
+          uploadError = 'INVALID_FILENAME'
+          return
+        }
+
+        const { access } = await import('fs/promises')
+        const exists = await access(finalPath).then(() => true).catch(() => false)
+        if (exists) {
+          part.resume()
+          uploadError = 'DUPLICATE_FILENAME'
+          return
+        }
+
+        filename = sanitized
+        tmpPath = finalPath + '.tmp'
+        const ws = createWriteStream(tmpPath)
+
+        await new Promise<void>((res, rej) => {
+          ws.on('error', rej)
+          ws.on('finish', res)
+          part.on('error', rej)
+          part.pipe(ws)
+        })
+
+        await rename(tmpPath, finalPath)
+        tmpPath = null
+      })
+
+      await request.multipart.process()
+
+      if (uploadError === 'INVALID_TYPE') {
+        return response.status(422).send({ message: 'Only .zim files are accepted' })
+      }
+      if (uploadError === 'INVALID_FILENAME') {
+        return response.status(422).send({ message: 'Invalid filename' })
+      }
+      if (uploadError === 'DUPLICATE_FILENAME') {
+        return response.status(409).send({ message: 'A ZIM file with that name already exists' })
+      }
+      if (!filename) {
+        return response.status(400).send({ message: 'No file received' })
+      }
+
+      const { added } = await this.zimService.registerLocalUpload(filename)
+
+      return response.status(201).send({
+        message: 'ZIM file uploaded and registered successfully',
+        filename,
+        added,
+      })
+    } catch (error) {
+      logger.error('[ZimController] Upload failed:', error)
+      if (tmpPath) {
+        const { unlink } = await import('fs/promises')
+        await unlink(tmpPath).catch(() => {})
+      }
+      return response.status(500).send({ message: 'Upload failed' })
+    }
+  }
+
   // Wikipedia selector endpoints
 
   async getWikipediaState({}: HttpContext) {

admin/app/services/zim_service.ts

@@ -25,6 +25,7 @@ import vine from '@vinejs/vine'
 import { wikipediaOptionsFileSchema } from '#validators/curated_collections'
 import WikipediaSelection from '#models/wikipedia_selection'
 import InstalledResource from '#models/installed_resource'
+import CollectionManifest from '#models/collection_manifest'
 import { RunDownloadJob } from '#jobs/run_download_job'
 import { SERVICE_NAMES } from '../../constants/service_names.js'
 import { CollectionManifestService } from './collection_manifest_service.js'
@@ -469,6 +470,97 @@ export class ZimService {
     return { before, after, added: Math.max(0, after - before) }
   }
 
+  async registerLocalUpload(filename: string): Promise<{ added: number }> {
+    let added = 0
+    try {
+      const result = await this.rescanLibrary()
+      added = result.added
+    } catch (err) {
+      logger.error('[ZimService] Failed to rebuild kiwix library after local upload:', err)
+    }
+
+    const parsed = CollectionManifestService.parseZimFilename(filename)
+    if (parsed) {
+      const filepath = join(process.cwd(), ZIM_STORAGE_PATH, filename)
+      const stats = await getFileStatsIfExists(filepath)
+      try {
+        const { DateTime } = await import('luxon')
+        await InstalledResource.updateOrCreate(
+          { resource_id: parsed.resource_id, resource_type: 'zim' },
+          {
+            version: parsed.version,
+            url: `local-upload://${filename}`,
+            file_path: filepath,
+            file_size_bytes: stats ? Number(stats.size) : null,
+            installed_at: DateTime.now(),
+          }
+        )
+      } catch (error) {
+        logger.error(`[ZimService] Failed to create InstalledResource for ${filename}:`, error)
+      }
+    }
+
+    // If the uploaded file matches a known Wikipedia option, mark it as installed
+    try {
+      const manifest = await CollectionManifest.find('wikipedia')
+      if (manifest) {
+        const spec = manifest.spec_data as { options: Array<{ id: string; url: string | null }> }
+        const matchedOption = spec.options.find(
+          (opt) => opt.url && opt.url.split('/').pop() === filename
+        )
+        if (matchedOption && matchedOption.url) {
+          const existing = await WikipediaSelection.query().first()
+          if (existing) {
+            existing.option_id = matchedOption.id
+            existing.url = matchedOption.url
+            existing.filename = filename
+            existing.status = 'installed'
+            await existing.save()
+          } else {
+            await WikipediaSelection.create({
+              option_id: matchedOption.id,
+              url: matchedOption.url,
+              filename,
+              status: 'installed',
+            })
+          }
+          logger.info(`[ZimService] Marked Wikipedia option '${matchedOption.id}' as installed from local upload`)
+
+          // Remove any other wikipedia_en_*.zim files, same as the download flow
+          const allFiles = await this.list()
+          const staleWikipediaFiles = allFiles.files.filter(
+            (f) => f.name.startsWith('wikipedia_en_') && f.name !== filename
+          )
+          for (const stale of staleWikipediaFiles) {
+            try {
+              await this.delete(stale.name)
+              logger.info(`[ZimService] Deleted stale Wikipedia file after upload: ${stale.name}`)
+            } catch (err) {
+              logger.warn(`[ZimService] Could not delete stale Wikipedia file: ${stale.name}`, err)
+            }
+          }
+        }
+      }
+    } catch (error) {
+      logger.error(`[ZimService] Failed to update WikipediaSelection for ${filename}:`, error)
+    }
+
+    const ollamaUrl = await this.dockerService.getServiceURL('nomad_ollama')
+    if (ollamaUrl) {
+      try {
+        const { EmbedFileJob } = await import('#jobs/embed_file_job')
+        await EmbedFileJob.dispatch({
+          fileName: filename,
+          filePath: join(process.cwd(), ZIM_STORAGE_PATH, filename),
+        })
+      } catch (error) {
+        logger.error(`[ZimService] EmbedFileJob dispatch failed after local upload:`, error)
+      }
+    }
+
+    return { added }
+  }
+
   async delete(file: string): Promise<void> {
     let fileName = file
     if (!fileName.endsWith('.zim')) {
@@ -505,6 +597,21 @@ export class ZimService {
         .delete()
       logger.info(`[ZimService] Deleted InstalledResource entry for: ${parsed.resource_id}`)
     }
+
+    // If this file was the active Wikipedia selection, clear the selection
+    try {
+      const selection = await WikipediaSelection.query().first()
+      if (selection && selection.filename === fileName) {
+        selection.option_id = 'none'
+        selection.status = 'none'
+        selection.filename = null
+        selection.url = null
+        await selection.save()
+        logger.info(`[ZimService] Cleared WikipediaSelection after deleting ${fileName}`)
+      }
+    } catch (error) {
+      logger.error(`[ZimService] Failed to clear WikipediaSelection after deleting ${fileName}:`, error)
+    }
   }
 
   // Wikipedia selector methods

admin/config/bodyparser.ts

@@ -41,7 +41,7 @@ const bodyParserConfig = defineConfig({
      */
     autoProcess: true,
     convertEmptyStringsToNull: true,
-    processManually: [],
+    processManually: ['/api/zim/upload'],
 
     /**
      * Maximum limit of data to parse including all files

admin/inertia/components/ZimUploader/index.tsx

@@ -0,0 +1,77 @@
+import Uppy from '@uppy/core'
+import Dashboard from '@uppy/react/dashboard'
+import XHRUpload from '@uppy/xhr-upload'
+import '@uppy/core/css/style.min.css'
+import '@uppy/dashboard/css/style.min.css'
+import { useEffect, useRef, useState } from 'react'
+
+interface ZimUploaderProps {
+  onUploadComplete: (added: number) => void
+  existingFilenames: string[]
+}
+
+export default function ZimUploader({ onUploadComplete, existingFilenames }: ZimUploaderProps) {
+  const existingFilenamesRef = useRef(existingFilenames)
+  useEffect(() => {
+    existingFilenamesRef.current = existingFilenames
+  }, [existingFilenames])
+
+  const [uppy] = useState(() =>
+    new Uppy({
+      restrictions: {
+        maxNumberOfFiles: 5,
+        allowedFileTypes: ['.zim'],
+      },
+      autoProceed: false,
+    }).use(XHRUpload, {
+      endpoint: '/api/zim/upload',
+      fieldName: 'file',
+      getResponseError: (responseText) => {
+        try {
+          const body = JSON.parse(responseText)
+          if (body?.message) return new Error(body.message)
+        } catch {}
+        return new Error('Upload failed')
+      },
+    })
+  )
+
+  useEffect(() => {
+    const handleFileAdded = (file: { id: string; name: string }) => {
+      if (existingFilenamesRef.current.includes(file.name)) {
+        uppy.removeFile(file.id)
+        uppy.info('A ZIM file with that name already exists', 'error', 6000)
+        return
+      }
+
+      const isWikipedia = (name: string) => name.startsWith('wikipedia_en_')
+      if (isWikipedia(file.name)) {
+        const alreadyQueued = uppy.getFiles().some((f) => f.id !== file.id && isWikipedia(f.name))
+        if (alreadyQueued) {
+          uppy.removeFile(file.id)
+          uppy.info('Only one Wikipedia file can be uploaded at a time', 'error', 6000)
+        }
+      }
+    }
+    const handleComplete = (result: { successful: Array<{ response?: { body?: { added?: number } } }> }) => {
+      const added = result.successful.reduce((sum, f) => sum + (f.response?.body?.added ?? 0), 0)
+      onUploadComplete(added)
+    }
+    uppy.on('file-added', handleFileAdded)
+    uppy.on('complete', handleComplete)
+    return () => {
+      uppy.off('file-added', handleFileAdded)
+      uppy.off('complete', handleComplete)
+      uppy.destroy()
+    }
+  }, [uppy, onUploadComplete])
+
+  return (
+    <Dashboard
+      uppy={uppy}
+      width="100%"
+      height={300}
+      note="ZIM files only. Large files (up to 20 GB) are supported. For best results, upload from the same machine or over a stable LAN connection. Larger files should be copied directly to the storage volume"
+    />
+  )
+}

admin/inertia/pages/settings/zim/index.tsx

@@ -10,6 +10,7 @@ import StyledModal from '~/components/StyledModal'
 import useServiceInstalledStatus from '~/hooks/useServiceInstalledStatus'
 import Alert from '~/components/Alert'
 import { useNotifications } from '~/context/NotificationContext'
+import ZimUploader from '~/components/ZimUploader'
 import { ZimFileWithMetadata } from '../../../../types/zim'
 import { SERVICE_NAMES } from '../../../../constants/service_names'
 import { formatBytes } from '~/lib/util'
@@ -25,6 +26,7 @@ export default function ZimPage() {
   const { isInstalled } = useServiceInstalledStatus(SERVICE_NAMES.KIWIX)
   const [sortKey, setSortKey] = useState<SortKey>('size')
   const [sortDirection, setSortDirection] = useState<SortDirection>('desc')
+  const [showUploader, setShowUploader] = useState(false)
   const { data, isLoading } = useQuery<ZimFileWithMetadata[]>({
     queryKey: ['zim-files'],
     queryFn: getFiles,
@@ -135,18 +137,50 @@ export default function ZimPage() {
                 Manage your stored content files.
               </p>
             </div>
-            {isInstalled && (
+            <div className="flex items-center gap-2">
               <StyledButton
                 variant="secondary"
-                icon={'IconRefresh'}
-                loading={rescanMutation.isPending}
-                title="Rebuild the Kiwix library index from the files on disk. Use this after manually adding ZIM files outside of NOMAD."
-                onClick={() => rescanMutation.mutate()}
+                icon={showUploader ? 'IconX' : 'IconUpload'}
+                onClick={() => setShowUploader((v) => !v)}
               >
-                Rescan Library
+                {showUploader ? 'Hide Uploader' : 'Upload ZIM File'}
               </StyledButton>
-            )}
+              {isInstalled && (
+                <StyledButton
+                  variant="secondary"
+                  icon={'IconRefresh'}
+                  loading={rescanMutation.isPending}
+                  title="Rebuild the Kiwix library index from the files on disk. Use this after manually adding ZIM files outside of NOMAD."
+                  onClick={() => rescanMutation.mutate()}
+                >
+                  Rescan Library
+                </StyledButton>
+              )}
+            </div>
           </div>
+          {showUploader && (
+            <div className="mt-6">
+              <p className="text-text-muted text-sm mb-3">
+                Upload a ZIM file from your browser. Files up to 20 GB are supported. For best results upload from the same machine or over a stable LAN connection. Larger files should be copied directly to the storage volume.
+              </p>
+              <ZimUploader
+                existingFilenames={data?.map((f) => f.name) ?? []}
+                onUploadComplete={(added) => {
+                  queryClient.invalidateQueries({ queryKey: ['zim-files'] })
+                  queryClient.invalidateQueries({ queryKey: ['wikipedia-state'] })
+                  queryClient.invalidateQueries({ queryKey: ['curated-categories'] })
+                  setShowUploader(false)
+                  addNotification({
+                    type: 'success',
+                    message:
+                      added > 0
+                        ? `Upload complete. ${added} new ${added === 1 ? 'book' : 'books'} added to the library.`
+                        : 'Upload complete. Library is up to date.',
+                  })
+                }}
+              />
+            </div>
+          )}
           {!isInstalled && (
             <Alert
               title="The Kiwix application is not installed. Please install it to view downloaded ZIM files"