Refactor file organization

server/modules/dataExport/service/dataExportJob/jobs/FilesExportJob.js

@@ -51,7 +51,7 @@ export default class FilesExportJob extends Job {
       this.logWarn(`File with UUID ${fileUuid} not found`)
       return false
     }
-    const recordFileContent = await SurveyFileService.fetchFileContentAsStream({ surveyId, fileUuid }, this.tx)
+    const recordFileContent = await SurveyFileService.fetchFileContentAsStream({ surveyId, fileSummary }, this.tx)
     if (!recordFileContent) {
       this.logWarn(`File content for file with UUID ${fileUuid} not found`)
       return false

server/modules/file/repository/fileRepositoryS3BucketCommon.js

@@ -1,4 +1,5 @@
 import {
+  CopyObjectCommand,
   DeleteObjectCommand,
   GetObjectCommand,
   HeadBucketCommand,
@@ -39,6 +40,33 @@ export const checkCanAccessS3Bucket = async () => {
 
 const createCommandParams = ({ getFileKey, params }) => ({ Bucket, Key: getFileKey(params) })
 
+export const checkFileExistsInS3 = async ({ key }) => {
+  try {
+    const command = new HeadObjectCommand({ Bucket, Key: key })
+    await _sendCommand(command)
+    return true
+  } catch (error) {
+    if (error.$metadata?.httpStatusCode === 404) {
+      return false
+    }
+    throw error
+  }
+}
+
+export const copyObjectInS3 = async ({ sourceKey, destinationKey }) => {
+  const command = new CopyObjectCommand({
+    Bucket,
+    CopySource: `${Bucket}/${sourceKey}`,
+    Key: destinationKey,
+  })
+  return _sendCommand(command)
+}
+
+export const deleteObjectFromS3 = async ({ key }) => {
+  const command = new DeleteObjectCommand({ Bucket, Key: key })
+  return _sendCommand(command)
+}
+
 export const createS3BucketRepository = ({ getFileKey }) => {
   const uploadFileContent = async ({ content, ...params }) => {
     const command = new PutObjectCommand({

server/modules/record/api/recordApi.js

@@ -42,7 +42,7 @@ const fetchRecordNodeFileAsStream = async ({ surveyId, nodeUuid }) => {
   const fileUuid = Node.getFileUuid(node)
   const file = await SurveyFileService.fetchFileSummaryByUuid(surveyId, fileUuid)
   const fileName = await RecordService.generateNodeFileNameForDownload({ surveyId, nodeUuid, file })
-  const contentStream = await SurveyFileService.fetchFileContentAsStream({ surveyId, fileUuid })
+  const contentStream = await SurveyFileService.fetchFileContentAsStream({ surveyId, fileSummary: file })
   return { fileName, file, contentStream }
 }
 

server/modules/record/repository/fileRepository.js

@@ -65,6 +65,16 @@ export const fetchFileUuidsBySurveyId = async ({ surveyId }, client = db) =>
     (row) => row.uuid
   )
 
+export const fetchFileSummariesByUuids = async ({ surveyId, fileUuids }, client = db) => {
+  if (fileUuids.length === 0) return []
+  return client.manyOrNone(
+    `SELECT ${SUMMARY_FIELDS_COMMA_SEPARATED}
+    FROM ${Schemata.getSchemaSurvey(surveyId)}.file
+    WHERE uuid IN ($1:csv)`,
+    [fileUuids]
+  )
+}
+
 export const fetchFileUuidsOfFilesWithContent = async ({ surveyId }, client = db) =>
   client.map(
     `

server/modules/record/repository/fileRepositoryFileSystem.js

@@ -6,10 +6,16 @@ export const getStorageFolderPath = () => ProcessUtils.ENV.fileStoragePath
 export const getSurveyFilesStorageFolderPath = ({ surveyId }) =>
   FileUtils.join(ProcessUtils.ENV.fileStoragePath, surveyId)
 
-const getFilePath = ({ surveyId, fileUuid }) => {
-  const storageFolderPath = getSurveyFilesStorageFolderPath({ surveyId })
-  return FileUtils.join(storageFolderPath, fileUuid)
-}
+const getSubfolder = ({ recordUuid }) => (recordUuid ? 'record_files' : 'survey_files')
+
+const getSubfolderPath = ({ surveyId, recordUuid }) =>
+  FileUtils.join(ProcessUtils.ENV.fileStoragePath, surveyId, getSubfolder({ recordUuid }))
+
+const getFilePath = ({ surveyId, fileUuid, recordUuid = null }) =>
+  FileUtils.join(getSubfolderPath({ surveyId, recordUuid }), fileUuid)
+
+const getLegacyFilePath = ({ surveyId, fileUuid }) =>
+  FileUtils.join(ProcessUtils.ENV.fileStoragePath, surveyId, fileUuid)
 
 export const checkCanAccessStorageFolder = async () => {
   const storageFolderPath = getStorageFolderPath()
@@ -22,24 +28,36 @@ export const checkCanAccessStorageFolder = async () => {
   throw new Error('Cannot access files storage path: ' + getStorageFolderPath())
 }
 
-export const writeFileContent = async ({ surveyId, fileUuid, content }) => {
-  const storageFolderPath = getSurveyFilesStorageFolderPath({ surveyId })
-  await FileUtils.mkdir(storageFolderPath)
-  const filePath = getFilePath({ surveyId, fileUuid })
+export const writeFileContent = async ({ surveyId, fileUuid, content, recordUuid = null }) => {
+  const subfolderPath = getSubfolderPath({ surveyId, recordUuid })
+  await FileUtils.mkdir(subfolderPath)
+  const filePath = getFilePath({ surveyId, fileUuid, recordUuid })
   await FileUtils.writeFile(filePath, content)
 }
 
-export const getFileContentAsStream = ({ surveyId, fileUuid }) => {
-  const filePath = getFilePath({ surveyId, fileUuid })
+export const getFileContentAsStream = ({ surveyId, fileUuid, recordUuid = null }) => {
+  const filePath = getFilePath({ surveyId, fileUuid, recordUuid })
   if (!FileUtils.exists(filePath)) {
     throw new Error(`File not found: ${filePath}`)
   }
   return FileUtils.createReadStream(filePath)
 }
 
-export const deleteFiles = async ({ surveyId, fileUuids }) => {
-  for (const fileUuid of fileUuids) {
-    const filePath = getFilePath({ surveyId, fileUuid })
+export const deleteFiles = async ({ surveyId, files }) => {
+  for (const { fileUuid, recordUuid } of files) {
+    const filePath = getFilePath({ surveyId, fileUuid, recordUuid })
     await FileUtils.deleteFileAsync(filePath)
   }
 }
+
+export const migrateFileToNewPath = async ({ surveyId, fileUuid, recordUuid }) => {
+  const legacyPath = getLegacyFilePath({ surveyId, fileUuid })
+  if (!FileUtils.exists(legacyPath)) {
+    return false
+  }
+  const subfolderPath = getSubfolderPath({ surveyId, recordUuid })
+  await FileUtils.mkdir(subfolderPath)
+  const newPath = getFilePath({ surveyId, fileUuid, recordUuid })
+  await FileUtils.rename(legacyPath, newPath)
+  return true
+}

server/modules/record/repository/fileRepositoryS3Bucket.js

@@ -1,9 +1,37 @@
-import { createS3BucketRepository } from '@server/modules/file/repository/fileRepositoryS3BucketCommon'
+import {
+  checkFileExistsInS3,
+  copyObjectInS3,
+  createS3BucketRepository,
+  deleteObjectFromS3,
+} from '@server/modules/file/repository/fileRepositoryS3BucketCommon'
 
 export { checkCanAccessS3Bucket } from '@server/modules/file/repository/fileRepositoryS3BucketCommon'
 
-const getFileKey = ({ surveyId, fileUuid }) => `${surveyId}_${fileUuid}`
+const getSubfolder = ({ recordUuid }) => (recordUuid ? 'record_files' : 'survey_files')
 
-const { uploadFileContent, getFileContentAsStream, deleteFiles } = createS3BucketRepository({ getFileKey })
+const getFileKey = ({ surveyId, fileUuid, recordUuid = null }) =>
+  `${surveyId}/${getSubfolder({ recordUuid })}/${fileUuid}`
+
+const getLegacyFileKey = ({ surveyId, fileUuid }) => `${surveyId}_${fileUuid}`
+
+const { uploadFileContent, getFileContentAsStream, deleteFile } = createS3BucketRepository({ getFileKey })
+
+const deleteFiles = async ({ surveyId, files }) => {
+  for (const { fileUuid, recordUuid } of files) {
+    await deleteFile({ surveyId, fileUuid, recordUuid })
+  }
+}
+
+export const migrateFileToNewKey = async ({ surveyId, fileUuid, recordUuid }) => {
+  const legacyKey = getLegacyFileKey({ surveyId, fileUuid })
+  const exists = await checkFileExistsInS3({ key: legacyKey })
+  if (!exists) {
+    return false
+  }
+  const newKey = getFileKey({ surveyId, fileUuid, recordUuid })
+  await copyObjectInS3({ sourceKey: legacyKey, destinationKey: newKey })
+  await deleteObjectFromS3({ key: legacyKey })
+  return true
+}
 
 export { uploadFileContent, getFileContentAsStream, deleteFiles }

server/modules/record/service/recordService.js

@@ -500,7 +500,10 @@ const exportRecordDocument = async ({
     record,
     lang: langToUse,
     i18n,
-    fileProvider: async (fileUuid) => SurveyFileService.fetchFileContentAsBuffer({ surveyId, fileUuid }),
+    fileProvider: async (fileUuid) => {
+      const fileSummary = await SurveyFileService.fetchFileSummaryByUuid(surveyId, fileUuid)
+      return SurveyFileService.fetchFileContentAsBuffer({ surveyId, fileSummary })
+    },
     readOnly: true,
   })
   const fileName = ExportFileNameGenerator.generate({ surveyName, cycle, fileType: 'RecordForm', extension })

server/modules/record/service/recordsCloneJob.js

@@ -122,7 +122,7 @@ export default class RecordsCloneJob extends Job {
     for (const [fileUuid, newFileUuid] of Object.entries(newFileUuidsByOldUuid)) {
       const fileSummary = await SurveyFileService.fetchFileSummaryByUuid(surveyId, fileUuid, tx)
       if (fileSummary) {
-        const content = await SurveyFileService.fetchFileContentAsBuffer({ surveyId, fileUuid }, tx)
+        const content = await SurveyFileService.fetchFileContentAsBuffer({ surveyId, fileSummary }, tx)
         const oldNodeUuid = SurveyFile.getNodeUuid(fileSummary)
         const newNodeUuid = oldNodeUuid ? newNodeUuidsByOldUuid[oldNodeUuid] : null
         const newFile = SurveyFile.createFile({

server/modules/survey/manager/surveyFileManager.js

@@ -43,17 +43,19 @@ const contentDeleteFunctionByStorageType = {
   [fileContentStorageTypes.s3Bucket]: FileRepositoryS3Bucket.deleteFiles,
 }
 
-export const fetchFileContentAsStream = async ({ surveyId, fileUuid }, client = db) => {
+export const fetchFileContentAsStream = async ({ surveyId, fileSummary }, client = db) => {
   const storageType = getFileContentStorageType()
   const fetchFn = contentAsStreamFetchFunctionByStorageType[storageType]
   if (fetchFn) {
-    return fetchFn({ surveyId, fileUuid }, client)
+    const fileUuid = SurveyFile.getUuid(fileSummary)
+    const recordUuid = SurveyFile.getRecordUuid(fileSummary)
+    return fetchFn({ surveyId, fileUuid, recordUuid }, client)
   }
   return null
 }
 
-export const fetchFileContentAsBuffer = async ({ surveyId, fileUuid }, client = db) => {
-  const contentStream = await fetchFileContentAsStream({ surveyId, fileUuid }, client)
+export const fetchFileContentAsBuffer = async ({ surveyId, fileSummary }, client = db) => {
+  const contentStream = await fetchFileContentAsStream({ surveyId, fileSummary }, client)
   return StreamUtils.readStreamToBuffer(contentStream)
 }
 
@@ -69,7 +71,8 @@ export const insertFile = async (surveyId, file, client = db) => {
   if (contentStoreFunction) {
     const fileUuid = SurveyFile.getUuid(file)
     const content = SurveyFile.getContent(file)
-    await contentStoreFunction({ surveyId, fileUuid, content })
+    const recordUuid = SurveyFile.getRecordUuid(file)
+    await contentStoreFunction({ surveyId, fileUuid, content, recordUuid })
     // clear content in file object so it won't be stored into DB
     file.content = null
   }
@@ -101,7 +104,8 @@ export const moveFilesToNewStorageIfNecessary = async ({ surveyId }, client = db
 
       const contentStoreFunction = contentStoreFunctionByStorageType[storageType]
       const content = SurveyFile.getContent(file)
-      await contentStoreFunction({ surveyId, fileUuid, content })
+      const recordUuid = SurveyFile.getRecordUuid(file)
+      await contentStoreFunction({ surveyId, fileUuid, content, recordUuid })
     }
     logger.debug(`Files moved from DB; clearing 'content' column in DB 'file' table`)
     await FileRepository.clearAllSurveyFilesContent({ surveyId }, tx)
@@ -119,31 +123,62 @@ export const deleteFileByUuid = async ({ surveyId, fileUuid }, client = db) => {
   // do not delete content if not in DB: deletion out of transaction
 }
 
-export const deleteFilesContentByUuids = async ({ surveyId, fileUuids }) => {
+export const deleteFilesContentByUuids = async ({ surveyId, fileSummaries }) => {
   const storageType = getFileContentStorageType()
   const deleteFn = contentDeleteFunctionByStorageType[storageType]
   if (deleteFn) {
-    await deleteFn({ surveyId, fileUuids })
+    const files = fileSummaries.map((s) => ({
+      fileUuid: SurveyFile.getUuid(s),
+      recordUuid: SurveyFile.getRecordUuid(s),
+    }))
+    await deleteFn({ surveyId, files })
   }
 }
 
-export const deleteFilesAndContentByUuids = async ({ surveyId, fileUuids }, client = db) => {
-  await deleteFilesContentByUuids({ surveyId, fileUuids })
+export const deleteFilesAndContent = async ({ surveyId, fileSummaries }, client = db) => {
+  await deleteFilesContentByUuids({ surveyId, fileSummaries })
+  const fileUuids = fileSummaries.map(SurveyFile.getUuid)
   await FileRepository.deleteFilesByUuids(surveyId, fileUuids, client)
 }
 
-export const deleteTemporaryFiles = async (surveyId, client = db) => {
-  const fileSummaries = await FileRepository.fetchFileSummariesBySurveyId(surveyId, client)
-  const temporaryFileUuids = []
+export const migrateFilesToNewPathFormat = async ({ surveyId }) => {
+  const storageType = getFileContentStorageType()
+  if (storageType === fileContentStorageTypes.db) {
+    return false
+  }
+
+  const fileSummaries = await FileRepository.fetchFileSummariesBySurveyId(surveyId)
+  if (fileSummaries.length === 0) {
+    return false
+  }
+
+  let migratedCount = 0
   for (const fileSummary of fileSummaries) {
     const fileUuid = SurveyFile.getUuid(fileSummary)
-    if (SurveyFile.isTemporary(fileSummary)) {
-      temporaryFileUuids.push(fileUuid)
+    const recordUuid = SurveyFile.getRecordUuid(fileSummary)
+    let migrated = false
+    if (storageType === fileContentStorageTypes.fileSystem) {
+      migrated = await FileRepositoryFileSystem.migrateFileToNewPath({ surveyId, fileUuid, recordUuid })
+    } else if (storageType === fileContentStorageTypes.s3Bucket) {
+      migrated = await FileRepositoryS3Bucket.migrateFileToNewKey({ surveyId, fileUuid, recordUuid })
+    }
+    if (migrated) {
+      migratedCount++
     }
   }
-  if (temporaryFileUuids.length > 0) {
-    logger.debug(`Deleting ${temporaryFileUuids.length} temporary files of survey ${surveyId}`)
-    await deleteFilesAndContentByUuids({ surveyId, fileUuids: temporaryFileUuids }, client)
+
+  if (migratedCount > 0) {
+    logger.debug(`Survey ${surveyId}: migrated ${migratedCount} files to new path format`)
+  }
+  return migratedCount > 0
+}
+
+export const deleteTemporaryFiles = async (surveyId, client = db) => {
+  const fileSummaries = await FileRepository.fetchFileSummariesBySurveyId(surveyId, client)
+  const temporaryFileSummaries = fileSummaries.filter(SurveyFile.isTemporary)
+  if (temporaryFileSummaries.length > 0) {
+    logger.debug(`Deleting ${temporaryFileSummaries.length} temporary files of survey ${surveyId}`)
+    await deleteFilesAndContent({ surveyId, fileSummaries: temporaryFileSummaries }, client)
   }
 }
 

server/modules/survey/manager/surveyManager.js

@@ -458,10 +458,14 @@ export const deleteUnusedSurveyFiles = async (surveyId, client = db) => {
   const preloadedMapLayerFileSummariesToDelete = preloadedMapLayerFileSummaries.filter(
     (fileSummary) => !preloadedMapLayerFileUuids.has(SurveyFile.getUuid(fileSummary))
   )
-  const fileUuidsToDelete = preloadedMapLayerFileSummariesToDelete.map(SurveyFile.getUuid)
-  if (fileUuidsToDelete.length > 0) {
-    await SurveyFileManager.deleteFilesAndContentByUuids({ surveyId, fileUuids: fileUuidsToDelete }, client)
-    Logger.debug(`Deleted ${fileUuidsToDelete.length} unused preloaded map layer files of survey ${surveyId}`)
+  if (preloadedMapLayerFileSummariesToDelete.length > 0) {
+    await SurveyFileManager.deleteFilesAndContent(
+      { surveyId, fileSummaries: preloadedMapLayerFileSummariesToDelete },
+      client
+    )
+    Logger.debug(
+      `Deleted ${preloadedMapLayerFileSummariesToDelete.length} unused preloaded map layer files of survey ${surveyId}`
+    )
   }
 }
 
@@ -510,10 +514,10 @@ export const { removeSurveyTemporaryFlag, updateSurveyDependencyGraphs } = Surve
 
 // ====== DELETE
 export const deleteSurvey = async (surveyId, { deleteUserPrefs = true } = {}, client = db) => {
-  // fetch file uuids to delete before survey schema is dropped
-  const filesToDeleteUuids = SurveyFileManager.isFileContentStoredInDB()
+  // fetch file summaries to delete before survey schema is dropped
+  const filesToDelete = SurveyFileManager.isFileContentStoredInDB()
     ? []
-    : await SurveyFileManager.fetchFileUuidsBySurveyId({ surveyId }, client)
+    : await SurveyFileManager.fetchFileSummariesBySurveyId(surveyId, client)
 
   await client.tx(async (t) => {
     if (deleteUserPrefs) {
@@ -523,8 +527,8 @@ export const deleteSurvey = async (surveyId, { deleteUserPrefs = true } = {}, cl
     await SurveyRepository.dropSurveySchema(surveyId, t)
     await SchemaRdbRepository.dropSchema(surveyId, t)
   })
-  if (filesToDeleteUuids.length > 0) {
-    await SurveyFileManager.deleteFilesContentByUuids({ surveyId, fileUuids: filesToDeleteUuids })
+  if (filesToDelete.length > 0) {
+    await SurveyFileManager.deleteFilesContentByUuids({ surveyId, fileSummaries: filesToDelete })
   }
 }
 

server/modules/survey/service/surveyExport/jobs/recordFilesExportJob.js

@@ -37,7 +37,7 @@ export default class RecordFilesExportJob extends Job {
           break
         }
         const fileUuid = SurveyFile.getUuid(fileSummary)
-        const fileContentStream = await SurveyFileService.fetchFileContentAsStream({ surveyId, fileUuid }, this.tx)
+        const fileContentStream = await SurveyFileService.fetchFileContentAsStream({ surveyId, fileSummary }, this.tx)
         archive.append(fileContentStream, { name: ExportFile.file({ fileUuid }) })
         this.incrementProcessedItems()
       }

server/modules/survey/service/surveyExport/jobs/surveyFilesExportJob.js

@@ -29,7 +29,7 @@ export default class SurveyFilesExportJob extends Job {
           break
         }
         const fileUuid = SurveyFile.getUuid(fileSummary)
-        const fileContentStream = await SurveyFileService.fetchFileContentAsStream({ surveyId, fileUuid }, this.tx)
+        const fileContentStream = await SurveyFileService.fetchFileContentAsStream({ surveyId, fileSummary }, this.tx)
         const archiveEntryName = ExportFile.surveyFile({ fileUuid })
         archive.append(fileContentStream, { name: archiveEntryName })