Refactor file organization

server/modules/file/repository/fileRepositoryS3BucketCommon.js

@@ -1,4 +1,5 @@
 import {
+  CopyObjectCommand,
   DeleteObjectCommand,
   GetObjectCommand,
   HeadBucketCommand,
@@ -39,6 +40,33 @@ export const checkCanAccessS3Bucket = async () => {
 
 const createCommandParams = ({ getFileKey, params }) => ({ Bucket, Key: getFileKey(params) })
 
+export const checkFileExistsInS3 = async ({ key }) => {
+  try {
+    const command = new HeadObjectCommand({ Bucket, Key: key })
+    await _sendCommand(command)
+    return true
+  } catch (error) {
+    if (error.$metadata?.httpStatusCode === 404) {
+      return false
+    }
+    throw error
+  }
+}
+
+export const copyObjectInS3 = async ({ sourceKey, destinationKey }) => {
+  const command = new CopyObjectCommand({
+    Bucket,
+    CopySource: `${Bucket}/${sourceKey}`,
+    Key: destinationKey,
+  })
+  return _sendCommand(command)
+}
+
+export const deleteObjectFromS3 = async ({ key }) => {
+  const command = new DeleteObjectCommand({ Bucket, Key: key })
+  return _sendCommand(command)
+}
+
 export const createS3BucketRepository = ({ getFileKey }) => {
   const uploadFileContent = async ({ content, ...params }) => {
     const command = new PutObjectCommand({

server/modules/record/repository/fileRepository.js

@@ -65,6 +65,16 @@ export const fetchFileUuidsBySurveyId = async ({ surveyId }, client = db) =>
     (row) => row.uuid
   )
 
+export const fetchFileSummariesByUuids = async ({ surveyId, fileUuids }, client = db) => {
+  if (fileUuids.length === 0) return []
+  return client.manyOrNone(
+    `SELECT ${SUMMARY_FIELDS_COMMA_SEPARATED}
+    FROM ${Schemata.getSchemaSurvey(surveyId)}.file
+    WHERE uuid IN ($1:csv)`,
+    [fileUuids]
+  )
+}
+
 export const fetchFileUuidsOfFilesWithContent = async ({ surveyId }, client = db) =>
   client.map(
     `

server/modules/record/repository/fileRepositoryFileSystem.js

@@ -6,10 +6,16 @@ export const getStorageFolderPath = () => ProcessUtils.ENV.fileStoragePath
 export const getSurveyFilesStorageFolderPath = ({ surveyId }) =>
   FileUtils.join(ProcessUtils.ENV.fileStoragePath, surveyId)
 
-const getFilePath = ({ surveyId, fileUuid }) => {
-  const storageFolderPath = getSurveyFilesStorageFolderPath({ surveyId })
-  return FileUtils.join(storageFolderPath, fileUuid)
-}
+const getSubfolder = ({ recordUuid }) => (recordUuid ? 'record_files' : 'survey_files')
+
+const getSubfolderPath = ({ surveyId, recordUuid }) =>
+  FileUtils.join(ProcessUtils.ENV.fileStoragePath, surveyId, getSubfolder({ recordUuid }))
+
+const getFilePath = ({ surveyId, fileUuid, recordUuid = null }) =>
+  FileUtils.join(getSubfolderPath({ surveyId, recordUuid }), fileUuid)
+
+const getLegacyFilePath = ({ surveyId, fileUuid }) =>
+  FileUtils.join(ProcessUtils.ENV.fileStoragePath, surveyId, fileUuid)
 
 export const checkCanAccessStorageFolder = async () => {
   const storageFolderPath = getStorageFolderPath()
@@ -22,24 +28,36 @@ export const checkCanAccessStorageFolder = async () => {
   throw new Error('Cannot access files storage path: ' + getStorageFolderPath())
 }
 
-export const writeFileContent = async ({ surveyId, fileUuid, content }) => {
-  const storageFolderPath = getSurveyFilesStorageFolderPath({ surveyId })
-  await FileUtils.mkdir(storageFolderPath)
-  const filePath = getFilePath({ surveyId, fileUuid })
+export const writeFileContent = async ({ surveyId, fileUuid, content, recordUuid = null }) => {
+  const subfolderPath = getSubfolderPath({ surveyId, recordUuid })
+  await FileUtils.mkdir(subfolderPath)
+  const filePath = getFilePath({ surveyId, fileUuid, recordUuid })
   await FileUtils.writeFile(filePath, content)
 }
 
-export const getFileContentAsStream = ({ surveyId, fileUuid }) => {
-  const filePath = getFilePath({ surveyId, fileUuid })
+export const getFileContentAsStream = ({ surveyId, fileUuid, recordUuid = null }) => {
+  const filePath = getFilePath({ surveyId, fileUuid, recordUuid })
   if (!FileUtils.exists(filePath)) {
     throw new Error(`File not found: ${filePath}`)
   }
   return FileUtils.createReadStream(filePath)
 }
 
-export const deleteFiles = async ({ surveyId, fileUuids }) => {
-  for (const fileUuid of fileUuids) {
-    const filePath = getFilePath({ surveyId, fileUuid })
+export const deleteFiles = async ({ surveyId, files }) => {
+  for (const { fileUuid, recordUuid } of files) {
+    const filePath = getFilePath({ surveyId, fileUuid, recordUuid })
     await FileUtils.deleteFileAsync(filePath)
   }
 }
+
+export const migrateFileToNewPath = async ({ surveyId, fileUuid, recordUuid }) => {
+  const legacyPath = getLegacyFilePath({ surveyId, fileUuid })
+  if (!FileUtils.exists(legacyPath)) {
+    return false
+  }
+  const subfolderPath = getSubfolderPath({ surveyId, recordUuid })
+  await FileUtils.mkdir(subfolderPath)
+  const newPath = getFilePath({ surveyId, fileUuid, recordUuid })
+  await FileUtils.rename(legacyPath, newPath)
+  return true
+}

server/modules/record/repository/fileRepositoryS3Bucket.js

@@ -1,9 +1,37 @@
-import { createS3BucketRepository } from '@server/modules/file/repository/fileRepositoryS3BucketCommon'
+import {
+  checkFileExistsInS3,
+  copyObjectInS3,
+  createS3BucketRepository,
+  deleteObjectFromS3,
+} from '@server/modules/file/repository/fileRepositoryS3BucketCommon'
 
 export { checkCanAccessS3Bucket } from '@server/modules/file/repository/fileRepositoryS3BucketCommon'
 
-const getFileKey = ({ surveyId, fileUuid }) => `${surveyId}_${fileUuid}`
+const getSubfolder = ({ recordUuid }) => (recordUuid ? 'record_files' : 'survey_files')
 
-const { uploadFileContent, getFileContentAsStream, deleteFiles } = createS3BucketRepository({ getFileKey })
+const getFileKey = ({ surveyId, fileUuid, recordUuid = null }) =>
+  `${surveyId}/${getSubfolder({ recordUuid })}/${fileUuid}`
+
+const getLegacyFileKey = ({ surveyId, fileUuid }) => `${surveyId}_${fileUuid}`
+
+const { uploadFileContent, getFileContentAsStream, deleteFile } = createS3BucketRepository({ getFileKey })
+
+const deleteFiles = async ({ surveyId, files }) => {
+  for (const { fileUuid, recordUuid } of files) {
+    await deleteFile({ surveyId, fileUuid, recordUuid })
+  }
+}
+
+export const migrateFileToNewKey = async ({ surveyId, fileUuid, recordUuid }) => {
+  const legacyKey = getLegacyFileKey({ surveyId, fileUuid })
+  const exists = await checkFileExistsInS3({ key: legacyKey })
+  if (!exists) {
+    return false
+  }
+  const newKey = getFileKey({ surveyId, fileUuid, recordUuid })
+  await copyObjectInS3({ sourceKey: legacyKey, destinationKey: newKey })
+  await deleteObjectFromS3({ key: legacyKey })
+  return true
+}
 
 export { uploadFileContent, getFileContentAsStream, deleteFiles }

server/modules/survey/manager/surveyFileManager.js

@@ -47,7 +47,12 @@ export const fetchFileContentAsStream = async ({ surveyId, fileUuid }, client =
   const storageType = getFileContentStorageType()
   const fetchFn = contentAsStreamFetchFunctionByStorageType[storageType]
   if (fetchFn) {
-    return fetchFn({ surveyId, fileUuid }, client)
+    let recordUuid = null
+    if (storageType !== fileContentStorageTypes.db) {
+      const fileSummary = await FileRepository.fetchFileSummaryByUuid(surveyId, fileUuid, client)
+      recordUuid = SurveyFile.getRecordUuid(fileSummary)
+    }
+    return fetchFn({ surveyId, fileUuid, recordUuid }, client)
   }
   return null
 }
@@ -69,7 +74,8 @@ export const insertFile = async (surveyId, file, client = db) => {
   if (contentStoreFunction) {
     const fileUuid = SurveyFile.getUuid(file)
     const content = SurveyFile.getContent(file)
-    await contentStoreFunction({ surveyId, fileUuid, content })
+    const recordUuid = SurveyFile.getRecordUuid(file)
+    await contentStoreFunction({ surveyId, fileUuid, content, recordUuid })
     // clear content in file object so it won't be stored into DB
     file.content = null
   }
@@ -101,7 +107,8 @@ export const moveFilesToNewStorageIfNecessary = async ({ surveyId }, client = db
 
       const contentStoreFunction = contentStoreFunctionByStorageType[storageType]
       const content = SurveyFile.getContent(file)
-      await contentStoreFunction({ surveyId, fileUuid, content })
+      const recordUuid = SurveyFile.getRecordUuid(file)
+      await contentStoreFunction({ surveyId, fileUuid, content, recordUuid })
     }
     logger.debug(`Files moved from DB; clearing 'content' column in DB 'file' table`)
     await FileRepository.clearAllSurveyFilesContent({ surveyId }, tx)
@@ -123,7 +130,12 @@ export const deleteFilesContentByUuids = async ({ surveyId, fileUuids }) => {
   const storageType = getFileContentStorageType()
   const deleteFn = contentDeleteFunctionByStorageType[storageType]
   if (deleteFn) {
-    await deleteFn({ surveyId, fileUuids })
+    const fileSummaries = await FileRepository.fetchFileSummariesByUuids({ surveyId, fileUuids })
+    const files = fileSummaries.map((s) => ({
+      fileUuid: SurveyFile.getUuid(s),
+      recordUuid: SurveyFile.getRecordUuid(s),
+    }))
+    await deleteFn({ surveyId, files })
   }
 }
 
@@ -132,6 +144,38 @@ export const deleteFilesAndContentByUuids = async ({ surveyId, fileUuids }, clie
   await FileRepository.deleteFilesByUuids(surveyId, fileUuids, client)
 }
 
+export const migrateFilesToNewPathFormat = async ({ surveyId }) => {
+  const storageType = getFileContentStorageType()
+  if (storageType === fileContentStorageTypes.db) {
+    return false
+  }
+
+  const fileSummaries = await FileRepository.fetchFileSummariesBySurveyId(surveyId)
+  if (fileSummaries.length === 0) {
+    return false
+  }
+
+  let migratedCount = 0
+  for (const fileSummary of fileSummaries) {
+    const fileUuid = SurveyFile.getUuid(fileSummary)
+    const recordUuid = SurveyFile.getRecordUuid(fileSummary)
+    let migrated = false
+    if (storageType === fileContentStorageTypes.fileSystem) {
+      migrated = await FileRepositoryFileSystem.migrateFileToNewPath({ surveyId, fileUuid, recordUuid })
+    } else if (storageType === fileContentStorageTypes.s3Bucket) {
+      migrated = await FileRepositoryS3Bucket.migrateFileToNewKey({ surveyId, fileUuid, recordUuid })
+    }
+    if (migrated) {
+      migratedCount++
+    }
+  }
+
+  if (migratedCount > 0) {
+    logger.debug(`Survey ${surveyId}: migrated ${migratedCount} files to new path format`)
+  }
+  return migratedCount > 0
+}
+
 export const deleteTemporaryFiles = async (surveyId, client = db) => {
   const fileSummaries = await FileRepository.fetchFileSummariesBySurveyId(surveyId, client)
   const temporaryFileUuids = []

server/modules/survey/service/surveyFileService.js

@@ -46,6 +46,28 @@ export const fetchFilesStatistics = async ({ surveyId }) => {
   return { availableSpace, totalSpace, usedSpace }
 }
 
+export const migrateAllSurveysFilesToNewPathFormat = async ({ logger }) => {
+  const surveyIds = await SurveyRepository.fetchAllSurveyIds()
+  let allMigrated = false
+  let errorsFound = false
+  for (const surveyId of surveyIds) {
+    try {
+      const migrated = await SurveyFileManager.migrateFilesToNewPathFormat({ surveyId })
+      allMigrated = allMigrated || migrated
+    } catch (error) {
+      errorsFound = true
+      logger.error(`Error migrating file paths for survey ${surveyId}`, error)
+    }
+  }
+  if (errorsFound) {
+    logger.error('There were errors migrating survey files to new path format')
+  } else if (allMigrated) {
+    logger.debug('Survey files migrated to new path format successfully')
+  } else {
+    logger.debug('Survey files path migration not necessary')
+  }
+}
+
 export const cleanupAllSurveysFilesProps = async () => {
   const surveyIds = await SurveyRepository.fetchAllSurveyIds()
   let count = 0

server/system/dataMigrator/index.js

@@ -1,12 +1,16 @@
 import { ServiceType, Versions } from '@openforis/arena-core'
 
 import * as CategoryService from '@server/modules/category/service/categoryService'
+import * as SurveyFileService from '@server/modules/survey/service/surveyFileService'
 
 const versionWithCategoryItemIndexFix = '2.3.20'
+const versionWithNewFilePathFormat = '2.5.6'
 
 const isCategoryItemsMigrationNeeded = (versionInDb) =>
   Versions.isLessThan(versionInDb, versionWithCategoryItemIndexFix)
 
+const isFilePathMigrationNeeded = (versionInDb) => Versions.isLessThan(versionInDb, versionWithNewFilePathFormat)
+
 const migrateCategoryItemsIfNeeded = async ({ logger, versionInDb }) => {
   if (isCategoryItemsMigrationNeeded(versionInDb)) {
     await CategoryService.initializeAllSurveysCategoryItemIndexes()
@@ -15,10 +19,19 @@ const migrateCategoryItemsIfNeeded = async ({ logger, versionInDb }) => {
   }
 }
 
+const migrateFilePathsIfNeeded = async ({ logger, versionInDb }) => {
+  if (isFilePathMigrationNeeded(versionInDb)) {
+    await SurveyFileService.migrateAllSurveysFilesToNewPathFormat({ logger })
+  } else {
+    logger.info(`File path format already migrated. App version in DB: ${versionInDb}`)
+  }
+}
+
 const migrateData = async ({ logger, serviceRegistry }) => {
   const infoService = serviceRegistry.getService(ServiceType.info)
   const versionInDb = await infoService.getVersion()
   await migrateCategoryItemsIfNeeded({ logger, versionInDb })
+  await migrateFilePathsIfNeeded({ logger, versionInDb })
 }
 
 export const DataMigrator = {

server/utils/file/fileUtils.js

@@ -104,6 +104,7 @@ export const getBaseName = (file) => {
 
 export const deleteFile = (path) => fs.unlinkSync(path)
 export const deleteFileAsync = (path) => fsp.unlink(path)
+export const rename = (oldPath, newPath) => fsp.rename(oldPath, newPath)
 
 // ======= Temp Files
 export const newTempFileName = () => `${uuidv4()}.tmp`