fix: update the existing file (#58)

Author: vpbs2

meerkat-dbm/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@devrev/meerkat-dbm",
-  "version": "0.0.15",
+  "version": "0.0.151",
   "dependencies": {
     "tslib": "^2.3.0",
     "@duckdb/duckdb-wasm": "^1.28.0",

meerkat-dbm/src/file-manager/file-manager-type.ts

@@ -130,6 +130,7 @@ export interface FileManagerType {
 export type BaseFileStore = {
   tableName: string;
   fileName: string;
+  partitions?: string[];
   staleTime?: number;
   cacheTime?: number;
   metadata?: object;

meerkat-dbm/src/utils/__tests__/merge-file-buffer-store-into-table.spec.ts

@@ -81,4 +81,24 @@ describe('mergeFileBufferStoreIntoTable', () => {
       files: [{ fileName: 'taxi1.parquet' }],
     });
   });
+
+  it('should update the file if already exists', () => {
+    const currentTableState = [
+      {
+        tableName: fileBufferStore.tableName,
+        files: [{ fileName: fileBufferStore.fileName }],
+      },
+    ];
+
+    const updatedTableMap = mergeFileBufferStoreIntoTable(
+      [{ ...fileBufferStore, partitions: ['dev_oid=DEV-787'] }],
+      currentTableState
+    );
+
+    // Verify that the taxi1 table still has one file (taxi1.parquet)
+    expect(updatedTableMap.get('taxi1')).toEqual({
+      tableName: 'taxi1',
+      files: [{ fileName: 'taxi1.parquet', partitions: ['dev_oid=DEV-787'] }],
+    });
+  });
 });

meerkat-dbm/src/utils/merge-file-buffer-store-into-table.ts

@@ -36,11 +36,18 @@ export const mergeFileBufferStoreIntoTable = (
      * Check if the table already exists in the map if it does, then update the existing table entry
      */
     if (existingTable) {
-      const fileExists = existingTable.files.some(
+      const existingFileIndex = existingTable.files.findIndex(
         (file) => file.fileName === fileBuffer.fileName
       );
 
-      if (!fileExists) {
+      if (existingFileIndex !== -1) {
+        // If file exists, update the fileData
+        existingTable.files[existingFileIndex] = {
+          ...existingTable.files[existingFileIndex],
+          ...fileData,
+        };
+      } else {
+        // If file does not exist, add it to the files array
         existingTable.files.push(fileData);
       }
     } else {