diff --git a/.erb/configs/webpack.paths.ts b/.erb/configs/webpack.paths.ts index e98415cfd..d11a65fe7 100644 --- a/.erb/configs/webpack.paths.ts +++ b/.erb/configs/webpack.paths.ts @@ -14,7 +14,7 @@ const distRendererPath = path.join(distPath, 'renderer'); const buildPath = path.join(rootPath, 'build'); -export const nativeDeps = ['@packages/addon']; +export const nativeDeps = ['@packages/addon', 'lmdb']; export default { rootPath, diff --git a/.github/workflows/e2e-tests.yml b/.github/workflows/e2e-tests.yml index c43d45aca..691d71490 100644 --- a/.github/workflows/e2e-tests.yml +++ b/.github/workflows/e2e-tests.yml @@ -6,7 +6,8 @@ on: jobs: e2e_tests: - runs-on: windows-latest + runs-on: windows-2022 + timeout-minutes: 10 steps: - name: Checkout code diff --git a/package-lock.json b/package-lock.json index 8cf1a30dd..b9cc37417 100644 --- a/package-lock.json +++ b/package-lock.json @@ -28,8 +28,10 @@ "electron-store": "^8.0.1", "framer-motion": "^5.6.0", "jwt-decode": "^3.1.2", + "lmdb": "^3.5.4", "openapi-fetch": "^0.13.4", "openpgp": "5.11.3", + "p-limit": "^7.3.0", "phosphor-react": "^1.4.1", "react": "^17.0.2", "react-circular-progressbar": "^2.1.0", @@ -3523,6 +3525,12 @@ "integrity": "sha512-aGTxbpbg8/b5JfU1HXSrbH3wXZuLPJcNEcZQFMxLs3oSzgtVu6nFPkbbGGUvBcUjKV2YyB9Wxxabo+HEH9tcRQ==", "license": "MIT" }, + "node_modules/@harperfast/extended-iterable": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/@harperfast/extended-iterable/-/extended-iterable-1.0.3.tgz", + "integrity": "sha512-sSAYhQca3rDWtQUHSAPeO7axFIUJOI6hn1gjRC5APVE1a90tuyT8f5WIgRsFhhWA7htNkju2veB9eWL6YHi/Lw==", + "license": "Apache-2.0" + }, "node_modules/@headlessui/react": { "version": "1.7.19", "resolved": "https://registry.npmjs.org/@headlessui/react/-/react-1.7.19.tgz", @@ -3606,35 +3614,21 @@ "node_modules/@internxt/drive-desktop-core": { "version": "0.1.13", "resolved": "file:packages/core/internxt-drive-desktop-core-0.1.13.tgz", - "integrity": "sha512-KPHveO7n5nMIER2ACIN9Y3l9rbx3mlD6jflj/hIOUI8xLeHfuNdDoI54XB2E0g9Sc5dflAcikuUplSrP11dbcA==", + "integrity": "sha512-B04f3pOlIy8dPl4bd4hV3aw3WASnCSWmgrWfaRaL0d7jGIz0caFU8v2TyVHjuDBxKHrWzXcVyOBE2ms2VNUMPA==", "license": "MIT", "dependencies": { - "@internxt/sdk": "^1.11.10", + "@internxt/sdk": "^1.16.2", "@phosphor-icons/react": "2.0.9", "@tanstack/react-virtual": "^3.13.12", "check-disk-space": "^3.4.0", "electron-log": "^5.4.1", - "react": "^17.0.2", - "uuid": "^13.0.0" + "react": "^17.0.2" }, "engines": { "node": ">=18.20.8", "npm": ">=10.0.0" } }, - "node_modules/@internxt/drive-desktop-core/node_modules/uuid": { - "version": "13.0.0", - "resolved": "https://registry.npmjs.org/uuid/-/uuid-13.0.0.tgz", - "integrity": "sha512-XQegIaBTVUjSHliKqcnFqYypAd4S+WCYt5NIeRs6w/UAry7z8Y9j5ZwRRL4kzq9U3sD6v+85er9FvkEaBpji2w==", - "funding": [ - "https://github.com/sponsors/broofa", - "https://github.com/sponsors/ctavan" - ], - "license": "MIT", - "bin": { - "uuid": "dist-node/bin/uuid" - } - }, "node_modules/@internxt/eslint-config-internxt": { "version": "1.0.9", "resolved": "https://registry.npmjs.org/@internxt/eslint-config-internxt/-/eslint-config-internxt-1.0.9.tgz", @@ -4085,6 +4079,97 @@ "dev": true, "license": "MIT" }, + "node_modules/@lmdb/lmdb-darwin-arm64": { + "version": "3.5.4", + "resolved": "https://registry.npmjs.org/@lmdb/lmdb-darwin-arm64/-/lmdb-darwin-arm64-3.5.4.tgz", + "integrity": "sha512-Kk4Kz3iyu1QiLsLZBS9Af1eSKUC8VR2T+/jyE2iAyuGw2VwK08pp5iTbZnXn6sWu0LogO/RFktMxOjiDA2sS3w==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ] + }, + "node_modules/@lmdb/lmdb-darwin-x64": { + "version": "3.5.4", + "resolved": "https://registry.npmjs.org/@lmdb/lmdb-darwin-x64/-/lmdb-darwin-x64-3.5.4.tgz", + "integrity": "sha512-BEe5Rp3trn26oxoXOVL5HVDoiYmjUDwr8NRPkBOdUdCSBEorKI+7JrZLRKAdxO+G6cGQLgseXk0gR7qIQa7aGw==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ] + }, + "node_modules/@lmdb/lmdb-linux-arm": { + "version": "3.5.4", + "resolved": "https://registry.npmjs.org/@lmdb/lmdb-linux-arm/-/lmdb-linux-arm-3.5.4.tgz", + "integrity": "sha512-SGbFR7816uBcTHc2ZY4S6WyOkl9bICnzqTQd2Mv4V/j24cfds88xx2nC6cm/y8zGQL7Ds31YF/5NGxjgcdM5Hw==", + "cpu": [ + "arm" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@lmdb/lmdb-linux-arm64": { + "version": "3.5.4", + "resolved": "https://registry.npmjs.org/@lmdb/lmdb-linux-arm64/-/lmdb-linux-arm64-3.5.4.tgz", + "integrity": "sha512-cUXEengO8o60v1SWerJTH4/RH4U3+9jC0/4njp2Z9NdmvaGzhKsbRM2wpXuRYrN8tytsoJCg0SvWEWwHAwLbCA==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@lmdb/lmdb-linux-x64": { + "version": "3.5.4", + "resolved": "https://registry.npmjs.org/@lmdb/lmdb-linux-x64/-/lmdb-linux-x64-3.5.4.tgz", + "integrity": "sha512-Gxq8jpgOWXwd0PUR+c9R2Ik1/uBnGd5GMIIzRRDqABCkvmjtC3KWcyhesV9jSPCz759isl0NlbsstZ2oyvk8lA==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@lmdb/lmdb-win32-arm64": { + "version": "3.5.4", + "resolved": "https://registry.npmjs.org/@lmdb/lmdb-win32-arm64/-/lmdb-win32-arm64-3.5.4.tgz", + "integrity": "sha512-pKv1DJ1bPZAaHkdFsSz5IDfUG8x9vntgquXF9/Dm2xuupcIe/EkLzylpoBxppFVK5vzbV561Dq26jNY2fIMA7g==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "win32" + ] + }, + "node_modules/@lmdb/lmdb-win32-x64": { + "version": "3.5.4", + "resolved": "https://registry.npmjs.org/@lmdb/lmdb-win32-x64/-/lmdb-win32-x64-3.5.4.tgz", + "integrity": "sha512-JF1BmLCm9kGEVZgYmJq43zeQVdHVgAJnTi/NURWEsy6L1ZrrlSmdltS+D17QN4LODwf+1LMXAA9auIZVXtWwzw==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "win32" + ] + }, "node_modules/@malept/cross-spawn-promise": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/@malept/cross-spawn-promise/-/cross-spawn-promise-2.0.0.tgz", @@ -4163,6 +4248,84 @@ "node": ">= 10.0.0" } }, + "node_modules/@msgpackr-extract/msgpackr-extract-darwin-arm64": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-darwin-arm64/-/msgpackr-extract-darwin-arm64-3.0.3.tgz", + "integrity": "sha512-QZHtlVgbAdy2zAqNA9Gu1UpIuI8Xvsd1v8ic6B2pZmeFnFcMWiPLfWXh7TVw4eGEZ/C9TH281KwhVoeQUKbyjw==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ] + }, + "node_modules/@msgpackr-extract/msgpackr-extract-darwin-x64": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-darwin-x64/-/msgpackr-extract-darwin-x64-3.0.3.tgz", + "integrity": "sha512-mdzd3AVzYKuUmiWOQ8GNhl64/IoFGol569zNRdkLReh6LRLHOXxU4U8eq0JwaD8iFHdVGqSy4IjFL4reoWCDFw==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ] + }, + "node_modules/@msgpackr-extract/msgpackr-extract-linux-arm": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-linux-arm/-/msgpackr-extract-linux-arm-3.0.3.tgz", + "integrity": "sha512-fg0uy/dG/nZEXfYilKoRe7yALaNmHoYeIoJuJ7KJ+YyU2bvY8vPv27f7UKhGRpY6euFYqEVhxCFZgAUNQBM3nw==", + "cpu": [ + "arm" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@msgpackr-extract/msgpackr-extract-linux-arm64": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-linux-arm64/-/msgpackr-extract-linux-arm64-3.0.3.tgz", + "integrity": "sha512-YxQL+ax0XqBJDZiKimS2XQaf+2wDGVa1enVRGzEvLLVFeqa5kx2bWbtcSXgsxjQB7nRqqIGFIcLteF/sHeVtQg==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@msgpackr-extract/msgpackr-extract-linux-x64": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-linux-x64/-/msgpackr-extract-linux-x64-3.0.3.tgz", + "integrity": "sha512-cvwNfbP07pKUfq1uH+S6KJ7dT9K8WOE4ZiAcsrSes+UY55E/0jLYc+vq+DO7jlmqRb5zAggExKm0H7O/CBaesg==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@msgpackr-extract/msgpackr-extract-win32-x64": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-win32-x64/-/msgpackr-extract-win32-x64-3.0.3.tgz", + "integrity": "sha512-x0fWaQtYp4E6sktbsdAqnehxDgEc/VwM7uLsRCYWaiGu0ykYdZPiS8zCWdnjHwyiumousxfBm4SO31eXqwEZhQ==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "win32" + ] + }, "node_modules/@napi-rs/wasm-runtime": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/@napi-rs/wasm-runtime/-/wasm-runtime-1.1.1.tgz", @@ -10653,7 +10816,6 @@ "version": "2.1.2", "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-2.1.2.tgz", "integrity": "sha512-Btj2BOOO83o3WyH59e8MgXsxEQVcarkUOpEYrubB0urwnN10yQ364rsiByU11nZlqWYZm05i/of7io4mzihBtQ==", - "dev": true, "license": "Apache-2.0", "engines": { "node": ">=8" @@ -10703,6 +10865,35 @@ "node": "*" } }, + "node_modules/dir-compare/node_modules/p-limit": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-3.1.0.tgz", + "integrity": "sha512-TYOanM3wGwNGsZN2cVTYPArw454xnXj5qmWF1bEoAc4+cU/ol7GVh7odevjp1FNHduHc3KZMcFduxU5Xc6uJRQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "yocto-queue": "^0.1.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/dir-compare/node_modules/yocto-queue": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/yocto-queue/-/yocto-queue-0.1.0.tgz", + "integrity": "sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/dir-glob": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/dir-glob/-/dir-glob-3.0.1.tgz", @@ -15902,6 +16093,39 @@ "dev": true, "license": "MIT" }, + "node_modules/lmdb": { + "version": "3.5.4", + "resolved": "https://registry.npmjs.org/lmdb/-/lmdb-3.5.4.tgz", + "integrity": "sha512-9FKQA6G1MMtqNxfxvSBNXD/axeG2QRjYbNh0/ykRL5xYcRbCm2vXq7B9bhc7nSuKdHzr8/BHIwfPuYYH1UsXXw==", + "hasInstallScript": true, + "license": "MIT", + "dependencies": { + "@harperfast/extended-iterable": "^1.0.3", + "msgpackr": "^1.11.2", + "node-addon-api": "^6.1.0", + "node-gyp-build-optional-packages": "5.2.2", + "ordered-binary": "^1.5.3", + "weak-lru-cache": "^1.2.2" + }, + "bin": { + "download-lmdb-prebuilds": "bin/download-prebuilds.js" + }, + "optionalDependencies": { + "@lmdb/lmdb-darwin-arm64": "3.5.4", + "@lmdb/lmdb-darwin-x64": "3.5.4", + "@lmdb/lmdb-linux-arm": "3.5.4", + "@lmdb/lmdb-linux-arm64": "3.5.4", + "@lmdb/lmdb-linux-x64": "3.5.4", + "@lmdb/lmdb-win32-arm64": "3.5.4", + "@lmdb/lmdb-win32-x64": "3.5.4" + } + }, + "node_modules/lmdb/node_modules/node-addon-api": { + "version": "6.1.0", + "resolved": "https://registry.npmjs.org/node-addon-api/-/node-addon-api-6.1.0.tgz", + "integrity": "sha512-+eawOlIgy680F0kBzPUNFhMZGtJ1YmqM6l4+Crf4IkImjYrO/mqPwRMh352g23uIaQKFItcQ64I7KMaJxHgAVA==", + "license": "MIT" + }, "node_modules/loader-runner": { "version": "4.3.1", "resolved": "https://registry.npmjs.org/loader-runner/-/loader-runner-4.3.1.tgz", @@ -16602,6 +16826,37 @@ "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", "license": "MIT" }, + "node_modules/msgpackr": { + "version": "1.11.12", + "resolved": "https://registry.npmjs.org/msgpackr/-/msgpackr-1.11.12.tgz", + "integrity": "sha512-RBdJ1Un7yGlXWajrkxcSa93nvQ0w4zBf60c0yYv7YtBelP8H2FA7XsfBbMHtXKXUMUxH7zV3Zuozh+kUQWhHvg==", + "license": "MIT", + "optionalDependencies": { + "msgpackr-extract": "^3.0.2" + } + }, + "node_modules/msgpackr-extract": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/msgpackr-extract/-/msgpackr-extract-3.0.3.tgz", + "integrity": "sha512-P0efT1C9jIdVRefqjzOQ9Xml57zpOXnIuS+csaB4MdZbTdmGDLo8XhzBG1N7aO11gKDDkJvBLULeFTo46wwreA==", + "hasInstallScript": true, + "license": "MIT", + "optional": true, + "dependencies": { + "node-gyp-build-optional-packages": "5.2.2" + }, + "bin": { + "download-msgpackr-prebuilds": "bin/download-prebuilds.js" + }, + "optionalDependencies": { + "@msgpackr-extract/msgpackr-extract-darwin-arm64": "3.0.3", + "@msgpackr-extract/msgpackr-extract-darwin-x64": "3.0.3", + "@msgpackr-extract/msgpackr-extract-linux-arm": "3.0.3", + "@msgpackr-extract/msgpackr-extract-linux-arm64": "3.0.3", + "@msgpackr-extract/msgpackr-extract-linux-x64": "3.0.3", + "@msgpackr-extract/msgpackr-extract-win32-x64": "3.0.3" + } + }, "node_modules/multicast-dns": { "version": "7.2.5", "resolved": "https://registry.npmjs.org/multicast-dns/-/multicast-dns-7.2.5.tgz", @@ -16720,6 +16975,20 @@ "node": ">=10" } }, + "node_modules/node-gyp-build-optional-packages": { + "version": "5.2.2", + "resolved": "https://registry.npmjs.org/node-gyp-build-optional-packages/-/node-gyp-build-optional-packages-5.2.2.tgz", + "integrity": "sha512-s+w+rBWnpTMwSFbaE0UXsRlg7hU4FjekKU4eyAih5T8nJuNZT1nNsskXpxmeqSK9UzkBl6UgRlnKc8hz8IEqOw==", + "license": "MIT", + "dependencies": { + "detect-libc": "^2.0.1" + }, + "bin": { + "node-gyp-build-optional-packages": "bin.js", + "node-gyp-build-optional-packages-optional": "optional.js", + "node-gyp-build-optional-packages-test": "build-test.js" + } + }, "node_modules/node-loader": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/node-loader/-/node-loader-2.1.0.tgz", @@ -17338,6 +17607,12 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/ordered-binary": { + "version": "1.6.1", + "resolved": "https://registry.npmjs.org/ordered-binary/-/ordered-binary-1.6.1.tgz", + "integrity": "sha512-QkCdPooczexPLiXIrbVOPYkR3VO3T6v2OyKRkR1Xbhpy7/LAVXwahnRCgRp78Oe/Ehf0C/HATAxfSr6eA1oX+w==", + "license": "MIT" + }, "node_modules/os-homedir": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/os-homedir/-/os-homedir-1.0.2.tgz", @@ -17417,16 +17692,15 @@ } }, "node_modules/p-limit": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-3.1.0.tgz", - "integrity": "sha512-TYOanM3wGwNGsZN2cVTYPArw454xnXj5qmWF1bEoAc4+cU/ol7GVh7odevjp1FNHduHc3KZMcFduxU5Xc6uJRQ==", - "dev": true, + "version": "7.3.0", + "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-7.3.0.tgz", + "integrity": "sha512-7cIXg/Z0M5WZRblrsOla88S4wAK+zOQQWeBYfV3qJuJXMr+LnbYjaadrFaS0JILfEDPVqHyKnZ1Z/1d6J9VVUw==", "license": "MIT", "dependencies": { - "yocto-queue": "^0.1.0" + "yocto-queue": "^1.2.1" }, "engines": { - "node": ">=10" + "node": ">=20" }, "funding": { "url": "https://github.com/sponsors/sindresorhus" @@ -17448,6 +17722,35 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/p-locate/node_modules/p-limit": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-3.1.0.tgz", + "integrity": "sha512-TYOanM3wGwNGsZN2cVTYPArw454xnXj5qmWF1bEoAc4+cU/ol7GVh7odevjp1FNHduHc3KZMcFduxU5Xc6uJRQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "yocto-queue": "^0.1.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/p-locate/node_modules/yocto-queue": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/yocto-queue/-/yocto-queue-0.1.0.tgz", + "integrity": "sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/p-map": { "version": "7.0.4", "resolved": "https://registry.npmjs.org/p-map/-/p-map-7.0.4.tgz", @@ -24093,6 +24396,12 @@ "defaults": "^1.0.3" } }, + "node_modules/weak-lru-cache": { + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/weak-lru-cache/-/weak-lru-cache-1.2.2.tgz", + "integrity": "sha512-DEAoo25RfSYMuTGc9vPJzZcZullwIqRDSI9LOy+fkCJPi6hykCnfKaXTuPBDuXAUcqHXyOgFtHNp/kB2FjYHbw==", + "license": "MIT" + }, "node_modules/webidl-conversions": { "version": "8.0.1", "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-8.0.1.tgz", @@ -25069,13 +25378,12 @@ } }, "node_modules/yocto-queue": { - "version": "0.1.0", - "resolved": "https://registry.npmjs.org/yocto-queue/-/yocto-queue-0.1.0.tgz", - "integrity": "sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q==", - "dev": true, + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/yocto-queue/-/yocto-queue-1.2.2.tgz", + "integrity": "sha512-4LCcse/U2MHZ63HAJVE+v71o7yOdIe4cZ70Wpf8D/IyjDKYQLV5GD46B+hSTjJsvV5PztjvHoU580EftxjDZFQ==", "license": "MIT", "engines": { - "node": ">=10" + "node": ">=12.20" }, "funding": { "url": "https://github.com/sponsors/sindresorhus" diff --git a/package.json b/package.json index 2ca6b3ee0..1046bd4c9 100644 --- a/package.json +++ b/package.json @@ -13,7 +13,7 @@ "build:dll": "cross-env NODE_ENV=development TS_NODE_TRANSPILE_ONLY=true webpack --config .erb/configs/webpack.config.renderer.dev.dll.ts", "package": "rimraf dist && npm run build && electron-builder build --publish never", "init:ci": "npm ci --ignore-scripts && node node_modules/electron/install.js", - "init:e2e": "npm ci --ignore-scripts && node node_modules/electron/install.js && npm run build && npm run rebuild:electron", + "init:e2e": "npm run init:ci && npm run build && npm run rebuild:electron", "init:dev": "npm install && node node_modules/electron/install.js && npm run build:dll && npm run rebuild:electron", "========== Code style ==========": "", "lint": "eslint src --ext .ts,.tsx --max-warnings 276", @@ -178,8 +178,10 @@ "electron-store": "^8.0.1", "framer-motion": "^5.6.0", "jwt-decode": "^3.1.2", + "lmdb": "^3.5.4", "openapi-fetch": "^0.13.4", "openpgp": "5.11.3", + "p-limit": "^7.3.0", "phosphor-react": "^1.4.1", "react": "^17.0.2", "react-circular-progressbar": "^2.1.0", diff --git a/packages/core/internxt-drive-desktop-core-0.1.13.tgz b/packages/core/internxt-drive-desktop-core-0.1.13.tgz index db479d79e..ed216d3e0 100644 Binary files a/packages/core/internxt-drive-desktop-core-0.1.13.tgz and b/packages/core/internxt-drive-desktop-core-0.1.13.tgz differ diff --git a/packages/core/package-lock.json b/packages/core/package-lock.json index 7fa638b7a..68111a086 100644 --- a/packages/core/package-lock.json +++ b/packages/core/package-lock.json @@ -9,7 +9,7 @@ "version": "0.1.13", "license": "MIT", "dependencies": { - "@internxt/sdk": "^1.15.10", + "@internxt/sdk": "^1.16.2", "@phosphor-icons/react": "2.0.9", "@tanstack/react-virtual": "^3.13.12", "check-disk-space": "^3.4.0", @@ -854,13 +854,12 @@ "license": "BSD-3-Clause" }, "node_modules/@internxt/sdk": { - "version": "1.15.10", - "resolved": "https://registry.npmjs.org/@internxt/sdk/-/sdk-1.15.10.tgz", - "integrity": "sha512-ao5Bd64EmO3KFaUoYlJBXD747NBE1M3qiarzigjuD4cEdFmUEjs2/T/iah/F8KBYxqA9KN3WXYKWEHkj8Ssbrw==", + "version": "1.16.2", + "resolved": "https://registry.npmjs.org/@internxt/sdk/-/sdk-1.16.2.tgz", + "integrity": "sha512-hTtOxR94v1MsFbV2eX1CNDf5L7BNoGFScn+SNDipzT17PY0LhHm2vVOa1ncRFjqR0FUBqGi3Hpm+T8/4aQefxQ==", "license": "MIT", "dependencies": { - "axios": "1.15.0", - "internxt-crypto": "1.0.2" + "axios": "^1.16.0" } }, "node_modules/@isaacs/balanced-match": { @@ -982,88 +981,6 @@ "@jridgewell/sourcemap-codec": "^1.4.14" } }, - "node_modules/@noble/ciphers": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/@noble/ciphers/-/ciphers-2.2.0.tgz", - "integrity": "sha512-Z6pjIZ/8IJcCGzb2S/0Px5J81yij85xASuk1teLNeg75bfT07MV3a/O2Mtn1I2se43k3lkVEcFaR10N4cgQcZA==", - "license": "MIT", - "engines": { - "node": ">= 20.19.0" - }, - "funding": { - "url": "https://paulmillr.com/funding/" - } - }, - "node_modules/@noble/curves": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/@noble/curves/-/curves-2.2.0.tgz", - "integrity": "sha512-T/BoHgFXirb0ENSPBquzX0rcjXeM6Lo892a2jlYJkqk83LqZx0l1Of7DzlKJ6jkpvMrkHSnAcgb5JegL8SeIkQ==", - "license": "MIT", - "dependencies": { - "@noble/hashes": "2.2.0" - }, - "engines": { - "node": ">= 20.19.0" - }, - "funding": { - "url": "https://paulmillr.com/funding/" - } - }, - "node_modules/@noble/hashes": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/@noble/hashes/-/hashes-2.2.0.tgz", - "integrity": "sha512-IYqDGiTXab6FniAgnSdZwgWbomxpy9FtYvLKs7wCUs2a8RkITG+DFGO1DM9cr+E3/RgADRpFjrKVaJ1z6sjtEg==", - "license": "MIT", - "engines": { - "node": ">= 20.19.0" - }, - "funding": { - "url": "https://paulmillr.com/funding/" - } - }, - "node_modules/@noble/post-quantum": { - "version": "0.5.4", - "resolved": "https://registry.npmjs.org/@noble/post-quantum/-/post-quantum-0.5.4.tgz", - "integrity": "sha512-leww0zzIirrvwaYMPI9fj6aRIlA/c6Y0/lifQQ1YOOyHEr0MNH3yYpjXeiVG+tWdPps4XxGclFWX2INPO3Yo5w==", - "license": "MIT", - "dependencies": { - "@noble/curves": "~2.0.0", - "@noble/hashes": "~2.0.0" - }, - "engines": { - "node": ">= 20.19.0" - }, - "funding": { - "url": "https://paulmillr.com/funding/" - } - }, - "node_modules/@noble/post-quantum/node_modules/@noble/curves": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/@noble/curves/-/curves-2.0.1.tgz", - "integrity": "sha512-vs1Az2OOTBiP4q0pwjW5aF0xp9n4MxVrmkFBxc6EKZc6ddYx5gaZiAsZoq0uRRXWbi3AT/sBqn05eRPtn1JCPw==", - "license": "MIT", - "dependencies": { - "@noble/hashes": "2.0.1" - }, - "engines": { - "node": ">= 20.19.0" - }, - "funding": { - "url": "https://paulmillr.com/funding/" - } - }, - "node_modules/@noble/post-quantum/node_modules/@noble/hashes": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/@noble/hashes/-/hashes-2.0.1.tgz", - "integrity": "sha512-XlOlEbQcE9fmuXxrVTXCTlG2nlRXa9Rj3rr5Ue/+tX+nmkgbX720YHh0VR3hBF9xDvwnb8D2shVGOwNx+ulArw==", - "license": "MIT", - "engines": { - "node": ">= 20.19.0" - }, - "funding": { - "url": "https://paulmillr.com/funding/" - } - }, "node_modules/@nodelib/fs.scandir": { "version": "2.1.5", "resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz", @@ -1496,28 +1413,6 @@ "dev": true, "license": "MIT" }, - "node_modules/@scure/base": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/@scure/base/-/base-2.2.0.tgz", - "integrity": "sha512-b8XEupJibegiXV+tDUseI8oLQc8ei3d/4Jkb2RpbHh3MfE054ov3uIz2dhFkB3FI8iwYkEh0gGCApkrYggkPNg==", - "license": "MIT", - "funding": { - "url": "https://paulmillr.com/funding/" - } - }, - "node_modules/@scure/bip39": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/@scure/bip39/-/bip39-2.2.0.tgz", - "integrity": "sha512-T/Bj/YvYMNkIPq6EENO6/rcs2e7qTNuyoUXf0KBFDmp0ZDu0H2X4Lq6yC3i0c8PcWkov5EbW+yQZZbdMmk154A==", - "license": "MIT", - "dependencies": { - "@noble/hashes": "2.2.0", - "@scure/base": "2.2.0" - }, - "funding": { - "url": "https://paulmillr.com/funding/" - } - }, "node_modules/@sindresorhus/is": { "version": "4.6.0", "resolved": "https://registry.npmjs.org/@sindresorhus/is/-/is-4.6.0.tgz", @@ -2569,12 +2464,12 @@ } }, "node_modules/axios": { - "version": "1.15.0", - "resolved": "https://registry.npmjs.org/axios/-/axios-1.15.0.tgz", - "integrity": "sha512-wWyJDlAatxk30ZJer+GeCWS209sA42X+N5jU2jy6oHTp7ufw8uzUTVFBX9+wTfAlhiJXGS0Bq7X6efruWjuK9Q==", + "version": "1.16.0", + "resolved": "https://registry.npmjs.org/axios/-/axios-1.16.0.tgz", + "integrity": "sha512-6hp5CwvTPlN2A31g5dxnwAX0orzM7pmCRDLnZSX772mv8WDqICwFjowHuPs04Mc8deIld1+ejhtaMn5vp6b+1w==", "license": "MIT", "dependencies": { - "follow-redirects": "^1.15.11", + "follow-redirects": "^1.16.0", "form-data": "^4.0.5", "proxy-from-env": "^2.1.0" } @@ -4549,34 +4444,6 @@ "dev": true, "license": "ISC" }, - "node_modules/flexsearch": { - "version": "0.8.212", - "resolved": "https://registry.npmjs.org/flexsearch/-/flexsearch-0.8.212.tgz", - "integrity": "sha512-wSyJr1GUWoOOIISRu+X2IXiOcVfg9qqBRyCPRUdLMIGJqPzMo+jMRlvE83t14v1j0dRMEaBbER/adQjp6Du2pw==", - "funding": [ - { - "type": "github", - "url": "https://github.com/ts-thomas" - }, - { - "type": "paypal", - "url": "https://www.paypal.com/donate/?hosted_button_id=GEVR88FC9BWRW" - }, - { - "type": "opencollective", - "url": "https://opencollective.com/flexsearch" - }, - { - "type": "patreon", - "url": "https://patreon.com/user?u=96245532" - }, - { - "type": "liberapay", - "url": "https://liberapay.com/ts-thomas" - } - ], - "license": "Apache-2.0" - }, "node_modules/follow-redirects": { "version": "1.16.0", "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.16.0.tgz", @@ -5166,12 +5033,6 @@ "url": "https://github.com/sponsors/ljharb" } }, - "node_modules/hash-wasm": { - "version": "4.12.0", - "resolved": "https://registry.npmjs.org/hash-wasm/-/hash-wasm-4.12.0.tgz", - "integrity": "sha512-+/2B2rYLb48I/evdOIhP+K/DD2ca2fgBjp6O+GBEnCDk2e4rpeXIK8GvIyRPjTezgmWn9gmKwkQjjx6BtqDHVQ==", - "license": "MIT" - }, "node_modules/hasown": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/hasown/-/hasown-2.0.2.tgz", @@ -5232,27 +5093,6 @@ "node": ">=10.19.0" } }, - "node_modules/husky": { - "version": "9.1.7", - "resolved": "https://registry.npmjs.org/husky/-/husky-9.1.7.tgz", - "integrity": "sha512-5gs5ytaNjBrh5Ow3zrvdUUY+0VxIuWVL4i9irt6friV+BqdCfmV11CQTWMiBYWHbXhco+J1kHfTOUkePhCDvMA==", - "license": "MIT", - "bin": { - "husky": "bin.js" - }, - "engines": { - "node": ">=18" - }, - "funding": { - "url": "https://github.com/sponsors/typicode" - } - }, - "node_modules/idb": { - "version": "8.0.3", - "resolved": "https://registry.npmjs.org/idb/-/idb-8.0.3.tgz", - "integrity": "sha512-LtwtVyVYO5BqRvcsKuB2iUMnHwPVByPCXFXOpuU96IZPPoPN6xjOGxZQ74pgSVVLQWtUOYgyeL4GE98BY5D3wg==", - "license": "ISC" - }, "node_modules/ignore": { "version": "7.0.5", "resolved": "https://registry.npmjs.org/ignore/-/ignore-7.0.5.tgz", @@ -5358,36 +5198,6 @@ "node": ">= 0.4" } }, - "node_modules/internxt-crypto": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/internxt-crypto/-/internxt-crypto-1.0.2.tgz", - "integrity": "sha512-F9PuXci0eU1wlgDwqEbGR7hVDNS0MX8VNh/W+pdpR4ZsEsjRDBrOD2g1DvViR2woCxPiu1AW9Wwekpw2YVKfnA==", - "dependencies": { - "@noble/ciphers": "^2.1.1", - "@noble/curves": "^2.0.1", - "@noble/hashes": "^2.0.1", - "@noble/post-quantum": "^0.5.2", - "@scure/bip39": "^2.0.1", - "flexsearch": "^0.8.205", - "hash-wasm": "^4.12.0", - "husky": "^9.1.7", - "idb": "^8.0.3", - "uuid": "^13.0.0" - } - }, - "node_modules/internxt-crypto/node_modules/uuid": { - "version": "13.0.0", - "resolved": "https://registry.npmjs.org/uuid/-/uuid-13.0.0.tgz", - "integrity": "sha512-XQegIaBTVUjSHliKqcnFqYypAd4S+WCYt5NIeRs6w/UAry7z8Y9j5ZwRRL4kzq9U3sD6v+85er9FvkEaBpji2w==", - "funding": [ - "https://github.com/sponsors/broofa", - "https://github.com/sponsors/ctavan" - ], - "license": "MIT", - "bin": { - "uuid": "dist-node/bin/uuid" - } - }, "node_modules/is-array-buffer": { "version": "3.0.5", "resolved": "https://registry.npmjs.org/is-array-buffer/-/is-array-buffer-3.0.5.tgz", diff --git a/packages/core/package.json b/packages/core/package.json index a5f2be04e..26eb7aa5a 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -58,7 +58,7 @@ "vitest-mock-extended": "^3.1.0" }, "dependencies": { - "@internxt/sdk": "^1.15.10", + "@internxt/sdk": "^1.16.2", "@phosphor-icons/react": "2.0.9", "@tanstack/react-virtual": "^3.13.12", "check-disk-space": "^3.4.0", diff --git a/src/apps/sync-engine/refresh-item-placeholders.ts b/src/apps/sync-engine/refresh-item-placeholders.ts index ad3d1497a..0127c95de 100644 --- a/src/apps/sync-engine/refresh-item-placeholders.ts +++ b/src/apps/sync-engine/refresh-item-placeholders.ts @@ -1,6 +1,8 @@ +import pLimit from 'p-limit'; import { loadInMemoryPaths } from '@/backend/features/remote-sync/sync-items-by-checkpoint/load-in-memory-paths'; import { traverse } from '@/context/virtual-drive/items/application/Traverser'; import { measurePerfomance } from '@/core/utils/measure-performance'; +import { Lmdb } from '@/infra/lmdb/lmdb'; import { SqliteModule } from '@/infra/sqlite/sqlite.module'; import { SyncContext } from './config'; @@ -12,7 +14,7 @@ type Props = { export async function refreshItemPlaceholders({ ctx, isFirstExecution }: Props) { try { const time = await measurePerfomance(async () => { - const [database, fileExplorer] = await Promise.all([getDatabaseItems({ ctx }), loadInMemoryPaths({ ctx })]); + const [database] = await Promise.all([getDatabaseItems({ ctx }), loadInMemoryPaths({ ctx })]); ctx.logger.debug({ msg: 'Refresh item placeholders', @@ -21,15 +23,11 @@ export async function refreshItemPlaceholders({ ctx, isFirstExecution }: Props) files: database.files.length, folders: database.folders.length, }, - fileExplorer: { - files: fileExplorer.files.size, - folders: fileExplorer.folders.size, - }, }); const currentFolder = { absolutePath: ctx.rootPath, uuid: ctx.rootUuid }; - - await traverse({ ctx, currentFolder, database, fileExplorer, isFirstExecution }); + await traverse({ ctx, currentFolder, database, isFirstExecution, limit: pLimit(20) }); + await Lmdb.clear(); }); ctx.logger.debug({ msg: 'Finish refresh placeholders in seconds', time }); diff --git a/src/backend/features/remote-sync/file-explorer/check-if-modified.infra.test.ts b/src/backend/features/remote-sync/file-explorer/check-if-modified.infra.test.ts index 7aa0f6f07..d4a7f4a90 100644 --- a/src/backend/features/remote-sync/file-explorer/check-if-modified.infra.test.ts +++ b/src/backend/features/remote-sync/file-explorer/check-if-modified.infra.test.ts @@ -44,7 +44,8 @@ describe('check-if-modified', () => { }, local: { path, - stats: { mtime: new Date('2000-01-01'), size: 7 }, + mtime: new Date('2000-01-01'), + size: 7, }, }); diff --git a/src/backend/features/remote-sync/file-explorer/check-if-modified.test.ts b/src/backend/features/remote-sync/file-explorer/check-if-modified.test.ts index 607691231..8dc3e0082 100644 --- a/src/backend/features/remote-sync/file-explorer/check-if-modified.test.ts +++ b/src/backend/features/remote-sync/file-explorer/check-if-modified.test.ts @@ -2,7 +2,7 @@ import { FileUuid } from '@/apps/main/database/entities/DriveFile'; import { AbsolutePath } from '@/context/local/localFile/infrastructure/AbsolutePath'; import { NodeWin } from '@/infra/node-win/node-win.module'; import { Addon } from '@/node-win/addon-wrapper'; -import { loggerMock } from '@/tests/vitest/mocks.helper.test'; +import { loggerFn } from '@/tests/vitest/mocks.helper.test'; import { call, calls, mockProps, partialSpyOn } from '@/tests/vitest/utils.helper.test'; import { Drive } from '../../drive'; import { checkIfModified } from './check-if-modified'; @@ -23,7 +23,7 @@ describe('check-if-modified', () => { props = mockProps({ local: { path: 'localPath' as AbsolutePath, - stats: { size: 512 }, + size: 512, }, remote: { absolutePath: 'remotePath' as AbsolutePath, @@ -36,26 +36,26 @@ describe('check-if-modified', () => { it('should not sync when file sizes are equal', async () => { // Given - props.local.stats.size = 1024; + props.local.size = 1024; // When await checkIfModified(props); // Then - calls(loggerMock.debug).toHaveLength(0); + calls(loggerFn).toHaveLength(0); }); it('should sync when remote file is newer', async () => { // Given - props.local.stats.mtime = new Date('2000-01-01'); + props.local.mtime = new Date('2000-01-01'); // When await checkIfModified(props); // Then - call(loggerMock.debug).toMatchObject({ msg: 'Sync remote changes to local' }); + call(loggerFn).toMatchObject({ msg: 'Sync remote changes to local' }); call(updatePlaceholderMock).toStrictEqual({ path: 'remotePath', placeholderId: 'FILE:uuid', size: 1024 }); }); describe('what happens when local file is newer', () => { beforeEach(() => { - props.local.stats.mtime = new Date('2000-01-03'); + props.local.mtime = new Date('2000-01-03'); }); it('should not sync when is not first execution', async () => { @@ -64,7 +64,7 @@ describe('check-if-modified', () => { // When await checkIfModified(props); // Then - calls(loggerMock.debug).toHaveLength(0); + calls(loggerFn).toHaveLength(0); }); it('should not sync when local file is dehydrated', async () => { @@ -74,8 +74,7 @@ describe('check-if-modified', () => { // When await checkIfModified(props); // Then - call(loggerMock.debug).toMatchObject({ msg: 'Sync local changes to remote' }); - call(loggerMock.error).toMatchObject({ msg: 'Cannot update file contents id, not hydrated' }); + calls(loggerFn).toMatchObject([{ msg: 'Sync local changes to remote' }, { msg: 'Cannot update file contents id, not hydrated' }]); }); it('should sync when local file is hydrated', async () => { @@ -85,7 +84,7 @@ describe('check-if-modified', () => { // When await checkIfModified(props); // Then - call(loggerMock.debug).toMatchObject({ msg: 'Sync local changes to remote' }); + call(loggerFn).toMatchObject({ msg: 'Sync local changes to remote' }); call(replaceFileMock).toMatchObject({ path: 'remotePath', uuid: 'uuid' }); }); }); diff --git a/src/backend/features/remote-sync/file-explorer/check-if-modified.ts b/src/backend/features/remote-sync/file-explorer/check-if-modified.ts index 2a793a0e9..038bd4dcf 100644 --- a/src/backend/features/remote-sync/file-explorer/check-if-modified.ts +++ b/src/backend/features/remote-sync/file-explorer/check-if-modified.ts @@ -1,7 +1,6 @@ -import { Stats } from 'node:fs'; import { ExtendedDriveFile } from '@/apps/main/database/entities/DriveFile'; import { SyncContext } from '@/apps/sync-engine/config'; -import { AbsolutePath } from '@/context/local/localFile/infrastructure/AbsolutePath'; +import { LmdbFile } from '@/infra/lmdb/lmdb'; import { NodeWin } from '@/infra/node-win/node-win.module'; import { Addon } from '@/node-win/addon-wrapper'; import { PinState } from '@/node-win/types/placeholder.type'; @@ -10,15 +9,15 @@ import { Drive } from '../../drive'; type Props = { ctx: SyncContext; remote: ExtendedDriveFile; - local: { path: AbsolutePath; stats: Stats }; + local: LmdbFile; isFirstExecution: boolean; }; export async function checkIfModified({ ctx, remote, local, isFirstExecution }: Props) { const path = remote.absolutePath; - const localSize = local.stats.size; - const localDate = local.stats.mtime; + const localSize = local.size; + const localDate = local.mtime; const remoteSize = remote.size; const remoteDate = new Date(remote.updatedAt); diff --git a/src/backend/features/remote-sync/file-explorer/delete-item-placeholder.test.ts b/src/backend/features/remote-sync/file-explorer/delete-item-placeholder.test.ts index 86adc4994..39d7dba24 100644 --- a/src/backend/features/remote-sync/file-explorer/delete-item-placeholder.test.ts +++ b/src/backend/features/remote-sync/file-explorer/delete-item-placeholder.test.ts @@ -3,6 +3,7 @@ import { mkdir, rename, rm } from 'node:fs/promises'; import trash from 'trash'; import { FileUuid } from '@/apps/main/database/entities/DriveFile'; import { abs } from '@/context/local/localFile/infrastructure/AbsolutePath'; +import { Lmdb } from '@/infra/lmdb/lmdb'; import { Addon } from '@/node-win/addon-wrapper'; import { loggerFn, loggerMock } from '@/tests/vitest/mocks.helper.test'; import { call, calls, deepMocked, partialSpyOn, TestProps } from '@/tests/vitest/utils.helper.test'; @@ -18,6 +19,7 @@ describe('delete-item-placeholder', () => { const rmMock = deepMocked(rm); const trashMock = deepMocked(trash); const randomUUIDMock = deepMocked(randomUUID); + const lmdbGet = partialSpyOn(Lmdb, 'get'); const getFirstNonPlaceholderMock = partialSpyOn(Addon, 'getFirstNonPlaceholder'); const uuid = 'uuid' as FileUuid; @@ -29,17 +31,18 @@ describe('delete-item-placeholder', () => { beforeEach(() => { randomUUIDMock.mockReturnValue('randomUUID' as any); + lmdbGet.mockReturnValue({ path: localPath }); + props = { ctx: { logger: loggerMock }, remote: { absolutePath: localPath, uuid }, - locals: new Map([[uuid, { path: localPath }]]), type: 'file', }; }); it('should skip if local item does not exist', async () => { // Given - props.locals = new Map(); + lmdbGet.mockReturnValue(undefined); // When await deleteItemPlaceholder(props as any); // Then diff --git a/src/backend/features/remote-sync/file-explorer/delete-item-placeholder.ts b/src/backend/features/remote-sync/file-explorer/delete-item-placeholder.ts index e0dfdf8c0..ce10ce9bd 100644 --- a/src/backend/features/remote-sync/file-explorer/delete-item-placeholder.ts +++ b/src/backend/features/remote-sync/file-explorer/delete-item-placeholder.ts @@ -1,20 +1,19 @@ -import { AbsolutePath } from '@internxt/drive-desktop-core/build/backend'; import { randomUUID } from 'node:crypto'; import { mkdir, rename, rm } from 'node:fs/promises'; import { join, parse } from 'node:path'; import { ExtendedDriveFile } from '@/apps/main/database/entities/DriveFile'; import { ExtendedDriveFolder } from '@/apps/main/database/entities/DriveFolder'; import { SyncContext } from '@/apps/sync-engine/config'; +import { Lmdb } from '@/infra/lmdb/lmdb'; import { Addon } from '@/node-win/addon-wrapper'; -import { FileExplorerFiles, FileExplorerFolders } from '../sync-items-by-checkpoint/load-in-memory-paths'; -type FileProps = { type: 'file'; remote: ExtendedDriveFile; locals: FileExplorerFiles }; -type FolderProps = { type: 'folder'; remote: ExtendedDriveFolder; locals: FileExplorerFolders }; +type FileProps = { type: 'file'; remote: ExtendedDriveFile }; +type FolderProps = { type: 'folder'; remote: ExtendedDriveFolder }; type Props = { ctx: SyncContext } & (FileProps | FolderProps); -export async function deleteItemPlaceholder({ ctx, type, remote, locals }: Props) { +export async function deleteItemPlaceholder({ ctx, type, remote }: Props) { try { - const local = (locals as Map).get(remote.uuid); + const local = Lmdb.get(remote.uuid); if (!local) return; @@ -25,7 +24,6 @@ export async function deleteItemPlaceholder({ ctx, type, remote, locals }: Props * so instead of deleting the placeholder, we are going to send the item to the trash * so the user can decide whether to delete it or recover it. */ - ctx.logger.error({ msg: 'Path does not match when deleting placeholder', remotePath: remote.absolutePath, diff --git a/src/backend/features/remote-sync/file-explorer/update-file-placeholder.test.ts b/src/backend/features/remote-sync/file-explorer/update-file-placeholder.test.ts index 17df1bc69..2b4e2433c 100644 --- a/src/backend/features/remote-sync/file-explorer/update-file-placeholder.test.ts +++ b/src/backend/features/remote-sync/file-explorer/update-file-placeholder.test.ts @@ -2,6 +2,7 @@ import { rename } from 'node:fs/promises'; import { FileUuid } from '@/apps/main/database/entities/DriveFile'; import { AbsolutePath } from '@/context/local/localFile/infrastructure/AbsolutePath'; import * as validateWindowsName from '@/context/virtual-drive/items/validate-windows-name'; +import { Lmdb } from '@/infra/lmdb/lmdb'; import { Addon } from '@/node-win/addon-wrapper'; import { PinState } from '@/node-win/types/placeholder.type'; import { loggerMock, loggerFn } from '@/tests/vitest/mocks.helper.test'; @@ -12,6 +13,7 @@ import { updateFilePlaceholder } from './update-file-placeholder'; vi.mock(import('node:fs/promises')); describe('update-file-placeholder', () => { + const lmdbGetFile = partialSpyOn(Lmdb, 'getFile'); const createFilePlaceholderMock = partialSpyOn(Addon, 'createFilePlaceholder'); const updateSyncStatusMock = partialSpyOn(Addon, 'updateSyncStatus'); const setPinStateMock = partialSpyOn(Addon, 'setPinState'); @@ -25,11 +27,11 @@ describe('update-file-placeholder', () => { beforeEach(() => { validateWindowsNameMock.mockReturnValue({ isValid: true }); + lmdbGetFile.mockReturnValue({ path: 'localPath' as AbsolutePath }); props = { ctx: { logger: loggerMock }, isFirstExecution: false, - files: new Map([['uuid' as FileUuid, { path: 'localPath' as AbsolutePath }]]), remote: { absolutePath: 'remotePath' as AbsolutePath, uuid: 'uuid' as FileUuid, @@ -51,7 +53,7 @@ describe('update-file-placeholder', () => { it('should create placeholder if file does not exist locally', async () => { // Given - props.files = new Map(); + lmdbGetFile.mockReturnValue(undefined); // When await updateFilePlaceholder(props as any); // Then @@ -68,12 +70,7 @@ describe('update-file-placeholder', () => { it('should reset pin state if file is partially hydrated', async () => { // Given props.isFirstExecution = true; - props.files = new Map([ - [ - 'uuid' as FileUuid, - { path: 'remotePath' as AbsolutePath, stats: { size: 1024 }, placeholder: { pinState: PinState.AlwaysLocal, onDiskSize: 512 } }, - ], - ]); + lmdbGetFile.mockReturnValue({ path: 'remotePath' as AbsolutePath, size: 1024, pinState: PinState.AlwaysLocal, onDiskSize: 512 }); // When await updateFilePlaceholder(props as any); // Then diff --git a/src/backend/features/remote-sync/file-explorer/update-file-placeholder.ts b/src/backend/features/remote-sync/file-explorer/update-file-placeholder.ts index 4fd7f615d..d94ea40d9 100644 --- a/src/backend/features/remote-sync/file-explorer/update-file-placeholder.ts +++ b/src/backend/features/remote-sync/file-explorer/update-file-placeholder.ts @@ -1,20 +1,19 @@ import { ExtendedDriveFile } from '@/apps/main/database/entities/DriveFile'; import { SyncContext } from '@/apps/sync-engine/config'; import { validateWindowsName } from '@/context/virtual-drive/items/validate-windows-name'; +import { Lmdb } from '@/infra/lmdb/lmdb'; import { Addon } from '@/node-win/addon-wrapper'; import { PinState } from '@/node-win/types/placeholder.type'; -import { FileExplorerFiles } from '../sync-items-by-checkpoint/load-in-memory-paths'; import { checkIfModified } from './check-if-modified'; import { checkIfMoved } from './check-if-moved'; type Props = { ctx: SyncContext; remote: ExtendedDriveFile; - files: FileExplorerFiles; isFirstExecution: boolean; }; -export async function updateFilePlaceholder({ ctx, remote, files, isFirstExecution }: Props) { +export async function updateFilePlaceholder({ ctx, remote, isFirstExecution }: Props) { const path = remote.absolutePath; const { size } = remote; @@ -22,7 +21,7 @@ export async function updateFilePlaceholder({ ctx, remote, files, isFirstExecuti const { isValid } = validateWindowsName({ path, name: remote.name }); if (!isValid) return; - const local = files.get(remote.uuid); + const local = Lmdb.getFile(remote.uuid); if (!local) { await Addon.createFilePlaceholder({ @@ -47,8 +46,8 @@ export async function updateFilePlaceholder({ ctx, remote, files, isFirstExecuti // the opposite, that the user wants to continue after opening the app again. We will keep the first use // case since it's easier to implement, consumes less resources and in case the user wants to hydrate the // folder he can perform the action again. - const { onDiskSize } = local.placeholder; - if (local.placeholder.pinState === PinState.AlwaysLocal && onDiskSize < size) { + const { onDiskSize } = local; + if (local.pinState === PinState.AlwaysLocal && onDiskSize < size) { ctx.logger.debug({ msg: 'File stuck in hydrated state', onDiskSize, size, path }); await Addon.setPinState({ path, pinState: PinState.Unspecified }); } diff --git a/src/backend/features/remote-sync/file-explorer/update-folder-placeholder.test.ts b/src/backend/features/remote-sync/file-explorer/update-folder-placeholder.test.ts index 76d11a67b..3c7aa04f1 100644 --- a/src/backend/features/remote-sync/file-explorer/update-folder-placeholder.test.ts +++ b/src/backend/features/remote-sync/file-explorer/update-folder-placeholder.test.ts @@ -2,15 +2,17 @@ import { rename } from 'node:fs/promises'; import { FolderUuid } from '@/apps/main/database/entities/DriveFolder'; import { AbsolutePath } from '@/context/local/localFile/infrastructure/AbsolutePath'; import * as validateWindowsName from '@/context/virtual-drive/items/validate-windows-name'; +import { Lmdb } from '@/infra/lmdb/lmdb'; import { Addon } from '@/node-win/addon-wrapper'; import { loggerMock } from '@/tests/vitest/mocks.helper.test'; -import { call, mockProps, partialSpyOn } from '@/tests/vitest/utils.helper.test'; +import { call, partialSpyOn, TestProps } from '@/tests/vitest/utils.helper.test'; import * as needsToBeMoved from './needs-to-be-moved'; import { updateFolderPlaceholder } from './update-folder-placeholder'; vi.mock(import('node:fs/promises')); describe('update-folder-placeholder', () => { + const lmdbGetFolder = partialSpyOn(Lmdb, 'getFolder'); const createFolderPlaceholderMock = partialSpyOn(Addon, 'createFolderPlaceholder'); const updateSyncStatusMock = partialSpyOn(Addon, 'updateSyncStatus'); const validateWindowsNameMock = partialSpyOn(validateWindowsName, 'validateWindowsName'); @@ -19,28 +21,28 @@ describe('update-folder-placeholder', () => { const date = '2000-01-01T00:00:00.000Z'; const time = new Date(date).getTime(); - - let props: Parameters[0]; + let props: TestProps; beforeEach(() => { validateWindowsNameMock.mockReturnValue({ isValid: true }); + lmdbGetFolder.mockReturnValue({ path: 'localPath' as AbsolutePath }); - props = mockProps({ - folders: new Map([['uuid' as FolderUuid, { path: 'localPath' as AbsolutePath }]]), + props = { + ctx: { logger: loggerMock }, remote: { absolutePath: 'remotePath' as AbsolutePath, uuid: 'uuid' as FolderUuid, createdAt: date, updatedAt: date, }, - }); + }; }); it('should do nothing if name is invalid', async () => { // Given validateWindowsNameMock.mockReturnValue({ isValid: false }); // When - const res = await updateFolderPlaceholder(props); + const res = await updateFolderPlaceholder(props as any); // Then expect(res).toBe(false); expect(needsToBeMovedMock).toBeCalledTimes(0); @@ -48,9 +50,9 @@ describe('update-folder-placeholder', () => { it('should create placeholder if folder does not exist locally', async () => { // Given - props.folders = new Map(); + lmdbGetFolder.mockReturnValue(undefined); // When - const res = await updateFolderPlaceholder(props); + const res = await updateFolderPlaceholder(props as any); // Then expect(res).toBe(true); expect(needsToBeMovedMock).toBeCalledTimes(0); @@ -67,7 +69,7 @@ describe('update-folder-placeholder', () => { // Given needsToBeMovedMock.mockResolvedValue(true); // When - const res = await updateFolderPlaceholder(props); + const res = await updateFolderPlaceholder(props as any); // Then expect(res).toBe(true); expect(createFolderPlaceholderMock).toBeCalledTimes(0); @@ -80,7 +82,7 @@ describe('update-folder-placeholder', () => { // Given needsToBeMovedMock.mockResolvedValue(false); // When - const res = await updateFolderPlaceholder(props); + const res = await updateFolderPlaceholder(props as any); // Then expect(res).toBe(true); expect(createFolderPlaceholderMock).toBeCalledTimes(0); @@ -93,7 +95,7 @@ describe('update-folder-placeholder', () => { throw new Error('Something failed'); }); // When - const res = await updateFolderPlaceholder(props); + const res = await updateFolderPlaceholder(props as any); // Then expect(res).toBe(false); expect(needsToBeMovedMock).toBeCalledTimes(0); diff --git a/src/backend/features/remote-sync/file-explorer/update-folder-placeholder.ts b/src/backend/features/remote-sync/file-explorer/update-folder-placeholder.ts index bdc1820cf..f62fb7859 100644 --- a/src/backend/features/remote-sync/file-explorer/update-folder-placeholder.ts +++ b/src/backend/features/remote-sync/file-explorer/update-folder-placeholder.ts @@ -1,24 +1,23 @@ import { ExtendedDriveFolder } from '@/apps/main/database/entities/DriveFolder'; import { SyncContext } from '@/apps/sync-engine/config'; import { validateWindowsName } from '@/context/virtual-drive/items/validate-windows-name'; +import { Lmdb } from '@/infra/lmdb/lmdb'; import { Addon } from '@/node-win/addon-wrapper'; -import { FileExplorerFolders } from '../sync-items-by-checkpoint/load-in-memory-paths'; import { checkIfMoved } from './check-if-moved'; type Props = { ctx: SyncContext; remote: ExtendedDriveFolder; - folders: FileExplorerFolders; }; -export async function updateFolderPlaceholder({ ctx, remote, folders }: Props) { +export async function updateFolderPlaceholder({ ctx, remote }: Props) { const path = remote.absolutePath; try { const { isValid } = validateWindowsName({ path, name: remote.name }); if (!isValid) return false; - const local = folders.get(remote.uuid); + const local = Lmdb.getFolder(remote.uuid); if (!local) { await Addon.createFolderPlaceholder({ diff --git a/src/backend/features/remote-sync/sync-items-by-checkpoint/load-in-memory-paths.test.ts b/src/backend/features/remote-sync/sync-items-by-checkpoint/load-in-memory-paths.test.ts index 6274a9bfb..79ec764ad 100644 --- a/src/backend/features/remote-sync/sync-items-by-checkpoint/load-in-memory-paths.test.ts +++ b/src/backend/features/remote-sync/sync-items-by-checkpoint/load-in-memory-paths.test.ts @@ -2,14 +2,17 @@ import { FileUuid } from '@/apps/main/database/entities/DriveFile'; import { FolderUuid } from '@/apps/main/database/entities/DriveFolder'; import { abs } from '@/context/local/localFile/infrastructure/AbsolutePath'; import * as statReaddir from '@/infra/file-system/services/stat-readdir'; +import { Lmdb } from '@/infra/lmdb/lmdb'; import { NodeWin } from '@/infra/node-win/node-win.module'; -import { mockProps, partialSpyOn } from '@/tests/vitest/utils.helper.test'; +import { call, calls, mockProps, partialSpyOn } from '@/tests/vitest/utils.helper.test'; import { loadInMemoryPaths } from './load-in-memory-paths'; describe('load-in-memory-paths', () => { const statReaddirMock = partialSpyOn(statReaddir, 'statReaddir'); const getFolderInfoMock = partialSpyOn(NodeWin, 'getFolderInfo'); const getFileInfoMock = partialSpyOn(NodeWin, 'getFileInfo'); + const lmdbAddFile = partialSpyOn(Lmdb, 'addFile'); + const lmdbAddFolder = partialSpyOn(Lmdb, 'addFolder'); const props = mockProps({ ctx: {} }); @@ -17,11 +20,11 @@ describe('load-in-memory-paths', () => { // Given statReaddirMock .mockResolvedValueOnce({ - files: [{ path: abs('/file1') }, { path: abs('/file2') }], + files: [{ path: abs('/file1') }, { path: abs('/file2'), stats: {} }], folders: [{ path: abs('/folder1') }, { path: abs('/folder2') }], }) .mockResolvedValueOnce({ - files: [{ path: abs('/file3') }], + files: [{ path: abs('/file3'), stats: {} }], folders: [], }); @@ -31,14 +34,12 @@ describe('load-in-memory-paths', () => { .mockResolvedValueOnce({ data: { uuid: 'fileUuid2' as FileUuid } }) .mockResolvedValueOnce({ data: { uuid: 'fileUuid3' as FileUuid } }); // When - const { files, folders } = await loadInMemoryPaths(props); + await loadInMemoryPaths(props); // Then - expect(folders).toStrictEqual(new Map([['folderUuid', { path: '/folder1' }]])); - expect(files).toMatchObject( - new Map([ - ['fileUuid2', { path: '/file2' }], - ['fileUuid3', { path: '/file3' }], - ]), - ); + call(lmdbAddFolder).toStrictEqual(['folderUuid', { path: '/folder1' }]); + calls(lmdbAddFile).toMatchObject([ + ['fileUuid2', { path: '/file2' }], + ['fileUuid3', { path: '/file3' }], + ]); }); }); diff --git a/src/backend/features/remote-sync/sync-items-by-checkpoint/load-in-memory-paths.ts b/src/backend/features/remote-sync/sync-items-by-checkpoint/load-in-memory-paths.ts index 305eab137..01b2ed2ad 100644 --- a/src/backend/features/remote-sync/sync-items-by-checkpoint/load-in-memory-paths.ts +++ b/src/backend/features/remote-sync/sync-items-by-checkpoint/load-in-memory-paths.ts @@ -1,45 +1,41 @@ -import { Stats } from 'node:fs'; -import { FileUuid } from '@/apps/main/database/entities/DriveFile'; -import { FolderUuid } from '@/apps/main/database/entities/DriveFolder'; +import pLimit from 'p-limit'; import { SyncContext } from '@/apps/sync-engine/config'; import { AbsolutePath } from '@/context/local/localFile/infrastructure/AbsolutePath'; import { statReaddir } from '@/infra/file-system/services/stat-readdir'; +import { Lmdb } from '@/infra/lmdb/lmdb'; import { NodeWin } from '@/infra/node-win/node-win.module'; -import { FilePlaceholder } from '@/infra/node-win/services/get-file-info'; -export type FileExplorerFiles = Map; -export type FileExplorerFolders = Map; - -type Props = { - ctx: SyncContext; -}; - -export async function loadInMemoryPaths({ ctx }: Props) { - const files: FileExplorerFiles = new Map(); - const folders: FileExplorerFolders = new Map(); +export async function loadInMemoryPaths({ ctx }: { ctx: SyncContext }) { + const limit = pLimit(20); async function walk(parentPath: AbsolutePath) { const items = await statReaddir({ folder: parentPath }); - const filePromises = items.files.map(async ({ path, stats }) => { - const { data: placeholder } = await NodeWin.getFileInfo({ path }); - if (placeholder) { - files.set(placeholder.uuid, { stats, path, placeholder }); - } - }); - - const folderPromises = items.folders.map(async ({ path }) => { + await Promise.all( + items.files.map(({ path, stats }) => + limit(async () => { + const { data: placeholder } = await NodeWin.getFileInfo({ path }); + if (placeholder) { + await Lmdb.addFile(placeholder.uuid, { + path, + mtime: stats.mtime, + size: stats.size, + onDiskSize: placeholder.onDiskSize, + pinState: placeholder.pinState, + }); + } + }), + ), + ); + + for (const { path } of items.folders) { const { data: placeholder } = await NodeWin.getFolderInfo({ ctx, path }); if (placeholder) { - folders.set(placeholder.uuid, { path }); + await Lmdb.addFolder(placeholder.uuid, { path }); await walk(path); } - }); - - await Promise.all(filePromises.concat(folderPromises)); + } } await walk(ctx.rootPath); - - return { files, folders }; } diff --git a/src/context/virtual-drive/items/application/Traverser.test.ts b/src/context/virtual-drive/items/application/Traverser.test.ts index 56b379c4c..914d7325e 100644 --- a/src/context/virtual-drive/items/application/Traverser.test.ts +++ b/src/context/virtual-drive/items/application/Traverser.test.ts @@ -1,3 +1,4 @@ +import pLimit from 'p-limit'; import { calls, mockProps, partialSpyOn } from 'tests/vitest/utils.helper.test'; import { FolderUuid } from '@/apps/main/database/entities/DriveFolder'; import * as deleteItemPlaceholder from '@/backend/features/remote-sync/file-explorer/delete-item-placeholder'; @@ -16,8 +17,8 @@ describe('traverse', () => { beforeEach(() => { props = mockProps({ ctx: { abortController: new AbortController() }, + limit: pLimit(20), currentFolder: { absolutePath: abs('/drive'), uuid: 'root' as FolderUuid }, - fileExplorer: {}, database: { files: [ { parentUuid: 'root' as FolderUuid, name: 'deleted', status: 'DELETED' }, @@ -73,9 +74,9 @@ describe('traverse', () => { calls(updateFolderPlaceholderMock).toMatchObject([ { remote: { absolutePath: '/drive/parent1' } }, - { remote: { absolutePath: '/drive/child1' } }, { remote: { absolutePath: '/drive/parent1/parent2' } }, { remote: { absolutePath: '/drive/parent1/child2' } }, + { remote: { absolutePath: '/drive/child1' } }, ]); calls(updateFilePlaceholderMock).toMatchObject([ @@ -99,10 +100,10 @@ describe('traverse', () => { calls(updateFolderPlaceholderMock).toMatchObject([ { remote: { absolutePath: '/drive/parent1' } }, - { remote: { absolutePath: '/drive/child1' } }, { remote: { absolutePath: '/drive/parent1/parent2' } }, - { remote: { absolutePath: '/drive/parent1/child2' } }, { remote: { absolutePath: '/drive/parent1/parent2/child3' } }, + { remote: { absolutePath: '/drive/parent1/child2' } }, + { remote: { absolutePath: '/drive/child1' } }, ]); calls(updateFilePlaceholderMock).toMatchObject([ diff --git a/src/context/virtual-drive/items/application/Traverser.ts b/src/context/virtual-drive/items/application/Traverser.ts index d8dfb088f..1fb7362e5 100644 --- a/src/context/virtual-drive/items/application/Traverser.ts +++ b/src/context/virtual-drive/items/application/Traverser.ts @@ -1,53 +1,54 @@ +import { LimitFunction } from 'p-limit'; import { SimpleDriveFile } from '@/apps/main/database/entities/DriveFile'; import { ExtendedDriveFolder, SimpleDriveFolder } from '@/apps/main/database/entities/DriveFolder'; import { ProcessSyncContext } from '@/apps/sync-engine/config'; import { deleteItemPlaceholder } from '@/backend/features/remote-sync/file-explorer/delete-item-placeholder'; import { updateFilePlaceholder } from '@/backend/features/remote-sync/file-explorer/update-file-placeholder'; import { updateFolderPlaceholder } from '@/backend/features/remote-sync/file-explorer/update-folder-placeholder'; -import { FileExplorerFiles, FileExplorerFolders } from '@/backend/features/remote-sync/sync-items-by-checkpoint/load-in-memory-paths'; import { join } from '@/context/local/localFile/infrastructure/AbsolutePath'; type Database = { files: SimpleDriveFile[]; folders: SimpleDriveFolder[] }; -type FileExplorer = { files: FileExplorerFiles; folders: FileExplorerFolders }; type Props = { ctx: ProcessSyncContext; database: Database; - fileExplorer: FileExplorer; currentFolder: Pick; isFirstExecution: boolean; + limit: LimitFunction; }; -export async function traverse({ ctx, database, fileExplorer, currentFolder, isFirstExecution }: Props) { +export async function traverse({ ctx, database, currentFolder, isFirstExecution, limit }: Props) { if (ctx.abortController.signal.aborted) return; const filesInThisFolder = database.files.filter((file) => file.parentUuid === currentFolder.uuid); const foldersInThisFolder = database.folders.filter((folder) => folder.parentUuid === currentFolder.uuid); - const filePromises = filesInThisFolder.map(async (file) => { - const absolutePath = join(currentFolder.absolutePath, file.name); - const remote = { ...file, absolutePath }; - - if (file.status === 'DELETED' || file.status === 'TRASHED') { - await deleteItemPlaceholder({ ctx, type: 'file', remote, locals: fileExplorer.files }); - } else { - await updateFilePlaceholder({ ctx, remote, files: fileExplorer.files, isFirstExecution }); - } - }); - - const folderPromises = foldersInThisFolder.map(async (folder) => { + await Promise.all( + filesInThisFolder.map((file) => + limit(async () => { + const absolutePath = join(currentFolder.absolutePath, file.name); + const remote = { ...file, absolutePath }; + + if (file.status === 'DELETED' || file.status === 'TRASHED') { + await deleteItemPlaceholder({ ctx, type: 'file', remote }); + } else { + await updateFilePlaceholder({ ctx, remote, isFirstExecution }); + } + }), + ), + ); + + for (const folder of foldersInThisFolder) { const absolutePath = join(currentFolder.absolutePath, folder.name); const remote = { ...folder, absolutePath }; if (folder.status === 'DELETED' || folder.status === 'TRASHED') { - await deleteItemPlaceholder({ ctx, type: 'folder', remote, locals: fileExplorer.folders }); + await deleteItemPlaceholder({ ctx, type: 'folder', remote }); } else { - const success = await updateFolderPlaceholder({ ctx, remote, folders: fileExplorer.folders }); + const success = await updateFolderPlaceholder({ ctx, remote }); if (success) { - await traverse({ ctx, database, fileExplorer, currentFolder: remote, isFirstExecution }); + await traverse({ ctx, database, currentFolder: remote, isFirstExecution, limit }); } } - }); - - await Promise.all(filePromises.concat(folderPromises)); + } } diff --git a/src/core/electron/paths.ts b/src/core/electron/paths.ts index 4fbaabb68..d134ec363 100644 --- a/src/core/electron/paths.ts +++ b/src/core/electron/paths.ts @@ -9,13 +9,13 @@ const APP_DATA_PATH = abs(e2ePaths?.e2eAppDataPath ?? app.getPath('appData')); const INTERNXT = join(APP_DATA_PATH, 'internxt-drive'); const SQLITE_DB = join(INTERNXT, 'internxt_desktop.db'); -const LOKIJS_DB = join(INTERNXT, 'internxt_desktop.json'); +const LMDB_DB = join(INTERNXT, 'internxt_desktop.lmdb'); const LOGS = join(INTERNXT, 'logs'); export const PATHS = { HOME_FOLDER_PATH, INTERNXT, SQLITE_DB, - LOKIJS_DB, + LMDB_DB, LOGS, }; diff --git a/src/infra/lmdb/lmdb.ts b/src/infra/lmdb/lmdb.ts new file mode 100644 index 000000000..23cca903b --- /dev/null +++ b/src/infra/lmdb/lmdb.ts @@ -0,0 +1,37 @@ +import { AbsolutePath } from '@internxt/drive-desktop-core/build/backend'; +import { open } from 'lmdb'; +import { FileUuid } from '@/apps/main/database/entities/DriveFile'; +import { FolderUuid } from '@/apps/main/database/entities/DriveFolder'; +import { PATHS } from '@/core/electron/paths'; +import { PinState } from '@/node-win/types/placeholder.type'; + +export type LmdbFile = { path: AbsolutePath; pinState: PinState; onDiskSize: number; size: number; mtime: Date }; +export type LmdbFolder = { path: AbsolutePath }; + +export class Lmdb { + static db = open({ path: PATHS.LMDB_DB, encoding: 'json' }); + + static get(uuid: FileUuid | FolderUuid) { + return this.db.get(uuid); + } + + static getFile(uuid: FileUuid) { + return this.db.get(uuid) as LmdbFile | undefined; + } + + static getFolder(uuid: FolderUuid) { + return this.db.get(uuid) as LmdbFolder | undefined; + } + + static addFile(uuid: FileUuid, file: LmdbFile) { + return this.db.put(uuid, file); + } + + static addFolder(uuid: FolderUuid, folder: LmdbFolder) { + return this.db.put(uuid, folder); + } + + static clear() { + return this.db.clearAsync(); + } +} diff --git a/src/infra/node-win/services/get-file-info.ts b/src/infra/node-win/services/get-file-info.ts index cd440bb39..1ab939283 100644 --- a/src/infra/node-win/services/get-file-info.ts +++ b/src/infra/node-win/services/get-file-info.ts @@ -3,8 +3,6 @@ import { FileUuid } from '@/apps/main/database/entities/DriveFile'; import { FilePlaceholderId } from '@/context/virtual-drive/files/domain/PlaceholderId'; import { Addon } from '@/node-win/addon-wrapper'; -export type FilePlaceholder = NonNullable>['data']>; - export class GetFileInfoError extends Error { constructor( public readonly code: 'NOT_A_PLACEHOLDER' | 'UNKNOWN', diff --git a/src/infra/sqlite/services/file/create-or-update-batch.test.ts b/src/infra/sqlite/services/file/create-or-update-batch.test.ts index 38f7ca2df..fe882a2f0 100644 --- a/src/infra/sqlite/services/file/create-or-update-batch.test.ts +++ b/src/infra/sqlite/services/file/create-or-update-batch.test.ts @@ -38,19 +38,19 @@ describe('create-or-update-batch', () => { }; }); - it('should ignore if no files', () => { + it('should ignore if no files', async () => { // Given props.files = []; // When - const error = createOrUpdateBatch(props); + const error = await createOrUpdateBatch(props); // Then expect(error).toBeUndefined(); expect({ ...db.prepare('SELECT COUNT(*) FROM drive_file').get() }).toStrictEqual({ 'COUNT(*)': 0 }); }); - it('should insert new files', () => { + it('should insert new files', async () => { // When - const error = createOrUpdateBatch(props); + const error = await createOrUpdateBatch(props); // Then expect(error).toBeUndefined(); expect({ ...db.prepare('SELECT COUNT(*) FROM drive_file').get() }).toStrictEqual({ 'COUNT(*)': 450 }); @@ -72,12 +72,12 @@ describe('create-or-update-batch', () => { }); }); - it('should update existing files', () => { + it('should update existing files', async () => { // Given props.files[1].uuid = 'uuid0'; props.files[1].plainName = 'file'; // When - const error = createOrUpdateBatch(props); + const error = await createOrUpdateBatch(props); // Then expect(error).toBeUndefined(); expect({ ...db.prepare('SELECT COUNT(*) FROM drive_file').get() }).toStrictEqual({ 'COUNT(*)': 449 }); @@ -85,11 +85,11 @@ describe('create-or-update-batch', () => { expect({ ...db.prepare('SELECT COUNT(*) FROM drive_file WHERE uuid = ?').get('uuid1') }).toStrictEqual({ 'COUNT(*)': 0 }); }); - it('should return UNKNOWN when error is thrown', () => { + it('should return UNKNOWN when error is thrown', async () => { // Given props.files.push({} as any); // When - const error = createOrUpdateBatch(props); + const error = await createOrUpdateBatch(props); // Then expect(error?.code).toBe('UNKNOWN'); expect({ ...db.prepare('SELECT COUNT(*) FROM drive_file').get() }).toStrictEqual({ 'COUNT(*)': 400 }); diff --git a/src/infra/sqlite/services/file/create-or-update-batch.ts b/src/infra/sqlite/services/file/create-or-update-batch.ts index 5912dd0da..9d6a04d0b 100644 --- a/src/infra/sqlite/services/file/create-or-update-batch.ts +++ b/src/infra/sqlite/services/file/create-or-update-batch.ts @@ -10,7 +10,7 @@ type Props = { files: DriveFile[]; }; -export function createOrUpdateBatch({ files }: Props) { +export async function createOrUpdateBatch({ files }: Props) { if (files.length === 0) return; try { @@ -39,15 +39,18 @@ export function createOrUpdateBatch({ files }: Props) { }); } db.exec('COMMIT'); + + /** + * v2.6.9 Daniel Jiménez + * Since node:sqlite is synchronous it will block the main thread if we try to create + * a lot of files. By awaiting this promise we release the main thread and wait the + * next iteration of the event loop. + */ + await new Promise((resolve) => setImmediate(resolve)); } } catch (error) { db.exec('ROLLBACK'); - logger.error({ - msg: 'Error batch creating or updating files', - count: files.length, - error, - }); - + logger.error({ msg: 'Error batch creating or updating files', count: files.length, error }); return new SqliteError('UNKNOWN', error); } } diff --git a/src/infra/sqlite/services/folder/create-or-update-batch.test.ts b/src/infra/sqlite/services/folder/create-or-update-batch.test.ts index cbdde05d6..a2a987a0e 100644 --- a/src/infra/sqlite/services/folder/create-or-update-batch.test.ts +++ b/src/infra/sqlite/services/folder/create-or-update-batch.test.ts @@ -34,19 +34,19 @@ describe('create-or-update-batch', () => { }; }); - it('should ignore if no folders', () => { + it('should ignore if no folders', async () => { // Given props.folders = []; // When - const error = createOrUpdateBatch(props); + const error = await createOrUpdateBatch(props); // Then expect(error).toBeUndefined(); expect({ ...db.prepare('SELECT COUNT(*) FROM drive_folder').get() }).toStrictEqual({ 'COUNT(*)': 0 }); }); - it('should insert new folders', () => { + it('should insert new folders', async () => { // When - const error = createOrUpdateBatch(props); + const error = await createOrUpdateBatch(props); // Then expect(error).toBeUndefined(); expect({ ...db.prepare('SELECT COUNT(*) FROM drive_folder').get() }).toStrictEqual({ 'COUNT(*)': 450 }); @@ -64,23 +64,23 @@ describe('create-or-update-batch', () => { }); }); - it('should update existing folders', () => { + it('should update existing folders', async () => { // Given props.folders[1].uuid = 'uuid0'; props.folders[1].plainName = 'folder'; // When - const error = createOrUpdateBatch(props); + const error = await createOrUpdateBatch(props); // Then expect(error).toBeUndefined(); expect({ ...db.prepare('SELECT COUNT(*) FROM drive_folder').get() }).toStrictEqual({ 'COUNT(*)': 449 }); expect({ ...db.prepare('SELECT COUNT(*) FROM drive_folder WHERE uuid = ?').get('uuid1') }).toStrictEqual({ 'COUNT(*)': 0 }); }); - it('should return UNKNOWN when error is thrown', () => { + it('should return UNKNOWN when error is thrown', async () => { // Given props.folders.push({} as any); // When - const error = createOrUpdateBatch(props); + const error = await createOrUpdateBatch(props); // Then expect(error?.code).toBe('UNKNOWN'); expect({ ...db.prepare('SELECT COUNT(*) FROM drive_folder').get() }).toStrictEqual({ 'COUNT(*)': 400 }); diff --git a/src/infra/sqlite/services/folder/create-or-update-batch.ts b/src/infra/sqlite/services/folder/create-or-update-batch.ts index bb0ba9d3a..ffe2ec529 100644 --- a/src/infra/sqlite/services/folder/create-or-update-batch.ts +++ b/src/infra/sqlite/services/folder/create-or-update-batch.ts @@ -10,7 +10,7 @@ type Props = { folders: DriveFolder[]; }; -export function createOrUpdateBatch({ folders }: Props) { +export async function createOrUpdateBatch({ folders }: Props) { if (folders.length === 0) return; try { @@ -35,15 +35,12 @@ export function createOrUpdateBatch({ folders }: Props) { }); } db.exec('COMMIT'); + + await new Promise((resolve) => setImmediate(resolve)); } } catch (error) { db.exec('ROLLBACK'); - logger.error({ - msg: 'Error batch creating or updating folders', - count: folders.length, - error, - }); - + logger.error({ msg: 'Error batch creating or updating folders', count: folders.length, error }); return new SqliteError('UNKNOWN', error); } } diff --git a/tests/vitest/setup.dom.helper.test.ts b/tests/vitest/setup.dom.helper.test.ts index 7fa3069d9..6bc167808 100644 --- a/tests/vitest/setup.dom.helper.test.ts +++ b/tests/vitest/setup.dom.helper.test.ts @@ -1,7 +1,7 @@ import '@testing-library/jest-dom'; import path from 'node:path/posix'; -window.electron = { +globalThis.window.electron = { path, driveGetSyncRoot: vi.fn(), -} as Partial as typeof window.electron; +} as Partial as typeof globalThis.window.electron; diff --git a/tests/vitest/setup.helper.test.ts b/tests/vitest/setup.helper.test.ts index 90dff0f72..d5423d111 100644 --- a/tests/vitest/setup.helper.test.ts +++ b/tests/vitest/setup.helper.test.ts @@ -11,6 +11,8 @@ process.env.NODE_ENV = 'test'; vi.mock(import('@internxt/drive-desktop-core/build/backend')); // We do not want to make network calls vi.mock(import('@/apps/shared/HttpClient/client')); +// We do not want to create the lmdb database +vi.mock(import('lmdb')); vi.mock(import('electron'), () => { const actual = vi.importActual('electron');