diff --git a/package-lock.json b/package-lock.json index 8cf1a30dd..778c011e0 100644 --- a/package-lock.json +++ b/package-lock.json @@ -30,6 +30,7 @@ "jwt-decode": "^3.1.2", "openapi-fetch": "^0.13.4", "openpgp": "5.11.3", + "p-limit": "^7.3.0", "phosphor-react": "^1.4.1", "react": "^17.0.2", "react-circular-progressbar": "^2.1.0", @@ -3606,35 +3607,21 @@ "node_modules/@internxt/drive-desktop-core": { "version": "0.1.13", "resolved": "file:packages/core/internxt-drive-desktop-core-0.1.13.tgz", - "integrity": "sha512-KPHveO7n5nMIER2ACIN9Y3l9rbx3mlD6jflj/hIOUI8xLeHfuNdDoI54XB2E0g9Sc5dflAcikuUplSrP11dbcA==", + "integrity": "sha512-B04f3pOlIy8dPl4bd4hV3aw3WASnCSWmgrWfaRaL0d7jGIz0caFU8v2TyVHjuDBxKHrWzXcVyOBE2ms2VNUMPA==", "license": "MIT", "dependencies": { - "@internxt/sdk": "^1.11.10", + "@internxt/sdk": "^1.16.2", "@phosphor-icons/react": "2.0.9", "@tanstack/react-virtual": "^3.13.12", "check-disk-space": "^3.4.0", "electron-log": "^5.4.1", - "react": "^17.0.2", - "uuid": "^13.0.0" + "react": "^17.0.2" }, "engines": { "node": ">=18.20.8", "npm": ">=10.0.0" } }, - "node_modules/@internxt/drive-desktop-core/node_modules/uuid": { - "version": "13.0.0", - "resolved": "https://registry.npmjs.org/uuid/-/uuid-13.0.0.tgz", - "integrity": "sha512-XQegIaBTVUjSHliKqcnFqYypAd4S+WCYt5NIeRs6w/UAry7z8Y9j5ZwRRL4kzq9U3sD6v+85er9FvkEaBpji2w==", - "funding": [ - "https://github.com/sponsors/broofa", - "https://github.com/sponsors/ctavan" - ], - "license": "MIT", - "bin": { - "uuid": "dist-node/bin/uuid" - } - }, "node_modules/@internxt/eslint-config-internxt": { "version": "1.0.9", "resolved": "https://registry.npmjs.org/@internxt/eslint-config-internxt/-/eslint-config-internxt-1.0.9.tgz", @@ -10703,6 +10690,35 @@ "node": "*" } }, + "node_modules/dir-compare/node_modules/p-limit": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-3.1.0.tgz", + "integrity": "sha512-TYOanM3wGwNGsZN2cVTYPArw454xnXj5qmWF1bEoAc4+cU/ol7GVh7odevjp1FNHduHc3KZMcFduxU5Xc6uJRQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "yocto-queue": "^0.1.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/dir-compare/node_modules/yocto-queue": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/yocto-queue/-/yocto-queue-0.1.0.tgz", + "integrity": "sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/dir-glob": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/dir-glob/-/dir-glob-3.0.1.tgz", @@ -17417,16 +17433,15 @@ } }, "node_modules/p-limit": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-3.1.0.tgz", - "integrity": "sha512-TYOanM3wGwNGsZN2cVTYPArw454xnXj5qmWF1bEoAc4+cU/ol7GVh7odevjp1FNHduHc3KZMcFduxU5Xc6uJRQ==", - "dev": true, + "version": "7.3.0", + "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-7.3.0.tgz", + "integrity": "sha512-7cIXg/Z0M5WZRblrsOla88S4wAK+zOQQWeBYfV3qJuJXMr+LnbYjaadrFaS0JILfEDPVqHyKnZ1Z/1d6J9VVUw==", "license": "MIT", "dependencies": { - "yocto-queue": "^0.1.0" + "yocto-queue": "^1.2.1" }, "engines": { - "node": ">=10" + "node": ">=20" }, "funding": { "url": "https://github.com/sponsors/sindresorhus" @@ -17448,6 +17463,35 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/p-locate/node_modules/p-limit": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-3.1.0.tgz", + "integrity": "sha512-TYOanM3wGwNGsZN2cVTYPArw454xnXj5qmWF1bEoAc4+cU/ol7GVh7odevjp1FNHduHc3KZMcFduxU5Xc6uJRQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "yocto-queue": "^0.1.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/p-locate/node_modules/yocto-queue": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/yocto-queue/-/yocto-queue-0.1.0.tgz", + "integrity": "sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/p-map": { "version": "7.0.4", "resolved": "https://registry.npmjs.org/p-map/-/p-map-7.0.4.tgz", @@ -25069,13 +25113,12 @@ } }, "node_modules/yocto-queue": { - "version": "0.1.0", - "resolved": "https://registry.npmjs.org/yocto-queue/-/yocto-queue-0.1.0.tgz", - "integrity": "sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q==", - "dev": true, + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/yocto-queue/-/yocto-queue-1.2.2.tgz", + "integrity": "sha512-4LCcse/U2MHZ63HAJVE+v71o7yOdIe4cZ70Wpf8D/IyjDKYQLV5GD46B+hSTjJsvV5PztjvHoU580EftxjDZFQ==", "license": "MIT", "engines": { - "node": ">=10" + "node": ">=12.20" }, "funding": { "url": "https://github.com/sponsors/sindresorhus" diff --git a/package.json b/package.json index 2ca6b3ee0..031a0f928 100644 --- a/package.json +++ b/package.json @@ -13,7 +13,7 @@ "build:dll": "cross-env NODE_ENV=development TS_NODE_TRANSPILE_ONLY=true webpack --config .erb/configs/webpack.config.renderer.dev.dll.ts", "package": "rimraf dist && npm run build && electron-builder build --publish never", "init:ci": "npm ci --ignore-scripts && node node_modules/electron/install.js", - "init:e2e": "npm ci --ignore-scripts && node node_modules/electron/install.js && npm run build && npm run rebuild:electron", + "init:e2e": "npm run init:ci && npm run build && npm run rebuild:electron", "init:dev": "npm install && node node_modules/electron/install.js && npm run build:dll && npm run rebuild:electron", "========== Code style ==========": "", "lint": "eslint src --ext .ts,.tsx --max-warnings 276", @@ -180,6 +180,7 @@ "jwt-decode": "^3.1.2", "openapi-fetch": "^0.13.4", "openpgp": "5.11.3", + "p-limit": "^7.3.0", "phosphor-react": "^1.4.1", "react": "^17.0.2", "react-circular-progressbar": "^2.1.0", diff --git a/packages/core/internxt-drive-desktop-core-0.1.13.tgz b/packages/core/internxt-drive-desktop-core-0.1.13.tgz index db479d79e..ed216d3e0 100644 Binary files a/packages/core/internxt-drive-desktop-core-0.1.13.tgz and b/packages/core/internxt-drive-desktop-core-0.1.13.tgz differ diff --git a/packages/core/package-lock.json b/packages/core/package-lock.json index 7fa638b7a..68111a086 100644 --- a/packages/core/package-lock.json +++ b/packages/core/package-lock.json @@ -9,7 +9,7 @@ "version": "0.1.13", "license": "MIT", "dependencies": { - "@internxt/sdk": "^1.15.10", + "@internxt/sdk": "^1.16.2", "@phosphor-icons/react": "2.0.9", "@tanstack/react-virtual": "^3.13.12", "check-disk-space": "^3.4.0", @@ -854,13 +854,12 @@ "license": "BSD-3-Clause" }, "node_modules/@internxt/sdk": { - "version": "1.15.10", - "resolved": "https://registry.npmjs.org/@internxt/sdk/-/sdk-1.15.10.tgz", - "integrity": "sha512-ao5Bd64EmO3KFaUoYlJBXD747NBE1M3qiarzigjuD4cEdFmUEjs2/T/iah/F8KBYxqA9KN3WXYKWEHkj8Ssbrw==", + "version": "1.16.2", + "resolved": "https://registry.npmjs.org/@internxt/sdk/-/sdk-1.16.2.tgz", + "integrity": "sha512-hTtOxR94v1MsFbV2eX1CNDf5L7BNoGFScn+SNDipzT17PY0LhHm2vVOa1ncRFjqR0FUBqGi3Hpm+T8/4aQefxQ==", "license": "MIT", "dependencies": { - "axios": "1.15.0", - "internxt-crypto": "1.0.2" + "axios": "^1.16.0" } }, "node_modules/@isaacs/balanced-match": { @@ -982,88 +981,6 @@ "@jridgewell/sourcemap-codec": "^1.4.14" } }, - "node_modules/@noble/ciphers": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/@noble/ciphers/-/ciphers-2.2.0.tgz", - "integrity": "sha512-Z6pjIZ/8IJcCGzb2S/0Px5J81yij85xASuk1teLNeg75bfT07MV3a/O2Mtn1I2se43k3lkVEcFaR10N4cgQcZA==", - "license": "MIT", - "engines": { - "node": ">= 20.19.0" - }, - "funding": { - "url": "https://paulmillr.com/funding/" - } - }, - "node_modules/@noble/curves": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/@noble/curves/-/curves-2.2.0.tgz", - "integrity": "sha512-T/BoHgFXirb0ENSPBquzX0rcjXeM6Lo892a2jlYJkqk83LqZx0l1Of7DzlKJ6jkpvMrkHSnAcgb5JegL8SeIkQ==", - "license": "MIT", - "dependencies": { - "@noble/hashes": "2.2.0" - }, - "engines": { - "node": ">= 20.19.0" - }, - "funding": { - "url": "https://paulmillr.com/funding/" - } - }, - "node_modules/@noble/hashes": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/@noble/hashes/-/hashes-2.2.0.tgz", - "integrity": "sha512-IYqDGiTXab6FniAgnSdZwgWbomxpy9FtYvLKs7wCUs2a8RkITG+DFGO1DM9cr+E3/RgADRpFjrKVaJ1z6sjtEg==", - "license": "MIT", - "engines": { - "node": ">= 20.19.0" - }, - "funding": { - "url": "https://paulmillr.com/funding/" - } - }, - "node_modules/@noble/post-quantum": { - "version": "0.5.4", - "resolved": "https://registry.npmjs.org/@noble/post-quantum/-/post-quantum-0.5.4.tgz", - "integrity": "sha512-leww0zzIirrvwaYMPI9fj6aRIlA/c6Y0/lifQQ1YOOyHEr0MNH3yYpjXeiVG+tWdPps4XxGclFWX2INPO3Yo5w==", - "license": "MIT", - "dependencies": { - "@noble/curves": "~2.0.0", - "@noble/hashes": "~2.0.0" - }, - "engines": { - "node": ">= 20.19.0" - }, - "funding": { - "url": "https://paulmillr.com/funding/" - } - }, - "node_modules/@noble/post-quantum/node_modules/@noble/curves": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/@noble/curves/-/curves-2.0.1.tgz", - "integrity": "sha512-vs1Az2OOTBiP4q0pwjW5aF0xp9n4MxVrmkFBxc6EKZc6ddYx5gaZiAsZoq0uRRXWbi3AT/sBqn05eRPtn1JCPw==", - "license": "MIT", - "dependencies": { - "@noble/hashes": "2.0.1" - }, - "engines": { - "node": ">= 20.19.0" - }, - "funding": { - "url": "https://paulmillr.com/funding/" - } - }, - "node_modules/@noble/post-quantum/node_modules/@noble/hashes": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/@noble/hashes/-/hashes-2.0.1.tgz", - "integrity": "sha512-XlOlEbQcE9fmuXxrVTXCTlG2nlRXa9Rj3rr5Ue/+tX+nmkgbX720YHh0VR3hBF9xDvwnb8D2shVGOwNx+ulArw==", - "license": "MIT", - "engines": { - "node": ">= 20.19.0" - }, - "funding": { - "url": "https://paulmillr.com/funding/" - } - }, "node_modules/@nodelib/fs.scandir": { "version": "2.1.5", "resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz", @@ -1496,28 +1413,6 @@ "dev": true, "license": "MIT" }, - "node_modules/@scure/base": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/@scure/base/-/base-2.2.0.tgz", - "integrity": "sha512-b8XEupJibegiXV+tDUseI8oLQc8ei3d/4Jkb2RpbHh3MfE054ov3uIz2dhFkB3FI8iwYkEh0gGCApkrYggkPNg==", - "license": "MIT", - "funding": { - "url": "https://paulmillr.com/funding/" - } - }, - "node_modules/@scure/bip39": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/@scure/bip39/-/bip39-2.2.0.tgz", - "integrity": "sha512-T/Bj/YvYMNkIPq6EENO6/rcs2e7qTNuyoUXf0KBFDmp0ZDu0H2X4Lq6yC3i0c8PcWkov5EbW+yQZZbdMmk154A==", - "license": "MIT", - "dependencies": { - "@noble/hashes": "2.2.0", - "@scure/base": "2.2.0" - }, - "funding": { - "url": "https://paulmillr.com/funding/" - } - }, "node_modules/@sindresorhus/is": { "version": "4.6.0", "resolved": "https://registry.npmjs.org/@sindresorhus/is/-/is-4.6.0.tgz", @@ -2569,12 +2464,12 @@ } }, "node_modules/axios": { - "version": "1.15.0", - "resolved": "https://registry.npmjs.org/axios/-/axios-1.15.0.tgz", - "integrity": "sha512-wWyJDlAatxk30ZJer+GeCWS209sA42X+N5jU2jy6oHTp7ufw8uzUTVFBX9+wTfAlhiJXGS0Bq7X6efruWjuK9Q==", + "version": "1.16.0", + "resolved": "https://registry.npmjs.org/axios/-/axios-1.16.0.tgz", + "integrity": "sha512-6hp5CwvTPlN2A31g5dxnwAX0orzM7pmCRDLnZSX772mv8WDqICwFjowHuPs04Mc8deIld1+ejhtaMn5vp6b+1w==", "license": "MIT", "dependencies": { - "follow-redirects": "^1.15.11", + "follow-redirects": "^1.16.0", "form-data": "^4.0.5", "proxy-from-env": "^2.1.0" } @@ -4549,34 +4444,6 @@ "dev": true, "license": "ISC" }, - "node_modules/flexsearch": { - "version": "0.8.212", - "resolved": "https://registry.npmjs.org/flexsearch/-/flexsearch-0.8.212.tgz", - "integrity": "sha512-wSyJr1GUWoOOIISRu+X2IXiOcVfg9qqBRyCPRUdLMIGJqPzMo+jMRlvE83t14v1j0dRMEaBbER/adQjp6Du2pw==", - "funding": [ - { - "type": "github", - "url": "https://github.com/ts-thomas" - }, - { - "type": "paypal", - "url": "https://www.paypal.com/donate/?hosted_button_id=GEVR88FC9BWRW" - }, - { - "type": "opencollective", - "url": "https://opencollective.com/flexsearch" - }, - { - "type": "patreon", - "url": "https://patreon.com/user?u=96245532" - }, - { - "type": "liberapay", - "url": "https://liberapay.com/ts-thomas" - } - ], - "license": "Apache-2.0" - }, "node_modules/follow-redirects": { "version": "1.16.0", "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.16.0.tgz", @@ -5166,12 +5033,6 @@ "url": "https://github.com/sponsors/ljharb" } }, - "node_modules/hash-wasm": { - "version": "4.12.0", - "resolved": "https://registry.npmjs.org/hash-wasm/-/hash-wasm-4.12.0.tgz", - "integrity": "sha512-+/2B2rYLb48I/evdOIhP+K/DD2ca2fgBjp6O+GBEnCDk2e4rpeXIK8GvIyRPjTezgmWn9gmKwkQjjx6BtqDHVQ==", - "license": "MIT" - }, "node_modules/hasown": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/hasown/-/hasown-2.0.2.tgz", @@ -5232,27 +5093,6 @@ "node": ">=10.19.0" } }, - "node_modules/husky": { - "version": "9.1.7", - "resolved": "https://registry.npmjs.org/husky/-/husky-9.1.7.tgz", - "integrity": "sha512-5gs5ytaNjBrh5Ow3zrvdUUY+0VxIuWVL4i9irt6friV+BqdCfmV11CQTWMiBYWHbXhco+J1kHfTOUkePhCDvMA==", - "license": "MIT", - "bin": { - "husky": "bin.js" - }, - "engines": { - "node": ">=18" - }, - "funding": { - "url": "https://github.com/sponsors/typicode" - } - }, - "node_modules/idb": { - "version": "8.0.3", - "resolved": "https://registry.npmjs.org/idb/-/idb-8.0.3.tgz", - "integrity": "sha512-LtwtVyVYO5BqRvcsKuB2iUMnHwPVByPCXFXOpuU96IZPPoPN6xjOGxZQ74pgSVVLQWtUOYgyeL4GE98BY5D3wg==", - "license": "ISC" - }, "node_modules/ignore": { "version": "7.0.5", "resolved": "https://registry.npmjs.org/ignore/-/ignore-7.0.5.tgz", @@ -5358,36 +5198,6 @@ "node": ">= 0.4" } }, - "node_modules/internxt-crypto": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/internxt-crypto/-/internxt-crypto-1.0.2.tgz", - "integrity": "sha512-F9PuXci0eU1wlgDwqEbGR7hVDNS0MX8VNh/W+pdpR4ZsEsjRDBrOD2g1DvViR2woCxPiu1AW9Wwekpw2YVKfnA==", - "dependencies": { - "@noble/ciphers": "^2.1.1", - "@noble/curves": "^2.0.1", - "@noble/hashes": "^2.0.1", - "@noble/post-quantum": "^0.5.2", - "@scure/bip39": "^2.0.1", - "flexsearch": "^0.8.205", - "hash-wasm": "^4.12.0", - "husky": "^9.1.7", - "idb": "^8.0.3", - "uuid": "^13.0.0" - } - }, - "node_modules/internxt-crypto/node_modules/uuid": { - "version": "13.0.0", - "resolved": "https://registry.npmjs.org/uuid/-/uuid-13.0.0.tgz", - "integrity": "sha512-XQegIaBTVUjSHliKqcnFqYypAd4S+WCYt5NIeRs6w/UAry7z8Y9j5ZwRRL4kzq9U3sD6v+85er9FvkEaBpji2w==", - "funding": [ - "https://github.com/sponsors/broofa", - "https://github.com/sponsors/ctavan" - ], - "license": "MIT", - "bin": { - "uuid": "dist-node/bin/uuid" - } - }, "node_modules/is-array-buffer": { "version": "3.0.5", "resolved": "https://registry.npmjs.org/is-array-buffer/-/is-array-buffer-3.0.5.tgz", diff --git a/packages/core/package.json b/packages/core/package.json index a5f2be04e..26eb7aa5a 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -58,7 +58,7 @@ "vitest-mock-extended": "^3.1.0" }, "dependencies": { - "@internxt/sdk": "^1.15.10", + "@internxt/sdk": "^1.16.2", "@phosphor-icons/react": "2.0.9", "@tanstack/react-virtual": "^3.13.12", "check-disk-space": "^3.4.0", diff --git a/src/apps/sync-engine/refresh-item-placeholders.ts b/src/apps/sync-engine/refresh-item-placeholders.ts index ad3d1497a..c672267c2 100644 --- a/src/apps/sync-engine/refresh-item-placeholders.ts +++ b/src/apps/sync-engine/refresh-item-placeholders.ts @@ -1,3 +1,4 @@ +import pLimit from 'p-limit'; import { loadInMemoryPaths } from '@/backend/features/remote-sync/sync-items-by-checkpoint/load-in-memory-paths'; import { traverse } from '@/context/virtual-drive/items/application/Traverser'; import { measurePerfomance } from '@/core/utils/measure-performance'; @@ -29,7 +30,7 @@ export async function refreshItemPlaceholders({ ctx, isFirstExecution }: Props) const currentFolder = { absolutePath: ctx.rootPath, uuid: ctx.rootUuid }; - await traverse({ ctx, currentFolder, database, fileExplorer, isFirstExecution }); + await traverse({ ctx, currentFolder, database, fileExplorer, isFirstExecution, limit: pLimit(20) }); }); ctx.logger.debug({ msg: 'Finish refresh placeholders in seconds', time }); diff --git a/src/backend/features/remote-sync/file-explorer/check-if-modified.infra.test.ts b/src/backend/features/remote-sync/file-explorer/check-if-modified.infra.test.ts index 7aa0f6f07..d4a7f4a90 100644 --- a/src/backend/features/remote-sync/file-explorer/check-if-modified.infra.test.ts +++ b/src/backend/features/remote-sync/file-explorer/check-if-modified.infra.test.ts @@ -44,7 +44,8 @@ describe('check-if-modified', () => { }, local: { path, - stats: { mtime: new Date('2000-01-01'), size: 7 }, + mtime: new Date('2000-01-01'), + size: 7, }, }); diff --git a/src/backend/features/remote-sync/file-explorer/check-if-modified.test.ts b/src/backend/features/remote-sync/file-explorer/check-if-modified.test.ts index 607691231..45fd0fb85 100644 --- a/src/backend/features/remote-sync/file-explorer/check-if-modified.test.ts +++ b/src/backend/features/remote-sync/file-explorer/check-if-modified.test.ts @@ -23,7 +23,7 @@ describe('check-if-modified', () => { props = mockProps({ local: { path: 'localPath' as AbsolutePath, - stats: { size: 512 }, + size: 512, }, remote: { absolutePath: 'remotePath' as AbsolutePath, @@ -36,7 +36,7 @@ describe('check-if-modified', () => { it('should not sync when file sizes are equal', async () => { // Given - props.local.stats.size = 1024; + props.local.size = 1024; // When await checkIfModified(props); // Then @@ -45,7 +45,7 @@ describe('check-if-modified', () => { it('should sync when remote file is newer', async () => { // Given - props.local.stats.mtime = new Date('2000-01-01'); + props.local.mtime = new Date('2000-01-01'); // When await checkIfModified(props); // Then @@ -55,7 +55,7 @@ describe('check-if-modified', () => { describe('what happens when local file is newer', () => { beforeEach(() => { - props.local.stats.mtime = new Date('2000-01-03'); + props.local.mtime = new Date('2000-01-03'); }); it('should not sync when is not first execution', async () => { diff --git a/src/backend/features/remote-sync/file-explorer/check-if-modified.ts b/src/backend/features/remote-sync/file-explorer/check-if-modified.ts index 2a793a0e9..5825235a3 100644 --- a/src/backend/features/remote-sync/file-explorer/check-if-modified.ts +++ b/src/backend/features/remote-sync/file-explorer/check-if-modified.ts @@ -1,4 +1,3 @@ -import { Stats } from 'node:fs'; import { ExtendedDriveFile } from '@/apps/main/database/entities/DriveFile'; import { SyncContext } from '@/apps/sync-engine/config'; import { AbsolutePath } from '@/context/local/localFile/infrastructure/AbsolutePath'; @@ -10,15 +9,15 @@ import { Drive } from '../../drive'; type Props = { ctx: SyncContext; remote: ExtendedDriveFile; - local: { path: AbsolutePath; stats: Stats }; + local: { path: AbsolutePath; mtime: Date; size: number }; isFirstExecution: boolean; }; export async function checkIfModified({ ctx, remote, local, isFirstExecution }: Props) { const path = remote.absolutePath; - const localSize = local.stats.size; - const localDate = local.stats.mtime; + const localSize = local.size; + const localDate = local.mtime; const remoteSize = remote.size; const remoteDate = new Date(remote.updatedAt); diff --git a/src/backend/features/remote-sync/file-explorer/delete-item-placeholder.ts b/src/backend/features/remote-sync/file-explorer/delete-item-placeholder.ts index e0dfdf8c0..0117f6f29 100644 --- a/src/backend/features/remote-sync/file-explorer/delete-item-placeholder.ts +++ b/src/backend/features/remote-sync/file-explorer/delete-item-placeholder.ts @@ -25,7 +25,6 @@ export async function deleteItemPlaceholder({ ctx, type, remote, locals }: Props * so instead of deleting the placeholder, we are going to send the item to the trash * so the user can decide whether to delete it or recover it. */ - ctx.logger.error({ msg: 'Path does not match when deleting placeholder', remotePath: remote.absolutePath, diff --git a/src/backend/features/remote-sync/file-explorer/update-file-placeholder.test.ts b/src/backend/features/remote-sync/file-explorer/update-file-placeholder.test.ts index 17df1bc69..749221f82 100644 --- a/src/backend/features/remote-sync/file-explorer/update-file-placeholder.test.ts +++ b/src/backend/features/remote-sync/file-explorer/update-file-placeholder.test.ts @@ -69,10 +69,7 @@ describe('update-file-placeholder', () => { // Given props.isFirstExecution = true; props.files = new Map([ - [ - 'uuid' as FileUuid, - { path: 'remotePath' as AbsolutePath, stats: { size: 1024 }, placeholder: { pinState: PinState.AlwaysLocal, onDiskSize: 512 } }, - ], + ['uuid' as FileUuid, { path: 'remotePath' as AbsolutePath, size: 1024, pinState: PinState.AlwaysLocal, onDiskSize: 512 }], ]); // When await updateFilePlaceholder(props as any); diff --git a/src/backend/features/remote-sync/file-explorer/update-file-placeholder.ts b/src/backend/features/remote-sync/file-explorer/update-file-placeholder.ts index 4fd7f615d..96b029c7a 100644 --- a/src/backend/features/remote-sync/file-explorer/update-file-placeholder.ts +++ b/src/backend/features/remote-sync/file-explorer/update-file-placeholder.ts @@ -47,8 +47,8 @@ export async function updateFilePlaceholder({ ctx, remote, files, isFirstExecuti // the opposite, that the user wants to continue after opening the app again. We will keep the first use // case since it's easier to implement, consumes less resources and in case the user wants to hydrate the // folder he can perform the action again. - const { onDiskSize } = local.placeholder; - if (local.placeholder.pinState === PinState.AlwaysLocal && onDiskSize < size) { + const { onDiskSize } = local; + if (local.pinState === PinState.AlwaysLocal && onDiskSize < size) { ctx.logger.debug({ msg: 'File stuck in hydrated state', onDiskSize, size, path }); await Addon.setPinState({ path, pinState: PinState.Unspecified }); } diff --git a/src/backend/features/remote-sync/sync-items-by-checkpoint/load-in-memory-paths.test.ts b/src/backend/features/remote-sync/sync-items-by-checkpoint/load-in-memory-paths.test.ts index 6274a9bfb..5a10ef049 100644 --- a/src/backend/features/remote-sync/sync-items-by-checkpoint/load-in-memory-paths.test.ts +++ b/src/backend/features/remote-sync/sync-items-by-checkpoint/load-in-memory-paths.test.ts @@ -17,11 +17,11 @@ describe('load-in-memory-paths', () => { // Given statReaddirMock .mockResolvedValueOnce({ - files: [{ path: abs('/file1') }, { path: abs('/file2') }], + files: [{ path: abs('/file1') }, { path: abs('/file2'), stats: {} }], folders: [{ path: abs('/folder1') }, { path: abs('/folder2') }], }) .mockResolvedValueOnce({ - files: [{ path: abs('/file3') }], + files: [{ path: abs('/file3'), stats: {} }], folders: [], }); diff --git a/src/backend/features/remote-sync/sync-items-by-checkpoint/load-in-memory-paths.ts b/src/backend/features/remote-sync/sync-items-by-checkpoint/load-in-memory-paths.ts index 305eab137..d95543d93 100644 --- a/src/backend/features/remote-sync/sync-items-by-checkpoint/load-in-memory-paths.ts +++ b/src/backend/features/remote-sync/sync-items-by-checkpoint/load-in-memory-paths.ts @@ -1,42 +1,48 @@ -import { Stats } from 'node:fs'; +import pLimit from 'p-limit'; import { FileUuid } from '@/apps/main/database/entities/DriveFile'; import { FolderUuid } from '@/apps/main/database/entities/DriveFolder'; import { SyncContext } from '@/apps/sync-engine/config'; import { AbsolutePath } from '@/context/local/localFile/infrastructure/AbsolutePath'; import { statReaddir } from '@/infra/file-system/services/stat-readdir'; import { NodeWin } from '@/infra/node-win/node-win.module'; -import { FilePlaceholder } from '@/infra/node-win/services/get-file-info'; +import { PinState } from '@/node-win/types/placeholder.type'; -export type FileExplorerFiles = Map; +// Store just the required properties to reduce RAM usage +export type FileExplorerFiles = Map; export type FileExplorerFolders = Map; -type Props = { - ctx: SyncContext; -}; - -export async function loadInMemoryPaths({ ctx }: Props) { +export async function loadInMemoryPaths({ ctx }: { ctx: SyncContext }) { const files: FileExplorerFiles = new Map(); const folders: FileExplorerFolders = new Map(); + const limit = pLimit(20); async function walk(parentPath: AbsolutePath) { const items = await statReaddir({ folder: parentPath }); - const filePromises = items.files.map(async ({ path, stats }) => { - const { data: placeholder } = await NodeWin.getFileInfo({ path }); - if (placeholder) { - files.set(placeholder.uuid, { stats, path, placeholder }); - } - }); - - const folderPromises = items.folders.map(async ({ path }) => { + await Promise.all( + items.files.map(({ path, stats }) => + limit(async () => { + const { data: placeholder } = await NodeWin.getFileInfo({ path }); + if (placeholder) { + files.set(placeholder.uuid, { + path, + mtime: stats.mtime, + size: stats.size, + onDiskSize: placeholder.onDiskSize, + pinState: placeholder.pinState, + }); + } + }), + ), + ); + + for (const { path } of items.folders) { const { data: placeholder } = await NodeWin.getFolderInfo({ ctx, path }); if (placeholder) { folders.set(placeholder.uuid, { path }); await walk(path); } - }); - - await Promise.all(filePromises.concat(folderPromises)); + } } await walk(ctx.rootPath); diff --git a/src/context/virtual-drive/items/application/Traverser.test.ts b/src/context/virtual-drive/items/application/Traverser.test.ts index 56b379c4c..8abc0b0d1 100644 --- a/src/context/virtual-drive/items/application/Traverser.test.ts +++ b/src/context/virtual-drive/items/application/Traverser.test.ts @@ -1,3 +1,4 @@ +import pLimit from 'p-limit'; import { calls, mockProps, partialSpyOn } from 'tests/vitest/utils.helper.test'; import { FolderUuid } from '@/apps/main/database/entities/DriveFolder'; import * as deleteItemPlaceholder from '@/backend/features/remote-sync/file-explorer/delete-item-placeholder'; @@ -16,6 +17,7 @@ describe('traverse', () => { beforeEach(() => { props = mockProps({ ctx: { abortController: new AbortController() }, + limit: pLimit(20), currentFolder: { absolutePath: abs('/drive'), uuid: 'root' as FolderUuid }, fileExplorer: {}, database: { @@ -73,9 +75,9 @@ describe('traverse', () => { calls(updateFolderPlaceholderMock).toMatchObject([ { remote: { absolutePath: '/drive/parent1' } }, - { remote: { absolutePath: '/drive/child1' } }, { remote: { absolutePath: '/drive/parent1/parent2' } }, { remote: { absolutePath: '/drive/parent1/child2' } }, + { remote: { absolutePath: '/drive/child1' } }, ]); calls(updateFilePlaceholderMock).toMatchObject([ @@ -99,10 +101,10 @@ describe('traverse', () => { calls(updateFolderPlaceholderMock).toMatchObject([ { remote: { absolutePath: '/drive/parent1' } }, - { remote: { absolutePath: '/drive/child1' } }, { remote: { absolutePath: '/drive/parent1/parent2' } }, - { remote: { absolutePath: '/drive/parent1/child2' } }, { remote: { absolutePath: '/drive/parent1/parent2/child3' } }, + { remote: { absolutePath: '/drive/parent1/child2' } }, + { remote: { absolutePath: '/drive/child1' } }, ]); calls(updateFilePlaceholderMock).toMatchObject([ diff --git a/src/context/virtual-drive/items/application/Traverser.ts b/src/context/virtual-drive/items/application/Traverser.ts index d8dfb088f..0d44b9d92 100644 --- a/src/context/virtual-drive/items/application/Traverser.ts +++ b/src/context/virtual-drive/items/application/Traverser.ts @@ -1,3 +1,4 @@ +import { LimitFunction } from 'p-limit'; import { SimpleDriveFile } from '@/apps/main/database/entities/DriveFile'; import { ExtendedDriveFolder, SimpleDriveFolder } from '@/apps/main/database/entities/DriveFolder'; import { ProcessSyncContext } from '@/apps/sync-engine/config'; @@ -16,26 +17,31 @@ type Props = { fileExplorer: FileExplorer; currentFolder: Pick; isFirstExecution: boolean; + limit: LimitFunction; }; -export async function traverse({ ctx, database, fileExplorer, currentFolder, isFirstExecution }: Props) { +export async function traverse({ ctx, database, fileExplorer, currentFolder, isFirstExecution, limit }: Props) { if (ctx.abortController.signal.aborted) return; const filesInThisFolder = database.files.filter((file) => file.parentUuid === currentFolder.uuid); const foldersInThisFolder = database.folders.filter((folder) => folder.parentUuid === currentFolder.uuid); - const filePromises = filesInThisFolder.map(async (file) => { - const absolutePath = join(currentFolder.absolutePath, file.name); - const remote = { ...file, absolutePath }; - - if (file.status === 'DELETED' || file.status === 'TRASHED') { - await deleteItemPlaceholder({ ctx, type: 'file', remote, locals: fileExplorer.files }); - } else { - await updateFilePlaceholder({ ctx, remote, files: fileExplorer.files, isFirstExecution }); - } - }); - - const folderPromises = foldersInThisFolder.map(async (folder) => { + await Promise.all( + filesInThisFolder.map((file) => + limit(async () => { + const absolutePath = join(currentFolder.absolutePath, file.name); + const remote = { ...file, absolutePath }; + + if (file.status === 'DELETED' || file.status === 'TRASHED') { + await deleteItemPlaceholder({ ctx, type: 'file', remote, locals: fileExplorer.files }); + } else { + await updateFilePlaceholder({ ctx, remote, files: fileExplorer.files, isFirstExecution }); + } + }), + ), + ); + + for (const folder of foldersInThisFolder) { const absolutePath = join(currentFolder.absolutePath, folder.name); const remote = { ...folder, absolutePath }; @@ -44,10 +50,8 @@ export async function traverse({ ctx, database, fileExplorer, currentFolder, isF } else { const success = await updateFolderPlaceholder({ ctx, remote, folders: fileExplorer.folders }); if (success) { - await traverse({ ctx, database, fileExplorer, currentFolder: remote, isFirstExecution }); + await traverse({ ctx, database, fileExplorer, currentFolder: remote, isFirstExecution, limit }); } } - }); - - await Promise.all(filePromises.concat(folderPromises)); + } } diff --git a/src/infra/node-win/services/get-file-info.ts b/src/infra/node-win/services/get-file-info.ts index cd440bb39..1ab939283 100644 --- a/src/infra/node-win/services/get-file-info.ts +++ b/src/infra/node-win/services/get-file-info.ts @@ -3,8 +3,6 @@ import { FileUuid } from '@/apps/main/database/entities/DriveFile'; import { FilePlaceholderId } from '@/context/virtual-drive/files/domain/PlaceholderId'; import { Addon } from '@/node-win/addon-wrapper'; -export type FilePlaceholder = NonNullable>['data']>; - export class GetFileInfoError extends Error { constructor( public readonly code: 'NOT_A_PLACEHOLDER' | 'UNKNOWN', diff --git a/src/infra/sqlite/services/file/create-or-update-batch.test.ts b/src/infra/sqlite/services/file/create-or-update-batch.test.ts index 38f7ca2df..fe882a2f0 100644 --- a/src/infra/sqlite/services/file/create-or-update-batch.test.ts +++ b/src/infra/sqlite/services/file/create-or-update-batch.test.ts @@ -38,19 +38,19 @@ describe('create-or-update-batch', () => { }; }); - it('should ignore if no files', () => { + it('should ignore if no files', async () => { // Given props.files = []; // When - const error = createOrUpdateBatch(props); + const error = await createOrUpdateBatch(props); // Then expect(error).toBeUndefined(); expect({ ...db.prepare('SELECT COUNT(*) FROM drive_file').get() }).toStrictEqual({ 'COUNT(*)': 0 }); }); - it('should insert new files', () => { + it('should insert new files', async () => { // When - const error = createOrUpdateBatch(props); + const error = await createOrUpdateBatch(props); // Then expect(error).toBeUndefined(); expect({ ...db.prepare('SELECT COUNT(*) FROM drive_file').get() }).toStrictEqual({ 'COUNT(*)': 450 }); @@ -72,12 +72,12 @@ describe('create-or-update-batch', () => { }); }); - it('should update existing files', () => { + it('should update existing files', async () => { // Given props.files[1].uuid = 'uuid0'; props.files[1].plainName = 'file'; // When - const error = createOrUpdateBatch(props); + const error = await createOrUpdateBatch(props); // Then expect(error).toBeUndefined(); expect({ ...db.prepare('SELECT COUNT(*) FROM drive_file').get() }).toStrictEqual({ 'COUNT(*)': 449 }); @@ -85,11 +85,11 @@ describe('create-or-update-batch', () => { expect({ ...db.prepare('SELECT COUNT(*) FROM drive_file WHERE uuid = ?').get('uuid1') }).toStrictEqual({ 'COUNT(*)': 0 }); }); - it('should return UNKNOWN when error is thrown', () => { + it('should return UNKNOWN when error is thrown', async () => { // Given props.files.push({} as any); // When - const error = createOrUpdateBatch(props); + const error = await createOrUpdateBatch(props); // Then expect(error?.code).toBe('UNKNOWN'); expect({ ...db.prepare('SELECT COUNT(*) FROM drive_file').get() }).toStrictEqual({ 'COUNT(*)': 400 }); diff --git a/src/infra/sqlite/services/file/create-or-update-batch.ts b/src/infra/sqlite/services/file/create-or-update-batch.ts index 5912dd0da..9d6a04d0b 100644 --- a/src/infra/sqlite/services/file/create-or-update-batch.ts +++ b/src/infra/sqlite/services/file/create-or-update-batch.ts @@ -10,7 +10,7 @@ type Props = { files: DriveFile[]; }; -export function createOrUpdateBatch({ files }: Props) { +export async function createOrUpdateBatch({ files }: Props) { if (files.length === 0) return; try { @@ -39,15 +39,18 @@ export function createOrUpdateBatch({ files }: Props) { }); } db.exec('COMMIT'); + + /** + * v2.6.9 Daniel Jiménez + * Since node:sqlite is synchronous it will block the main thread if we try to create + * a lot of files. By awaiting this promise we release the main thread and wait the + * next iteration of the event loop. + */ + await new Promise((resolve) => setImmediate(resolve)); } } catch (error) { db.exec('ROLLBACK'); - logger.error({ - msg: 'Error batch creating or updating files', - count: files.length, - error, - }); - + logger.error({ msg: 'Error batch creating or updating files', count: files.length, error }); return new SqliteError('UNKNOWN', error); } } diff --git a/src/infra/sqlite/services/folder/create-or-update-batch.test.ts b/src/infra/sqlite/services/folder/create-or-update-batch.test.ts index cbdde05d6..a2a987a0e 100644 --- a/src/infra/sqlite/services/folder/create-or-update-batch.test.ts +++ b/src/infra/sqlite/services/folder/create-or-update-batch.test.ts @@ -34,19 +34,19 @@ describe('create-or-update-batch', () => { }; }); - it('should ignore if no folders', () => { + it('should ignore if no folders', async () => { // Given props.folders = []; // When - const error = createOrUpdateBatch(props); + const error = await createOrUpdateBatch(props); // Then expect(error).toBeUndefined(); expect({ ...db.prepare('SELECT COUNT(*) FROM drive_folder').get() }).toStrictEqual({ 'COUNT(*)': 0 }); }); - it('should insert new folders', () => { + it('should insert new folders', async () => { // When - const error = createOrUpdateBatch(props); + const error = await createOrUpdateBatch(props); // Then expect(error).toBeUndefined(); expect({ ...db.prepare('SELECT COUNT(*) FROM drive_folder').get() }).toStrictEqual({ 'COUNT(*)': 450 }); @@ -64,23 +64,23 @@ describe('create-or-update-batch', () => { }); }); - it('should update existing folders', () => { + it('should update existing folders', async () => { // Given props.folders[1].uuid = 'uuid0'; props.folders[1].plainName = 'folder'; // When - const error = createOrUpdateBatch(props); + const error = await createOrUpdateBatch(props); // Then expect(error).toBeUndefined(); expect({ ...db.prepare('SELECT COUNT(*) FROM drive_folder').get() }).toStrictEqual({ 'COUNT(*)': 449 }); expect({ ...db.prepare('SELECT COUNT(*) FROM drive_folder WHERE uuid = ?').get('uuid1') }).toStrictEqual({ 'COUNT(*)': 0 }); }); - it('should return UNKNOWN when error is thrown', () => { + it('should return UNKNOWN when error is thrown', async () => { // Given props.folders.push({} as any); // When - const error = createOrUpdateBatch(props); + const error = await createOrUpdateBatch(props); // Then expect(error?.code).toBe('UNKNOWN'); expect({ ...db.prepare('SELECT COUNT(*) FROM drive_folder').get() }).toStrictEqual({ 'COUNT(*)': 400 }); diff --git a/src/infra/sqlite/services/folder/create-or-update-batch.ts b/src/infra/sqlite/services/folder/create-or-update-batch.ts index bb0ba9d3a..ffe2ec529 100644 --- a/src/infra/sqlite/services/folder/create-or-update-batch.ts +++ b/src/infra/sqlite/services/folder/create-or-update-batch.ts @@ -10,7 +10,7 @@ type Props = { folders: DriveFolder[]; }; -export function createOrUpdateBatch({ folders }: Props) { +export async function createOrUpdateBatch({ folders }: Props) { if (folders.length === 0) return; try { @@ -35,15 +35,12 @@ export function createOrUpdateBatch({ folders }: Props) { }); } db.exec('COMMIT'); + + await new Promise((resolve) => setImmediate(resolve)); } } catch (error) { db.exec('ROLLBACK'); - logger.error({ - msg: 'Error batch creating or updating folders', - count: folders.length, - error, - }); - + logger.error({ msg: 'Error batch creating or updating folders', count: folders.length, error }); return new SqliteError('UNKNOWN', error); } } diff --git a/tests/vitest/setup.dom.helper.test.ts b/tests/vitest/setup.dom.helper.test.ts index 7fa3069d9..6bc167808 100644 --- a/tests/vitest/setup.dom.helper.test.ts +++ b/tests/vitest/setup.dom.helper.test.ts @@ -1,7 +1,7 @@ import '@testing-library/jest-dom'; import path from 'node:path/posix'; -window.electron = { +globalThis.window.electron = { path, driveGetSyncRoot: vi.fn(), -} as Partial as typeof window.electron; +} as Partial as typeof globalThis.window.electron;