From 47b620d5864e70c49f527af27f4b58732c0d29ea Mon Sep 17 00:00:00 2001 From: jingkarqi Date: Sat, 23 May 2026 23:51:04 +0800 Subject: [PATCH 1/5] test: add vitest harness --- package-lock.json | 385 +++++++++++++++++++++++++++++++++++++++++++++- package.json | 7 +- vitest.config.ts | 16 ++ 3 files changed, 405 insertions(+), 3 deletions(-) create mode 100644 vitest.config.ts diff --git a/package-lock.json b/package-lock.json index d69ba360..a79627f5 100644 --- a/package-lock.json +++ b/package-lock.json @@ -46,7 +46,8 @@ "typescript": "^6.0.3", "vite": "^8.0.10", "vite-plugin-electron": "^0.28.8", - "vite-plugin-electron-renderer": "^0.14.6" + "vite-plugin-electron-renderer": "^0.14.6", + "vitest": "^4.1.7" } }, "node_modules/@derhuerst/http-basic": { @@ -1675,6 +1676,13 @@ "node": ">=18.0.0" } }, + "node_modules/@jridgewell/sourcemap-codec": { + "version": "1.5.5", + "resolved": "https://registry.npmjs.org/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.5.5.tgz", + "integrity": "sha512-cYQ9310grqxueWbl+WuIUIaiUaDcj7WOq5fVhEljNVgRfOUhY9fy2zTvfoqWsnebh8Sl70VScFbICvJnLKB0Og==", + "dev": true, + "license": "MIT" + }, "node_modules/@malept/cross-spawn-promise": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/@malept/cross-spawn-promise/-/cross-spawn-promise-2.0.0.tgz", @@ -2425,6 +2433,13 @@ "url": "https://github.com/sindresorhus/is?sponsor=1" } }, + "node_modules/@standard-schema/spec": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@standard-schema/spec/-/spec-1.1.0.tgz", + "integrity": "sha512-l2aFy5jALhniG5HgqrD6jXLi/rUWrKvqN/qJx6yoJsgKhblVd+iqqU4RCXavm/jPityDo5TCvKMnpjKnOriy0w==", + "dev": true, + "license": "MIT" + }, "node_modules/@szmarczak/http-timer": { "version": "4.0.6", "resolved": "https://registry.npmjs.org/@szmarczak/http-timer/-/http-timer-4.0.6.tgz", @@ -2470,6 +2485,17 @@ "@types/responselike": "^1.0.0" } }, + "node_modules/@types/chai": { + "version": "5.2.3", + "resolved": "https://registry.npmjs.org/@types/chai/-/chai-5.2.3.tgz", + "integrity": "sha512-Mw558oeA9fFbv65/y4mHtXDs9bPnFMZAL/jxdPFUpOHHIXX91mcgEHbS5Lahr+pwZFR8A7GQleRWeI6cGFC2UA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/deep-eql": "*", + "assertion-error": "^2.0.1" + } + }, "node_modules/@types/debug": { "version": "4.1.13", "resolved": "https://registry.npmjs.org/@types/debug/-/debug-4.1.13.tgz", @@ -2479,6 +2505,13 @@ "@types/ms": "*" } }, + "node_modules/@types/deep-eql": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/@types/deep-eql/-/deep-eql-4.0.2.tgz", + "integrity": "sha512-c9h9dVVMigMPc4bwTvC5dxqtqJZwQPePsWjPlpSOnojbor6pGqdk541lfA7AqFQr5pB1BRdq0juY9db81BwyFw==", + "dev": true, + "license": "MIT" + }, "node_modules/@types/estree": { "version": "1.0.8", "resolved": "https://registry.npmjs.org/@types/estree/-/estree-1.0.8.tgz", @@ -2653,6 +2686,119 @@ } } }, + "node_modules/@vitest/expect": { + "version": "4.1.7", + "resolved": "https://registry.npmjs.org/@vitest/expect/-/expect-4.1.7.tgz", + "integrity": "sha512-1R+tw0ortHEbZDGMymm+pN7/AFQ/RkFFdtd7EN+VBpynKmLbP8A3rpEXdshBJ7+8hQ9zBJh/i1s0yKNtxAnU7w==", + "dev": true, + "license": "MIT", + "dependencies": { + "@standard-schema/spec": "^1.1.0", + "@types/chai": "^5.2.2", + "@vitest/spy": "4.1.7", + "@vitest/utils": "4.1.7", + "chai": "^6.2.2", + "tinyrainbow": "^3.1.0" + }, + "funding": { + "url": "https://opencollective.com/vitest" + } + }, + "node_modules/@vitest/mocker": { + "version": "4.1.7", + "resolved": "https://registry.npmjs.org/@vitest/mocker/-/mocker-4.1.7.tgz", + "integrity": "sha512-vY7nuamKgfvpA1Koa3oYIw/k7D6kZnpGyNMZW8loow2bsBYla1TFdqTaXncWdRn4pgwNs+90RhnXhJScDwQeJA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@vitest/spy": "4.1.7", + "estree-walker": "^3.0.3", + "magic-string": "^0.30.21" + }, + "funding": { + "url": "https://opencollective.com/vitest" + }, + "peerDependencies": { + "msw": "^2.4.9", + "vite": "^6.0.0 || ^7.0.0 || ^8.0.0" + }, + "peerDependenciesMeta": { + "msw": { + "optional": true + }, + "vite": { + "optional": true + } + } + }, + "node_modules/@vitest/pretty-format": { + "version": "4.1.7", + "resolved": "https://registry.npmjs.org/@vitest/pretty-format/-/pretty-format-4.1.7.tgz", + "integrity": "sha512-umgCarTOYQWIaDMvGDRZij+6b9oVeLIyJzfN+AS88e0ZOU3QTgNNSTtjQOpcvWr3np1N0j4WgZj+sb3oYBDscw==", + "dev": true, + "license": "MIT", + "dependencies": { + "tinyrainbow": "^3.1.0" + }, + "funding": { + "url": "https://opencollective.com/vitest" + } + }, + "node_modules/@vitest/runner": { + "version": "4.1.7", + "resolved": "https://registry.npmjs.org/@vitest/runner/-/runner-4.1.7.tgz", + "integrity": "sha512-BapjmAQ2aI78WdMEfeUWivnfVzB+VPGwWRQcJE0OUq7qEeEcBsCSf+0T5iREBNE5nBb4wA5Ya0W6IA+sghdEFw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@vitest/utils": "4.1.7", + "pathe": "^2.0.3" + }, + "funding": { + "url": "https://opencollective.com/vitest" + } + }, + "node_modules/@vitest/snapshot": { + "version": "4.1.7", + "resolved": "https://registry.npmjs.org/@vitest/snapshot/-/snapshot-4.1.7.tgz", + "integrity": "sha512-ZacLzja+TmJeZ1h14xW2FB/WpeimUD3haBXQPyJqxvo8jQTmfeA8zv58mtjN2C7EHXZDYVcVYdYmAxjkWVvKCw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@vitest/pretty-format": "4.1.7", + "@vitest/utils": "4.1.7", + "magic-string": "^0.30.21", + "pathe": "^2.0.3" + }, + "funding": { + "url": "https://opencollective.com/vitest" + } + }, + "node_modules/@vitest/spy": { + "version": "4.1.7", + "resolved": "https://registry.npmjs.org/@vitest/spy/-/spy-4.1.7.tgz", + "integrity": "sha512-kbkI5LMWakyuTIvs6fUJ5qdIVb1XVKsYJAT4OJ938cHMROYMSfmoQdZy0aaAnjbbc8F61vkoTqz/Az+/HiIu5Q==", + "dev": true, + "license": "MIT", + "funding": { + "url": "https://opencollective.com/vitest" + } + }, + "node_modules/@vitest/utils": { + "version": "4.1.7", + "resolved": "https://registry.npmjs.org/@vitest/utils/-/utils-4.1.7.tgz", + "integrity": "sha512-T532WBu791cBxJlCl6SO+J14l81DQx6uQHm1bQbmCDY7nqlEIgkza/UFnSBNaUtSf41unldDFjdOBYEQC4b5Hw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@vitest/pretty-format": "4.1.7", + "convert-source-map": "^2.0.0", + "tinyrainbow": "^3.1.0" + }, + "funding": { + "url": "https://opencollective.com/vitest" + } + }, "node_modules/@vscode/sudo-prompt": { "version": "9.3.2", "resolved": "https://registry.npmjs.org/@vscode/sudo-prompt/-/sudo-prompt-9.3.2.tgz", @@ -3060,6 +3206,16 @@ "node": ">=0.8" } }, + "node_modules/assertion-error": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/assertion-error/-/assertion-error-2.0.1.tgz", + "integrity": "sha512-Izi8RQcffqCeNVgFigKli1ssklIbpHnCYc6AknXGYoB6grJqyeby7jv12JUQgmTAnIDnbck1uxksT4dzN3PWBA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=12" + } + }, "node_modules/astral-regex": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/astral-regex/-/astral-regex-2.0.0.tgz", @@ -3501,6 +3657,16 @@ "url": "https://github.com/sponsors/wooorm" } }, + "node_modules/chai": { + "version": "6.2.2", + "resolved": "https://registry.npmjs.org/chai/-/chai-6.2.2.tgz", + "integrity": "sha512-NUPRluOfOiTKBKvWPtSD4PhFvWCqOi0BGStNWs57X9js7XGTprSmFoz5F0tWhR4WPjNeR9jXqdC7/UpSJTnlRg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=18" + } + }, "node_modules/chainsaw": { "version": "0.1.0", "resolved": "https://registry.npmjs.org/chainsaw/-/chainsaw-0.1.0.tgz", @@ -3857,6 +4023,13 @@ "integrity": "sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==", "license": "MIT" }, + "node_modules/convert-source-map": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/convert-source-map/-/convert-source-map-2.0.0.tgz", + "integrity": "sha512-Kvp459HrV2FEJ1CAsi1Ku+MY3kasH19TFykTz2xWmMeq6bk2NU3XXvfJ+Q61m0xktWwt+1HSYf3JZsTms3aRJg==", + "dev": true, + "license": "MIT" + }, "node_modules/cookie": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/cookie/-/cookie-1.1.1.tgz", @@ -4730,6 +4903,13 @@ "node": ">= 0.4" } }, + "node_modules/es-module-lexer": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/es-module-lexer/-/es-module-lexer-2.1.0.tgz", + "integrity": "sha512-n27zTYMjYu1aj4MjCWzSP7G9r75utsaoc8m61weK+W8JMBGGQybd43GstCXZ3WNmSFtGT9wi59qQTW6mhTR5LQ==", + "dev": true, + "license": "MIT" + }, "node_modules/es-object-atoms": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/es-object-atoms/-/es-object-atoms-1.1.1.tgz", @@ -4843,6 +5023,16 @@ "url": "https://opencollective.com/unified" } }, + "node_modules/estree-walker": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/estree-walker/-/estree-walker-3.0.3.tgz", + "integrity": "sha512-7RUKfXgSMMkzt6ZuXmqapOurLGPPfgj6l9uRZ7lRGolvk0y2yocc35LdcxKC5PQZdn2DMqioAQ2NoWcrTKmm6g==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/estree": "^1.0.0" + } + }, "node_modules/exceljs": { "version": "4.4.0", "resolved": "https://registry.npmjs.org/exceljs/-/exceljs-4.4.0.tgz", @@ -4863,6 +5053,16 @@ "node": ">=8.3.0" } }, + "node_modules/expect-type": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/expect-type/-/expect-type-1.3.0.tgz", + "integrity": "sha512-knvyeauYhqjOYvQ66MznSMs83wmHrCycNEN6Ao+2AeYEfxUIkuiVxdEa1qlGEPK+We3n0THiDciYSsCcgW/DoA==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": ">=12.0.0" + } + }, "node_modules/exponential-backoff": { "version": "3.1.3", "resolved": "https://registry.npmjs.org/exponential-backoff/-/exponential-backoff-3.1.3.tgz", @@ -6493,6 +6693,16 @@ "react": "^16.5.1 || ^17.0.0 || ^18.0.0 || ^19.0.0" } }, + "node_modules/magic-string": { + "version": "0.30.21", + "resolved": "https://registry.npmjs.org/magic-string/-/magic-string-0.30.21.tgz", + "integrity": "sha512-vd2F4YUyEXKGcLHoq+TEyCjxueSeHnFxyyjNp80yg0XV4vUhnDer/lvvlqM/arB5bXQN5K2/3oinyCRyx8T2CQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jridgewell/sourcemap-codec": "^1.5.5" + } + }, "node_modules/make-fetch-happen": { "version": "14.0.3", "resolved": "https://registry.npmjs.org/make-fetch-happen/-/make-fetch-happen-14.0.3.tgz", @@ -7819,6 +8029,17 @@ "node": ">= 0.4" } }, + "node_modules/obug": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/obug/-/obug-2.1.1.tgz", + "integrity": "sha512-uTqF9MuPraAQ+IsnPf366RG4cP9RtUi7MLO1N3KEc+wb0a6yKpeL0lmk2IB1jY5KHPAlTc6T/JRdC/YqxHNwkQ==", + "dev": true, + "funding": [ + "https://github.com/sponsors/sxzz", + "https://opencollective.com/debug" + ], + "license": "MIT" + }, "node_modules/once": { "version": "1.4.0", "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", @@ -7993,6 +8214,13 @@ "dev": true, "license": "ISC" }, + "node_modules/pathe": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/pathe/-/pathe-2.0.3.tgz", + "integrity": "sha512-WUjGcAqP1gQacoQe+OBJsFA7Ld4DyXuUIjZ5cc75cLHvJ7dtNsTugphxIADwspS+AraAUePCKrSVtPLFj/F88w==", + "dev": true, + "license": "MIT" + }, "node_modules/pe-library": { "version": "0.4.1", "resolved": "https://registry.npmjs.org/pe-library/-/pe-library-0.4.1.tgz", @@ -8905,6 +9133,13 @@ "win32" ] }, + "node_modules/siginfo": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/siginfo/-/siginfo-2.0.0.tgz", + "integrity": "sha512-ybx0WO1/8bSBLEWXZvEd7gMW3Sn3JFlW3TvX1nREbDLRNQNaeNN8WK0meBwPdAaOI7TtRRRJn/Es1zhrrCHu7g==", + "dev": true, + "license": "ISC" + }, "node_modules/signal-exit": { "version": "3.0.7", "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-3.0.7.tgz", @@ -9059,6 +9294,13 @@ "node": "^18.17.0 || >=20.5.0" } }, + "node_modules/stackback": { + "version": "0.0.2", + "resolved": "https://registry.npmjs.org/stackback/-/stackback-0.0.2.tgz", + "integrity": "sha512-1XMJE5fQo1jGH6Y/7ebnwPOBEkIEnT4QF32d5R1+VXdXveM0IBMJt8zfaxX1P3QhVwrYe+576+jkANtSS2mBbw==", + "dev": true, + "license": "MIT" + }, "node_modules/stat-mode": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/stat-mode/-/stat-mode-1.0.0.tgz", @@ -9069,6 +9311,13 @@ "node": ">= 6" } }, + "node_modules/std-env": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/std-env/-/std-env-4.1.0.tgz", + "integrity": "sha512-Rq7ybcX2RuC55r9oaPVEW7/xu3tj8u4GeBYHBWCychFtzMIr86A7e3PPEBPT37sHStKX3+TiX/Fr/ACmJLVlLQ==", + "dev": true, + "license": "MIT" + }, "node_modules/string_decoder": { "version": "1.3.0", "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz", @@ -9363,6 +9612,23 @@ "integrity": "sha512-qVtvMxeXbVej0cQWKqVSSAHmKZEHAvxdF8HEUBFWts8h+xEo5m/lEiPakuyZ3BnCBjOD8i24kzNOiOLLgsSxhA==", "license": "MIT" }, + "node_modules/tinybench": { + "version": "2.9.0", + "resolved": "https://registry.npmjs.org/tinybench/-/tinybench-2.9.0.tgz", + "integrity": "sha512-0+DUvqWMValLmha6lr4kD8iAMK1HzV0/aKnCtWb9v9641TnP/MFb7Pc2bxoxQjTXAErryXVgUOfv2YqNllqGeg==", + "dev": true, + "license": "MIT" + }, + "node_modules/tinyexec": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/tinyexec/-/tinyexec-1.1.2.tgz", + "integrity": "sha512-dAqSqE/RabpBKI8+h26GfLq6Vb3JVXs30XYQjdMjaj/c2tS8IYYMbIzP599KtRj7c57/wYApb3QjgRgXmrCukA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=18" + } + }, "node_modules/tinyglobby": { "version": "0.2.16", "resolved": "https://registry.npmjs.org/tinyglobby/-/tinyglobby-0.2.16.tgz", @@ -9380,6 +9646,16 @@ "url": "https://github.com/sponsors/SuperchupuDev" } }, + "node_modules/tinyrainbow": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/tinyrainbow/-/tinyrainbow-3.1.0.tgz", + "integrity": "sha512-Bf+ILmBgretUrdJxzXM0SgXLZ3XfiaUuOj/IKQHuTXip+05Xn+uyEYdVg0kYDipTBcLrCVyUzAPz7QmArb0mmw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=14.0.0" + } + }, "node_modules/tmp": { "version": "0.2.5", "resolved": "https://registry.npmjs.org/tmp/-/tmp-0.2.5.tgz", @@ -9854,6 +10130,96 @@ "dev": true, "license": "MIT" }, + "node_modules/vitest": { + "version": "4.1.7", + "resolved": "https://registry.npmjs.org/vitest/-/vitest-4.1.7.tgz", + "integrity": "sha512-flYyaFd2CgoCoU+0UKt3pxksgC+S02iTDN0n3LtqaMeXsI9SBcdNujc2k0DeFLzUn/0k538yNjOSdwgCqcrwJA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@vitest/expect": "4.1.7", + "@vitest/mocker": "4.1.7", + "@vitest/pretty-format": "4.1.7", + "@vitest/runner": "4.1.7", + "@vitest/snapshot": "4.1.7", + "@vitest/spy": "4.1.7", + "@vitest/utils": "4.1.7", + "es-module-lexer": "^2.0.0", + "expect-type": "^1.3.0", + "magic-string": "^0.30.21", + "obug": "^2.1.1", + "pathe": "^2.0.3", + "picomatch": "^4.0.3", + "std-env": "^4.0.0-rc.1", + "tinybench": "^2.9.0", + "tinyexec": "^1.0.2", + "tinyglobby": "^0.2.15", + "tinyrainbow": "^3.1.0", + "vite": "^6.0.0 || ^7.0.0 || ^8.0.0", + "why-is-node-running": "^2.3.0" + }, + "bin": { + "vitest": "vitest.mjs" + }, + "engines": { + "node": "^20.0.0 || ^22.0.0 || >=24.0.0" + }, + "funding": { + "url": "https://opencollective.com/vitest" + }, + "peerDependencies": { + "@edge-runtime/vm": "*", + "@opentelemetry/api": "^1.9.0", + "@types/node": "^20.0.0 || ^22.0.0 || >=24.0.0", + "@vitest/browser-playwright": "4.1.7", + "@vitest/browser-preview": "4.1.7", + "@vitest/browser-webdriverio": "4.1.7", + "@vitest/coverage-istanbul": "4.1.7", + "@vitest/coverage-v8": "4.1.7", + "@vitest/ui": "4.1.7", + "happy-dom": "*", + "jsdom": "*", + "vite": "^6.0.0 || ^7.0.0 || ^8.0.0" + }, + "peerDependenciesMeta": { + "@edge-runtime/vm": { + "optional": true + }, + "@opentelemetry/api": { + "optional": true + }, + "@types/node": { + "optional": true + }, + "@vitest/browser-playwright": { + "optional": true + }, + "@vitest/browser-preview": { + "optional": true + }, + "@vitest/browser-webdriverio": { + "optional": true + }, + "@vitest/coverage-istanbul": { + "optional": true + }, + "@vitest/coverage-v8": { + "optional": true + }, + "@vitest/ui": { + "optional": true + }, + "happy-dom": { + "optional": true + }, + "jsdom": { + "optional": true + }, + "vite": { + "optional": false + } + } + }, "node_modules/wcwidth": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/wcwidth/-/wcwidth-1.0.1.tgz", @@ -9895,6 +10261,23 @@ "node": ">= 8" } }, + "node_modules/why-is-node-running": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/why-is-node-running/-/why-is-node-running-2.3.0.tgz", + "integrity": "sha512-hUrmaWBdVDcxvYqnyh09zunKzROWjbZTiNy8dBEjkS7ehEDQibXJ7XvlmtbwuTclUiIyN+CyXQD4Vmko8fNm8w==", + "dev": true, + "license": "MIT", + "dependencies": { + "siginfo": "^2.0.0", + "stackback": "0.0.2" + }, + "bin": { + "why-is-node-running": "cli.js" + }, + "engines": { + "node": ">=8" + } + }, "node_modules/wrap-ansi": { "version": "7.0.0", "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz", diff --git a/package.json b/package.json index 1534c31a..d51d0081 100644 --- a/package.json +++ b/package.json @@ -17,10 +17,12 @@ "rebuild": "electron-rebuild", "dev": "node scripts/prepare-electron-runtime.cjs && vite", "typecheck": "tsc --noEmit", + "test": "vitest run --config vitest.config.ts", "build": "tsc && vite build && electron-builder", "preview": "vite preview", "electron:dev": "node scripts/prepare-electron-runtime.cjs && vite --mode electron", - "electron:build": "npm run build" + "electron:build": "npm run build", + "build:rust-exporter": "node scripts/build-rust-exporter.cjs" }, "dependencies": { "@vscode/sudo-prompt": "^9.3.2", @@ -60,7 +62,8 @@ "typescript": "^6.0.3", "vite": "^8.0.10", "vite-plugin-electron": "^0.28.8", - "vite-plugin-electron-renderer": "^0.14.6" + "vite-plugin-electron-renderer": "^0.14.6", + "vitest": "^4.1.7" }, "pnpm": { "overrides": { diff --git a/vitest.config.ts b/vitest.config.ts new file mode 100644 index 00000000..d3802e6e --- /dev/null +++ b/vitest.config.ts @@ -0,0 +1,16 @@ +import { defineConfig } from 'vitest/config' + +export default defineConfig({ + test: { + environment: 'node', + include: [ + 'electron/**/*.test.ts', + 'src/**/*.test.ts' + ] + }, + resolve: { + alias: { + '@': new URL('./src', import.meta.url).pathname + } + } +}) From e36c16f4d1d33171671ff3e15ea6c9df9bcc8599 Mon Sep 17 00:00:00 2001 From: jingkarqi Date: Sat, 23 May 2026 23:51:08 +0800 Subject: [PATCH 2/5] refactor(chat): share system message formatting --- electron/services/chatService.ts | 30 +--- .../services/systemMessageFormatter.test.ts | 60 +++++++ electron/services/systemMessageFormatter.ts | 157 ++++++++++++++++++ 3 files changed, 223 insertions(+), 24 deletions(-) create mode 100644 electron/services/systemMessageFormatter.test.ts create mode 100644 electron/services/systemMessageFormatter.ts diff --git a/electron/services/chatService.ts b/electron/services/chatService.ts index b827d416..cccb9ff4 100644 --- a/electron/services/chatService.ts +++ b/electron/services/chatService.ts @@ -18,6 +18,10 @@ import { voiceTranscribeService } from './voiceTranscribeService' import { ImageDecryptService } from './imageDecryptService' import { CONTACT_REGION_LOOKUP_DATA } from './contactRegionLookupData' import { LRUCache } from '../utils/LRUCache.js' +import { + cleanSystemMessageContent, + extractReadableSystemMessageText as extractReadableSystemMessageTextValue +} from './systemMessageFormatter' export interface ChatSession { username: string @@ -6812,33 +6816,11 @@ class ChatService { } private cleanSystemMessage(content: string): string { - if (!content) return '[系统消息]' - - const normalized = this.cleanUtf16(this.decodeHtmlEntities(String(content))) - const readableSysmsg = this.extractReadableSystemMessageText(normalized) - if (readableSysmsg) { - return readableSysmsg - } - - // 移除 XML 声明 - let cleaned = normalized.replace(/<\?xml[^?]*\?>/gi, '') - // 移除所有 XML/HTML 标签 - cleaned = cleaned.replace(/<[^>]+>/g, '') - // 移除尾部的数字（如撤回消息后的时间戳） - cleaned = cleaned.replace(/\d+\s*$/, '') - // 清理多余空白 - cleaned = this.stripSenderPrefix(cleaned).replace(/\s+/g, ' ').trim() - return cleaned || '[系统消息]' + return cleanSystemMessageContent(content) } private extractReadableSystemMessageText(content: string): string { - const sysmsgMatch = /]*>([\s\S]*?)<\/sysmsg>/i.exec(content) - const source = sysmsgMatch?.[1] || content - const text = - this.extractXmlValue(source, 'plain') || - this.extractXmlValue(source, 'text') || - '' - return this.stripSenderPrefix(text).replace(/\s+/g, ' ').trim() + return extractReadableSystemMessageTextValue(content) } private stripSenderPrefix(content: string): string { diff --git a/electron/services/systemMessageFormatter.test.ts b/electron/services/systemMessageFormatter.test.ts new file mode 100644 index 00000000..bc1636f0 --- /dev/null +++ b/electron/services/systemMessageFormatter.test.ts @@ -0,0 +1,60 @@ +import { describe, expect, it } from 'vitest' +import { cleanSystemMessageContent, extractReadableSystemMessageText } from './systemMessageFormatter' + +const inviteSysmsg = ` + + + + + + + + + + + + + + + + + + + +` + +const qrcodeJoinSysmsg = ` + + + + + + + + + + + + + + + + + +` + +describe('systemMessageFormatter', () => { + it('expands group system message template variables from link members', () => { + expect(extractReadableSystemMessageText(inviteSysmsg)).toBe('"张三"邀请"李四、王五"加入了群聊') + expect(cleanSystemMessageContent(inviteSysmsg)).toBe('"张三"邀请"李四、王五"加入了群聊') + }) + + it('keeps unknown placeholders instead of deleting useful context', () => { + expect(extractReadableSystemMessageText('')).toBe('$unknown$加入了群聊') + }) + + it('expands QR code join system message template variables', () => { + expect(extractReadableSystemMessageText(qrcodeJoinSysmsg)).toBe('"新成员"通过扫描"分享者"分享的二维码加入群聊') + expect(cleanSystemMessageContent(qrcodeJoinSysmsg)).toBe('"新成员"通过扫描"分享者"分享的二维码加入群聊') + }) +}) diff --git a/electron/services/systemMessageFormatter.ts b/electron/services/systemMessageFormatter.ts new file mode 100644 index 00000000..b67c867c --- /dev/null +++ b/electron/services/systemMessageFormatter.ts @@ -0,0 +1,157 @@ +export function cleanSystemMessageContent(content: string): string { + if (!content) return '[系统消息]' + + const normalized = normalizeSystemMessageContent(content) + const readable = extractReadableSystemMessageText(normalized) + if (readable) return readable + + const revokeMatch = /<\/replacemsg>/i.exec(normalized) + if (revokeMatch) { + return revokeMatch[1].trim() + } + + const title = extractXmlValue(normalized, 'title') + if (title) return title + + return stripSenderPrefix(normalized) + .replace(/]*>/gi, '') + .replace(/<\/?[a-zA-Z0-9_:]+[^>]*>/g, '') + .replace(/\d+\s*$/, '') + .replace(/\s+/g, ' ') + .trim() || '[系统消息]' +} + +export function extractReadableSystemMessageText(content: string): string { + if (!content) return '' + + const normalized = normalizeSystemMessageContent(content) + const source = extractSysmsgBody(normalized) + const template = + extractXmlValue(source, 'plain') || + extractXmlValue(source, 'text') || + extractXmlValue(source, 'template') || + '' + + if (!template) return '' + + return normalizeReadableText(resolveSystemTemplate(template, source)) +} + +export function resolveSystemTemplate(template: string, source: string): string { + const normalizedSource = normalizeSystemMessageContent(source) + const normalizedTemplate = stripCdata(template) + return normalizedTemplate.replace(/\$(\{)?([a-zA-Z0-9_:-]+)(\})?\$/g, (match, _open, varName) => { + const value = resolveTemplateVariable(normalizedSource, String(varName || '')) + return value || match + }).replace(/\$\{([a-zA-Z0-9_:-]+)\}/g, (match, varName) => { + const value = resolveTemplateVariable(normalizedSource, String(varName || '')) + return value || match + }) +} + +function resolveTemplateVariable(source: string, varName: string): string { + if (!varName) return '' + + const linkValue = resolveLinkVariable(source, varName) + if (linkValue) return linkValue + + const directValue = extractXmlValue(source, varName) + if (directValue) return directValue + + return '' +} + +function resolveLinkVariable(source: string, varName: string): string { + const escapedName = escapeRegExp(varName) + const linkRegex = new RegExp(`]*\\bname\\s*=\\s*["']${escapedName}["'])[^>]*>([\\s\\S]*?)<\\/link>`, 'i') + const match = linkRegex.exec(source) + if (!match) return '' + + const body = match[1] || '' + const names = collectMemberNames(body) + if (names.length > 0) return names.join('、') + + return ( + extractXmlValue(body, 'nickname') || + extractXmlValue(body, 'displayname') || + extractXmlValue(body, 'displayName') || + extractXmlValue(body, 'plain') || + extractXmlValue(body, 'username') || + '' + ) +} + +function collectMemberNames(body: string): string[] { + const names: string[] = [] + const seen = new Set() + const add = (value: string) => { + const name = normalizeReadableText(value) + if (!name || seen.has(name)) return + seen.add(name) + names.push(name) + } + + for (const memberMatch of body.matchAll(/]*>([\s\S]*?)<\/member>/gi)) { + const member = memberMatch[0] || memberMatch[1] || '' + add(extractXmlValue(member, 'nickname')) + add(extractXmlValue(member, 'displayname')) + add(extractXmlValue(member, 'displayName')) + if (names.length === 0) add(extractXmlValue(member, 'username')) + } + + for (const attrMatch of body.matchAll(/\b(?:nickname|displayname|displayName)\s*=\s*["']([^"']+)["']/gi)) { + add(attrMatch[1] || '') + } + + return names +} + +function extractSysmsgBody(content: string): string { + const sysmsgMatch = /]*>([\s\S]*?)<\/sysmsg>/i.exec(stripSenderPrefix(content)) + return sysmsgMatch?.[1] || content +} + +function extractXmlValue(xml: string, tagName: string): string { + if (!xml || !tagName) return '' + const regex = new RegExp(`<${escapeRegExp(tagName)}\\b[^>]*>([\\s\\S]*?)<\\/${escapeRegExp(tagName)}>`, 'i') + const match = regex.exec(xml) + if (!match) return '' + return stripCdata(match[1]).trim() +} + +function stripCdata(value: string): string { + return decodeHtmlEntities(String(value || '')) + .replace(//g, '') +} + +function normalizeReadableText(value: string): string { + return stripSenderPrefix(stripCdata(value)) + .replace(/]*>/gi, '') + .replace(/<\/?[a-zA-Z0-9_:]+[^>]*>/g, '') + .replace(/\s+/g, ' ') + .trim() +} + +function normalizeSystemMessageContent(content: string): string { + return decodeHtmlEntities(String(content || '')) + .replace(/<\?xml[^?]*\?>/gi, '') + .replace(/[\x00-\x08\x0B-\x0C\x0E-\x1F\x7F-\x9F]/g, '') +} + +function stripSenderPrefix(content: string): string { + return String(content || '').replace(/^[\s]*([a-zA-Z0-9_@-]+):(?!\/\/)(?:\s*(?:\r?\n|)\s*|\s*)/i, '') +} + +function decodeHtmlEntities(content: string): string { + return content + .replace(/&/g, '&') + .replace(/</g, '<') + .replace(/>/g, '>') + .replace(/"/g, '"') + .replace(/'/g, "'") +} + +function escapeRegExp(value: string): string { + return value.replace(/[.*+?^${}()|[\]\\]/g, '\\$&') +} From 42ab11a16d7de5cc2a2becccb27075d1a0a28a2a Mon Sep 17 00:00:00 2001 From: jingkarqi Date: Sat, 23 May 2026 23:51:18 +0800 Subject: [PATCH 3/5] feat(export): add streaming export engines --- electron/exportWorker.ts | 134 +++- electron/main.ts | 2 + electron/preload.ts | 2 + .../export/exportEngineRouter.test.ts | 42 ++ .../services/export/exportEngineRouter.ts | 91 +++ .../services/export/messageStream.test.ts | 98 +++ electron/services/export/messageStream.ts | 248 +++++++ .../services/export/rustExportBridge.test.ts | 30 + electron/services/export/rustExportBridge.ts | 72 ++ .../services/export/rustStreamingExporter.ts | 656 ++++++++++++++++++ .../services/export/streamingWriters.test.ts | 109 +++ electron/services/export/streamingWriters.ts | 159 +++++ .../export/syntheticLargeExport.test.ts | 86 +++ .../typescriptStreamingExporter.test.ts | 113 +++ .../export/typescriptStreamingExporter.ts | 309 +++++++++ electron/services/exportService.ts | 101 ++- src/types/electron.d.ts | 3 + 17 files changed, 2195 insertions(+), 60 deletions(-) create mode 100644 electron/services/export/exportEngineRouter.test.ts create mode 100644 electron/services/export/exportEngineRouter.ts create mode 100644 electron/services/export/messageStream.test.ts create mode 100644 electron/services/export/messageStream.ts create mode 100644 electron/services/export/rustExportBridge.test.ts create mode 100644 electron/services/export/rustExportBridge.ts create mode 100644 electron/services/export/rustStreamingExporter.ts create mode 100644 electron/services/export/streamingWriters.test.ts create mode 100644 electron/services/export/streamingWriters.ts create mode 100644 electron/services/export/syntheticLargeExport.test.ts create mode 100644 electron/services/export/typescriptStreamingExporter.test.ts create mode 100644 electron/services/export/typescriptStreamingExporter.ts diff --git a/electron/exportWorker.ts b/electron/exportWorker.ts index 60f896e6..ccd54d7c 100644 --- a/electron/exportWorker.ts +++ b/electron/exportWorker.ts @@ -9,6 +9,7 @@ interface ExportWorkerConfig { options?: any taskId?: string dbPath?: string + accountDir?: string decryptKey?: string myWxid?: string imageXorKey?: unknown @@ -137,10 +138,31 @@ if (config.userDataPath) { } process.env.WEFLOW_PROJECT_NAME = process.env.WEFLOW_PROJECT_NAME || 'WeFlow' +function isExportControlInterruption(error: unknown): boolean { + const text = error instanceof Error + ? `${(error as Error & { code?: string }).code || ''} ${error.message}` + : String(error || '') + return ( + text.includes('WEFLOW_EXPORT_STOP_REQUESTED') || + text.includes('WEFLOW_EXPORT_PAUSE_REQUESTED') || + text.includes('导出任务已停止') || + text.includes('导出任务已暂停') + ) +} + async function run() { - const [{ wcdbService }, { exportService }] = await Promise.all([ + const [ + { wcdbService }, + { exportService }, + { chooseExportEngine, getRustExportDisabledReason }, + { exportSessionsWithRustStreaming }, + { canUseTypeScriptStreamingExport, exportSessionsWithTypeScriptStreaming } + ] = await Promise.all([ import('./services/wcdbService'), - import('./services/exportService') + import('./services/exportService'), + import('./services/export/exportEngineRouter'), + import('./services/export/rustStreamingExporter'), + import('./services/export/typescriptStreamingExporter') ]) wcdbService.setPaths(config.resourcesPath || '', config.userDataPath || '') @@ -188,13 +210,115 @@ async function run() { taskControl ) } else { - result = await exportService.exportSessions( + const options = config.options || { format: 'json' } + const requestedEngine = String(options.engine || 'auto') + const resolvedEngine = chooseExportEngine(options) + const rustDisabledReason = getRustExportDisabledReason(options) + const rustProgress = (progress: any) => onProgress({ + ...progress, + exportEngine: 'rust', + exportEngineLabel: 'Rust' + }) + let typeScriptEngineLabel = requestedEngine === 'typescript' + ? 'TypeScript · 手动指定' + : rustDisabledReason + ? `TypeScript · Rust未启用：${rustDisabledReason}` + : 'TypeScript' + const typeScriptProgress = (progress: any) => onProgress({ + ...progress, + exportEngine: 'typescript', + exportEngineLabel: typeScriptEngineLabel + }) + const runTypeScriptExport = async () => exportService.exportSessions( Array.isArray(config.sessionIds) ? config.sessionIds : [], String(config.outputDir || ''), - config.options || { format: 'json' }, - onProgress, + options, + typeScriptProgress, taskControl ) + const runTypeScriptStreamingExport = async () => exportSessionsWithTypeScriptStreaming({ + source: wcdbService, + sessionIds: Array.isArray(config.sessionIds) ? config.sessionIds : [], + outputDir: String(config.outputDir || ''), + options, + accountDir: String(config.accountDir || config.dbPath || ''), + decryptKey: String(config.decryptKey || ''), + cleanedMyWxid: String(config.myWxid || ''), + onProgress: typeScriptProgress, + control: taskControl + }) + const runRustStreamingExport = async () => exportSessionsWithRustStreaming({ + source: wcdbService, + sessionIds: Array.isArray(config.sessionIds) ? config.sessionIds : [], + outputDir: String(config.outputDir || ''), + options, + accountDir: String(config.accountDir || config.dbPath || ''), + decryptKey: String(config.decryptKey || ''), + cleanedMyWxid: String(config.myWxid || ''), + resourcesPath: String(config.resourcesPath || ''), + onProgress: rustProgress, + control: taskControl + }) + + if (resolvedEngine === 'rust') { + try { + onProgress({ + current: 0, + total: 100, + currentSession: '', + currentSessionId: '', + phase: 'preparing', + phaseLabel: 'Rust 引擎准备导出', + exportEngine: 'rust', + exportEngineLabel: 'Rust' + }) + result = await runRustStreamingExport() + } catch (error) { + if (requestedEngine === 'rust' || isExportControlInterruption(error)) { + throw error + } + const fallbackReason = error instanceof Error ? error.message : String(error) + typeScriptEngineLabel = `TypeScript · Rust回退：${fallbackReason.slice(0, 160)}` + console.warn(`[exportWorker] Rust exporter unavailable, falling back to TypeScript: ${fallbackReason}`) + onProgress({ + current: 0, + total: 100, + currentSession: '', + currentSessionId: '', + phase: 'preparing', + phaseLabel: `Rust 引擎不可用，已回退 TypeScript：${fallbackReason.slice(0, 160)}`, + exportEngine: 'typescript', + exportEngineLabel: typeScriptEngineLabel + }) + result = await runTypeScriptExport() + } + } else { + if (requestedEngine === 'auto' && rustDisabledReason) { + onProgress({ + current: 0, + total: 100, + currentSession: '', + currentSessionId: '', + phase: 'preparing', + phaseLabel: `TypeScript 引擎导出（Rust 未启用：${rustDisabledReason}）` + }) + } + if (requestedEngine === 'typescript') { + onProgress({ + current: 0, + total: 100, + currentSession: '', + currentSessionId: '', + phase: 'preparing', + phaseLabel: canUseTypeScriptStreamingExport(options) + ? 'TypeScript 流式引擎准备导出' + : 'TypeScript 引擎准备导出' + }) + } + result = requestedEngine === 'typescript' && canUseTypeScriptStreamingExport(options) + ? await runTypeScriptStreamingExport() + : await runTypeScriptExport() + } } flushProgress() diff --git a/electron/main.ts b/electron/main.ts index cf80daf9..4c51f002 100644 --- a/electron/main.ts +++ b/electron/main.ts @@ -3191,6 +3191,7 @@ function registerIpcHandlers() { const dbPath = String(cfg.get('dbPath') || '').trim() const decryptKey = String(cfg.get('decryptKey') || '').trim() const myWxid = String(cfg.getMyWxidCleaned() || '').trim() + const accountDir = cfg.getAccountDir(dbPath, myWxid) || '' const imageKeys = cfg.getImageKeysForCurrentWxid() const resourcesPath = app.isPackaged ? join(process.resourcesPath, 'resources') @@ -3207,6 +3208,7 @@ function registerIpcHandlers() { options, taskId, dbPath, + accountDir, decryptKey, myWxid, imageXorKey: imageKeys.xorKey, diff --git a/electron/preload.ts b/electron/preload.ts index bb175c04..423332ef 100644 --- a/electron/preload.ts +++ b/electron/preload.ts @@ -494,6 +494,8 @@ contextBridge.exposeInMainWorld('electronAPI', { total: number currentSession: string currentSessionId?: string + exportEngine?: 'rust' | 'typescript' + exportEngineLabel?: string phase: string phaseProgress?: number phaseTotal?: number diff --git a/electron/services/export/exportEngineRouter.test.ts b/electron/services/export/exportEngineRouter.test.ts new file mode 100644 index 00000000..3e0f8957 --- /dev/null +++ b/electron/services/export/exportEngineRouter.test.ts @@ -0,0 +1,42 @@ +import { describe, expect, it } from 'vitest' +import { chooseExportEngine, getRustExportDisabledReason, isRustSupportedFormat, isTextOnlyExport } from './exportEngineRouter' + +describe('export engine routing', () => { + it('routes auto text-only supported formats to rust', () => { + expect(chooseExportEngine({ format: 'txt' })).toBe('rust') + expect(chooseExportEngine({ format: 'html', contentType: 'text' })).toBe('rust') + expect(chooseExportEngine({ format: 'json' })).toBe('rust') + expect(chooseExportEngine({ format: 'weclone' })).toBe('rust') + expect(chooseExportEngine({ format: 'chatlab-jsonl' })).toBe('rust') + }) + + it('routes unsupported formats and media-heavy options to typescript in auto mode', () => { + expect(chooseExportEngine({ format: 'excel' })).toBe('typescript') + expect(chooseExportEngine({ format: 'txt', exportMedia: true, exportImages: true })).toBe('typescript') + expect(chooseExportEngine({ format: 'json', exportMedia: true, exportImages: true })).toBe('typescript') + expect(chooseExportEngine({ format: 'html', exportAvatars: true })).toBe('typescript') + expect(chooseExportEngine({ format: 'txt', exportVoiceAsText: true })).toBe('typescript') + expect(chooseExportEngine({ format: 'txt', contentType: 'image' })).toBe('typescript') + }) + + it('honors explicit engine requests', () => { + expect(chooseExportEngine({ format: 'excel', engine: 'rust' })).toBe('rust') + expect(chooseExportEngine({ format: 'txt', engine: 'typescript' })).toBe('typescript') + expect(chooseExportEngine({ format: 'txt', engine: 'auto' })).toBe('rust') + }) + + it('exposes narrow predicates for bridge fallback decisions', () => { + expect(isRustSupportedFormat('txt')).toBe(true) + expect(isRustSupportedFormat('json')).toBe(true) + expect(isRustSupportedFormat('chatlab')).toBe(false) + expect(isTextOnlyExport({ format: 'txt' })).toBe(true) + expect(isTextOnlyExport({ format: 'txt', exportFiles: true })).toBe(false) + }) + + it('explains why rust is disabled', () => { + expect(getRustExportDisabledReason({ format: 'chatlab' })).toContain('暂不支持 Rust') + expect(getRustExportDisabledReason({ format: 'json', exportMedia: true })).toBe('媒体导出已开启') + expect(getRustExportDisabledReason({ format: 'txt', exportAvatars: true })).toBe('头像导出已开启') + expect(getRustExportDisabledReason({ format: 'txt' })).toBeNull() + }) +}) diff --git a/electron/services/export/exportEngineRouter.ts b/electron/services/export/exportEngineRouter.ts new file mode 100644 index 00000000..226fb67c --- /dev/null +++ b/electron/services/export/exportEngineRouter.ts @@ -0,0 +1,91 @@ +export type ExportEngine = 'auto' | 'typescript' | 'rust' + +export type ExportFormat = + | 'chatlab' + | 'chatlab-jsonl' + | 'json' + | 'arkme-json' + | 'html' + | 'txt' + | 'excel' + | 'weclone' + | 'sql' + +export interface ExportEngineOptions { + format: ExportFormat | string + engine?: ExportEngine + contentType?: 'text' | 'voice' | 'image' | 'video' | 'emoji' | 'file' | string + exportMedia?: boolean + exportAvatars?: boolean + exportImages?: boolean + exportVoices?: boolean + exportVideos?: boolean + exportEmojis?: boolean + exportFiles?: boolean + exportVoiceAsText?: boolean +} + +const RUST_SUPPORTED_FORMATS = new Set([ + 'json', + 'txt', + 'html', + 'weclone', + 'chatlab-jsonl' +]) + +const TYPESCRIPT_STREAMING_SUPPORTED_FORMATS = new Set([ + 'txt', + 'html', + 'chatlab-jsonl' +]) + +export type ResolvedExportEngine = Exclude + +export function isRustSupportedFormat(format: ExportFormat | string): boolean { + return RUST_SUPPORTED_FORMATS.has(format) +} + +export function isTypeScriptStreamingSupportedFormat(format: ExportFormat | string): boolean { + return TYPESCRIPT_STREAMING_SUPPORTED_FORMATS.has(format) +} + +export function isTextOnlyExport(options: ExportEngineOptions): boolean { + if (options.contentType && options.contentType !== 'text') return false + if (options.exportMedia === true) return false + if (options.exportAvatars === true) return false + if (options.exportImages === true) return false + if (options.exportVoices === true) return false + if (options.exportVideos === true) return false + if (options.exportEmojis === true) return false + if (options.exportFiles === true) return false + if (options.exportVoiceAsText === true) return false + return true +} + +export function canUseRustExportEngine(options: ExportEngineOptions): boolean { + return isRustSupportedFormat(options.format) && isTextOnlyExport(options) +} + +export function canUseTypeScriptStreamingEngine(options: ExportEngineOptions): boolean { + return isTypeScriptStreamingSupportedFormat(options.format) && isTextOnlyExport(options) +} + +export function getRustExportDisabledReason(options: ExportEngineOptions): string | null { + if (!isRustSupportedFormat(options.format)) return `格式 ${options.format} 暂不支持 Rust` + if (options.contentType && options.contentType !== 'text') return `内容类型 ${options.contentType} 不是纯文本` + if (options.exportMedia === true) return '媒体导出已开启' + if (options.exportAvatars === true) return '头像导出已开启' + if (options.exportImages === true) return '图片导出已开启' + if (options.exportVoices === true) return '语音导出已开启' + if (options.exportVideos === true) return '视频导出已开启' + if (options.exportEmojis === true) return '表情导出已开启' + if (options.exportFiles === true) return '文件导出已开启' + if (options.exportVoiceAsText === true) return '语音转文字已开启' + return null +} + +export function chooseExportEngine(options: ExportEngineOptions): ResolvedExportEngine { + if (options.engine === 'rust') return 'rust' + if (options.engine === 'typescript') return 'typescript' + return canUseRustExportEngine(options) ? 'rust' : 'typescript' +} diff --git a/electron/services/export/messageStream.test.ts b/electron/services/export/messageStream.test.ts new file mode 100644 index 00000000..222c675b --- /dev/null +++ b/electron/services/export/messageStream.test.ts @@ -0,0 +1,98 @@ +import { describe, expect, it } from 'vitest' +import { createMessageStream } from './messageStream' + +describe('message stream', () => { + it('streams normalized rows across batches and closes the cursor', async () => { + const closed: number[] = [] + const source = { + openMessageCursorLite: async () => ({ success: true, cursor: 42 }), + openMessageCursor: async () => ({ success: true, cursor: 99 }), + fetchMessageBatch: async (cursor: number) => { + expect(cursor).toBe(42) + const calls = closed.length + if (calls === 0) { + closed.push(-1) + return { + success: true, + rows: [ + { local_id: 1, create_time: 10, local_type: 1, message_content: 'old', is_send: 0, sender_username: 'a' }, + { local_id: 2, create_time: 20, local_type: 1, message_content: 'hello', is_send: 1, sender_username: 'ignored' } + ], + hasMore: true + } + } + return { + success: true, + rows: [ + { local_id: 3, create_time: 30, local_type: 1, message_content: 'world', is_send: 0, sender_username: 'b' } + ], + hasMore: false + } + }, + closeMessageCursor: async (cursor: number) => { + closed.push(cursor) + } + } + + const rows = [] + for await (const row of createMessageStream({ + source, + sessionId: 'room', + cleanedMyWxid: 'me', + dateRange: { start: 15, end: 35 }, + batchSize: 2, + useLiteCursor: true, + decodeContent: (row) => String(row.message_content || '') + })) { + rows.push(row) + } + + expect(rows).toEqual([ + { localId: 2, serverId: 0, createTime: 20, localType: 1, content: 'hello', senderUsername: 'me', isSend: true }, + { localId: 3, serverId: 0, createTime: 30, localType: 1, content: 'world', senderUsername: 'b', isSend: false } + ]) + expect(closed).toContain(42) + }) + + it('throws when cancellation is requested during streaming', async () => { + const source = { + openMessageCursorLite: async () => ({ success: true, cursor: 7 }), + openMessageCursor: async () => ({ success: true, cursor: 7 }), + fetchMessageBatch: async () => ({ success: true, rows: [{ create_time: 1, message_content: 'x' }], hasMore: true }), + closeMessageCursor: async () => {} + } + let shouldStop = false + const iterator = createMessageStream({ + source, + sessionId: 's', + cleanedMyWxid: 'me', + decodeContent: () => 'x', + control: { shouldStop: () => shouldStop } + })[Symbol.asyncIterator]() + + await iterator.next() + shouldStop = true + await expect(iterator.next()).rejects.toThrow(/导出任务已停止/) + }) + + it('marks pause requests with the shared pause code', async () => { + const source = { + openMessageCursorLite: async () => ({ success: true, cursor: 7 }), + openMessageCursor: async () => ({ success: true, cursor: 7 }), + fetchMessageBatch: async () => ({ success: true, rows: [{ create_time: 1, message_content: 'x' }], hasMore: false }), + closeMessageCursor: async () => {} + } + + const iterator = createMessageStream({ + source, + sessionId: 's', + cleanedMyWxid: 'me', + decodeContent: () => 'x', + control: { shouldPause: () => true } + })[Symbol.asyncIterator]() + + await expect(iterator.next()).rejects.toMatchObject({ + code: 'WEFLOW_EXPORT_PAUSE_REQUESTED' + }) + }) +}) diff --git a/electron/services/export/messageStream.ts b/electron/services/export/messageStream.ts new file mode 100644 index 00000000..b1888930 --- /dev/null +++ b/electron/services/export/messageStream.ts @@ -0,0 +1,248 @@ +export interface MessageStreamRow { + localId: number + serverId: number + serverIdRaw?: string + createTime: number + localType: number + content: string + senderUsername: string + isSend: boolean + emojiMd5?: string + emojiCdnUrl?: string + emojiCaption?: string + locationLat?: number + locationLng?: number + locationPoiname?: string + locationLabel?: string +} + +export interface MessageCursorSource { + openMessageCursor: ( + sessionId: string, + batchSize: number, + ascending: boolean, + beginTimestamp: number, + endTimestamp: number + ) => Promise<{ success: boolean; cursor?: number; error?: string }> + openMessageCursorLite?: ( + sessionId: string, + batchSize: number, + ascending: boolean, + beginTimestamp: number, + endTimestamp: number + ) => Promise<{ success: boolean; cursor?: number; error?: string }> + fetchMessageBatch: (cursor: number) => Promise<{ success: boolean; rows?: any[]; hasMore?: boolean; error?: string }> + closeMessageCursor: (cursor: number) => Promise +} + +export interface MessageStreamControl { + shouldStop?: () => boolean + shouldPause?: () => boolean +} + +export const MESSAGE_STREAM_STOP_CODE = 'WEFLOW_EXPORT_STOP_REQUESTED' +export const MESSAGE_STREAM_PAUSE_CODE = 'WEFLOW_EXPORT_PAUSE_REQUESTED' + +export interface MessageStreamOptions { + source: MessageCursorSource + sessionId: string + cleanedMyWxid: string + dateRange?: { start: number; end: number } | null + senderUsername?: string + batchSize?: number + ascending?: boolean + useLiteCursor?: boolean + control?: MessageStreamControl + decodeContent?: (row: any, localType: number) => string +} + +export function normalizeTimestampSeconds(value: unknown): number { + const raw = Number(value) + if (!Number.isFinite(raw) || raw <= 0) return 0 + let normalized = Math.floor(raw) + while (normalized > 10000000000) { + normalized = Math.floor(normalized / 1000) + } + return normalized +} + +export function normalizeMessageStreamRow( + row: any, + options: Pick +): MessageStreamRow { + const localType = getIntFromRow(row, ['local_type', 'localType', 'type', 'msg_type', 'msgType', 'WCDB_CT_local_type'], 1) + const createTime = normalizeTimestampSeconds(getRowField(row, ['create_time', 'createTime', 'timestamp', 'msgCreateTime', 'WCDB_CT_create_time'])) + const localId = getIntFromRow(row, [ + 'local_id', 'localId', 'LocalId', + 'msg_local_id', 'msgLocalId', 'MsgLocalId', + 'msg_id', 'msgId', 'MsgId', 'id', + 'WCDB_CT_local_id' + ], 0) + const rawServerIdValue = getRowField(row, [ + 'server_id', 'serverId', 'ServerId', + 'msg_server_id', 'msgServerId', 'MsgServerId', + 'svr_id', 'svrId', 'msg_svr_id', 'msgSvrId', 'MsgSvrId', + 'WCDB_CT_server_id' + ]) + const serverIdRaw = normalizeUnsignedIntToken(rawServerIdValue) + const serverId = getIntFromRow(row, [ + 'server_id', 'serverId', 'ServerId', + 'msg_server_id', 'msgServerId', 'MsgServerId', + 'svr_id', 'svrId', 'msg_svr_id', 'msgSvrId', 'MsgSvrId', + 'WCDB_CT_server_id' + ], 0) + const isSend = Number.parseInt(String(row?.computed_is_send ?? row?.is_send ?? row?.isSend ?? '0'), 10) === 1 + const senderUsername = isSend + ? options.cleanedMyWxid + : String(row?.sender_username || row?.senderUsername || options.sessionId || '').trim() + const content = options.decodeContent + ? options.decodeContent(row, localType) + : String(row?.content || row?.message_content || row?.messageContent || '') + const emojiMd5 = getStringFromRow(row, ['emoji_md5', 'emojiMd5']) + const emojiCdnUrl = getStringFromRow(row, ['emoji_cdn_url', 'emojiCdnUrl']) + const emojiCaption = getStringFromRow(row, ['emoji_caption', 'emojiCaption']) + const locationLat = getNumberFromRow(row, ['location_lat', 'locationLat']) + const locationLng = getNumberFromRow(row, ['location_lng', 'locationLng']) + const locationPoiname = getStringFromRow(row, ['location_poiname', 'locationPoiname']) + const locationLabel = getStringFromRow(row, ['location_label', 'locationLabel']) + + const normalized: MessageStreamRow = { + localId, + serverId, + serverIdRaw: serverIdRaw !== '0' ? serverIdRaw : undefined, + createTime, + localType, + content, + senderUsername, + isSend + } + if (emojiMd5) normalized.emojiMd5 = emojiMd5 + if (emojiCdnUrl) normalized.emojiCdnUrl = emojiCdnUrl + if (emojiCaption) normalized.emojiCaption = emojiCaption + if (typeof locationLat === 'number') normalized.locationLat = locationLat + if (typeof locationLng === 'number') normalized.locationLng = locationLng + if (locationPoiname) normalized.locationPoiname = locationPoiname + if (locationLabel) normalized.locationLabel = locationLabel + return normalized +} + +export async function* createMessageStream(options: MessageStreamOptions): AsyncGenerator { + const batchSize = Math.max(1, Math.floor(options.batchSize || 2000)) + const ascending = options.ascending !== false + const range = normalizeDateRange(options.dateRange) + const begin = range?.start || 0 + const end = range?.end || 0 + const useLite = options.useLiteCursor === true && Boolean(options.source.openMessageCursorLite) + throwIfMessageStreamControlRequested(options.control) + const opened = useLite + ? await options.source.openMessageCursorLite!(options.sessionId, batchSize, ascending, begin, end) + : await options.source.openMessageCursor(options.sessionId, batchSize, ascending, begin, end) + if (!opened.success || !opened.cursor) { + throw new Error(opened.error || 'open message cursor failed') + } + + try { + let hasMore = true + while (hasMore) { + throwIfMessageStreamControlRequested(options.control) + const batch = await options.source.fetchMessageBatch(opened.cursor) + if (!batch.success) { + throw new Error(batch.error || 'fetch message batch failed') + } + + for (const rawRow of batch.rows || []) { + throwIfMessageStreamControlRequested(options.control) + const row = normalizeMessageStreamRow(rawRow, options) + if (range) { + if (row.createTime > 0 && range.start > 0 && row.createTime < range.start) continue + if (row.createTime > 0 && range.end > 0 && row.createTime > range.end) continue + } + if (options.senderUsername && row.senderUsername !== options.senderUsername) continue + yield row + } + + hasMore = batch.hasMore === true + } + } finally { + await options.source.closeMessageCursor(opened.cursor) + } +} + +function normalizeDateRange(dateRange?: { start: number; end: number } | null): { start: number; end: number } | null { + if (!dateRange) return null + let start = normalizeTimestampSeconds(dateRange.start) + let end = normalizeTimestampSeconds(dateRange.end) + if (start > 0 && end > 0 && start > end) { + const tmp = start + start = end + end = tmp + } + if (start <= 0 && end <= 0) return null + return { start, end } +} + +export function throwIfMessageStreamControlRequested(control?: MessageStreamControl): void { + if (control?.shouldStop?.()) throw createMessageStreamControlError('stop') + if (control?.shouldPause?.()) throw createMessageStreamControlError('pause') +} + +export function isMessageStreamStopError(error: unknown): boolean { + return hasMessageStreamControlError(error, MESSAGE_STREAM_STOP_CODE, '导出任务已停止') +} + +export function isMessageStreamPauseError(error: unknown): boolean { + return hasMessageStreamControlError(error, MESSAGE_STREAM_PAUSE_CODE, '导出任务已暂停') +} + +function createMessageStreamControlError(type: 'stop' | 'pause'): Error { + const error = new Error(type === 'stop' ? '导出任务已停止' : '导出任务已暂停') + ;(error as Error & { code?: string }).code = type === 'stop' + ? MESSAGE_STREAM_STOP_CODE + : MESSAGE_STREAM_PAUSE_CODE + return error +} + +function hasMessageStreamControlError(error: unknown, code: string, message: string): boolean { + if (!error) return false + if (typeof error === 'string') return error.includes(code) || error.includes(message) + if (error instanceof Error) { + const errorCode = (error as Error & { code?: string }).code + return errorCode === code || error.message.includes(code) || error.message.includes(message) + } + return false +} + +function getRowField(row: any, names: string[]): unknown { + for (const name of names) { + if (row && row[name] !== undefined && row[name] !== null) return row[name] + } + return undefined +} + +function getIntFromRow(row: any, names: string[], fallback: number): number { + const value = getRowField(row, names) + const parsed = Number(value) + if (!Number.isFinite(parsed)) return fallback + return Math.floor(parsed) +} + +function getNumberFromRow(row: any, names: string[]): number | undefined { + const value = getRowField(row, names) + const parsed = Number(value) + return Number.isFinite(parsed) ? parsed : undefined +} + +function getStringFromRow(row: any, names: string[]): string | undefined { + const value = getRowField(row, names) + const text = String(value ?? '').trim() + return text || undefined +} + +function normalizeUnsignedIntToken(value: unknown): string { + const text = String(value ?? '').trim() + if (!text) return '0' + if (/^\d+$/.test(text)) return text.replace(/^0+(?=\d)/, '') || '0' + const numberValue = Number(text) + if (!Number.isFinite(numberValue) || numberValue < 0) return '0' + return String(Math.floor(numberValue)) +} diff --git a/electron/services/export/rustExportBridge.test.ts b/electron/services/export/rustExportBridge.test.ts new file mode 100644 index 00000000..b98aef4e --- /dev/null +++ b/electron/services/export/rustExportBridge.test.ts @@ -0,0 +1,30 @@ +import { describe, expect, it } from 'vitest' +import { + parseRustExportEventLine, + resolveRustExporterExecutableName +} from './rustExportBridge' + +describe('rust export bridge protocol helpers', () => { + it('parses known NDJSON events', () => { + expect(parseRustExportEventLine('{"type":"createdFile","path":"C:/tmp/a.txt"}')).toEqual({ + type: 'createdFile', + path: 'C:/tmp/a.txt' + }) + expect(parseRustExportEventLine('{"type":"result","success":true,"successCount":1,"failCount":0}')).toEqual({ + type: 'result', + success: true, + successCount: 1, + failCount: 0 + }) + }) + + it('rejects invalid and unknown event lines with a readable error', () => { + expect(() => parseRustExportEventLine('not-json')).toThrow(/Invalid Rust exporter event/) + expect(() => parseRustExportEventLine('{"type":"surprise"}')).toThrow(/Unknown Rust exporter event/) + }) + + it('uses platform executable naming without leaking request fields into args', () => { + expect(resolveRustExporterExecutableName('win32')).toBe('weflow-exporter.exe') + expect(resolveRustExporterExecutableName('linux')).toBe('weflow-exporter') + }) +}) diff --git a/electron/services/export/rustExportBridge.ts b/electron/services/export/rustExportBridge.ts new file mode 100644 index 00000000..0ac0de74 --- /dev/null +++ b/electron/services/export/rustExportBridge.ts @@ -0,0 +1,72 @@ +import * as path from 'path' + +export type RustExportEvent = + | { type: 'progress'; data?: Record; [key: string]: unknown } + | { type: 'createdFile'; path: string } + | { type: 'createdDir'; path: string } + | { type: 'result'; success: boolean; successCount?: number; failCount?: number; [key: string]: unknown } + | { type: 'error'; error: string } + +export interface RustExporterPathConfig { + resourcesPath: string + platform?: NodeJS.Platform + arch?: string + executablePath?: string +} + +export function resolveRustExporterExecutableName(platform: NodeJS.Platform = process.platform): string { + return platform === 'win32' ? 'weflow-exporter.exe' : 'weflow-exporter' +} + +export function resolveRustExporterPlatformDir(platform: NodeJS.Platform = process.platform): string { + if (platform === 'darwin') return 'macos' + return platform +} + +export function resolveRustExporterArchDir(platform: NodeJS.Platform = process.platform, arch: string = process.arch): string { + if (platform === 'darwin') return 'universal' + return arch +} + +export function resolveRustExporterPath(config: RustExporterPathConfig): string { + if (config.executablePath) return config.executablePath + return path.join( + config.resourcesPath, + 'exporter', + resolveRustExporterPlatformDir(config.platform), + resolveRustExporterArchDir(config.platform, config.arch), + resolveRustExporterExecutableName(config.platform) + ) +} + +export function parseRustExportEventLine(line: string): RustExportEvent { + let parsed: unknown + try { + parsed = JSON.parse(line) + } catch (error) { + throw new Error(`Invalid Rust exporter event: ${error instanceof Error ? error.message : String(error)}`) + } + + if (!parsed || typeof parsed !== 'object') { + throw new Error('Invalid Rust exporter event: expected object') + } + + const event = parsed as Record + const type = String(event.type || '') + if (type === 'progress') return event as RustExportEvent + if (type === 'createdFile') { + const filePath = String(event.path || '').trim() + if (!filePath) throw new Error('Invalid Rust exporter event: createdFile.path is required') + return { type, path: filePath } + } + if (type === 'createdDir') { + const dirPath = String(event.path || '').trim() + if (!dirPath) throw new Error('Invalid Rust exporter event: createdDir.path is required') + return { type, path: dirPath } + } + if (type === 'result') return event as RustExportEvent + if (type === 'error') { + return { type, error: String(event.error || 'Rust exporter failed') } + } + throw new Error(`Unknown Rust exporter event: ${type || ''}`) +} diff --git a/electron/services/export/rustStreamingExporter.ts b/electron/services/export/rustStreamingExporter.ts new file mode 100644 index 00000000..efb7ef58 --- /dev/null +++ b/electron/services/export/rustStreamingExporter.ts @@ -0,0 +1,656 @@ +import { spawn, type ChildProcessWithoutNullStreams } from 'child_process' +import * as readline from 'readline' +import { canUseRustExportEngine } from './exportEngineRouter' +import { + createMessageStream, + isMessageStreamPauseError, + isMessageStreamStopError, + throwIfMessageStreamControlRequested, + type MessageCursorSource, + type MessageStreamControl, + type MessageStreamRow +} from './messageStream' +import { parseRustExportEventLine, resolveRustExporterPath, type RustExportEvent } from './rustExportBridge' +import { exportService } from '../exportService' +import { extractReadableSystemMessageText } from '../systemMessageFormatter' + +export interface RustStreamingExportOptions { + format: 'txt' | 'html' | 'chatlab-jsonl' | 'weclone' | 'json' | string + dateRange?: { start: number; end: number } | null + senderUsername?: string + fileNameSuffix?: string + contentType?: string + exportMedia?: boolean + exportAvatars?: boolean + exportImages?: boolean + exportVoices?: boolean + exportVideos?: boolean + exportEmojis?: boolean + exportFiles?: boolean + exportVoiceAsText?: boolean + displayNamePreference?: 'group-nickname' | 'remark' | 'nickname' +} + +export interface RustStreamingExportRequest { + source: MessageCursorSource & { + open: (accountDir: string, decryptKey: string) => Promise + getDisplayNames: (usernames: string[]) => Promise<{ success: boolean; map?: Record; error?: string }> + getContact?: (username: string) => Promise<{ success: boolean; contact?: any; error?: string }> + getGroupNicknames?: (chatroomId: string) => Promise<{ success: boolean; nicknames?: Record; error?: string }> + } + sessionIds: string[] + outputDir: string + options: RustStreamingExportOptions + accountDir: string + decryptKey: string + cleanedMyWxid: string + resourcesPath: string + onProgress?: (progress: Record) => void + control?: MessageStreamControl & { + recordCreatedFile?: (filePath: string) => void + recordCreatedDir?: (dirPath: string) => void + } +} + +class RustWriterProcess { + private child: ChildProcessWithoutNullStreams | null = null + private resultPromise: Promise> | null = null + private settleResult: ((value: Record) => void) | null = null + private rejectResult: ((error: unknown) => void) | null = null + private stderr = '' + + constructor( + private readonly executablePath: string, + private readonly callbacks: { + onProgress?: (progress: Record) => void + onCreatedFile?: (filePath: string) => void + onCreatedDir?: (dirPath: string) => void + } + ) {} + + async start(request: Pick): Promise { + this.child = spawn(this.executablePath, [], { + stdio: ['pipe', 'pipe', 'pipe'], + windowsHide: true + }) + + this.resultPromise = new Promise((resolve, reject) => { + this.settleResult = resolve + this.rejectResult = reject + }) + + const rl = readline.createInterface({ input: this.child.stdout }) + rl.on('line', (line) => { + const trimmed = line.trim() + if (!trimmed) return + let event: RustExportEvent + try { + event = parseRustExportEventLine(trimmed) + } catch (error) { + this.rejectResult?.(error) + return + } + + if (event.type === 'progress') { + this.callbacks.onProgress?.((event.data ?? event) as Record) + } else if (event.type === 'createdFile') { + this.callbacks.onCreatedFile?.(event.path) + } else if (event.type === 'createdDir') { + this.callbacks.onCreatedDir?.(event.path) + } else if (event.type === 'result') { + this.settleResult?.(event as Record) + } else if (event.type === 'error') { + this.rejectResult?.(new Error(event.error)) + } + }) + + this.child.stderr.on('data', (chunk) => { + this.stderr += String(chunk || '') + }) + this.child.on('error', (error) => this.rejectResult?.(error)) + this.child.on('exit', (code) => { + if (code === 0) return + const suffix = this.stderr.trim() ? `: ${this.stderr.trim().slice(0, 500)}` : '' + this.rejectResult?.(new Error(`Rust writer exited before result (code ${code})${suffix}`)) + }) + + await this.writeEvent({ + type: 'writerRequest', + outputDir: request.outputDir, + options: request.options + }) + } + + async beginSession(sessionId: string, displayName: string, session?: Record): Promise { + await this.writeEvent({ type: 'beginSession', sessionId, displayName, session }) + } + + async writeMessage(row: MessageStreamRow, senderName: string, jsonMessage?: Record): Promise { + await this.writeEvent({ type: 'message', row, senderName, jsonMessage }) + } + + async endSession(): Promise { + await this.writeEvent({ type: 'endSession' }) + } + + async finish(): Promise> { + await this.writeEvent({ type: 'finish' }) + this.child?.stdin.end() + return await this.resultPromise! + } + + cancel(): void { + try { + if (this.child && !this.child.stdin.destroyed) { + this.child.stdin.write(`${JSON.stringify({ type: 'cancel' })}\n`) + } + } catch {} + try { + this.child?.kill() + } catch {} + } + + private async writeEvent(event: Record): Promise { + if (!this.child || this.child.stdin.destroyed) { + throw new Error('Rust writer is not running') + } + const line = `${JSON.stringify(event)}\n` + await new Promise((resolve, reject) => { + const onError = (error: Error) => { + cleanup() + reject(error) + } + const onDrain = () => { + cleanup() + resolve() + } + const cleanup = () => { + this.child?.stdin.off('error', onError) + this.child?.stdin.off('drain', onDrain) + } + this.child!.stdin.once('error', onError) + if (!this.child!.stdin.write(line)) { + this.child!.stdin.once('drain', onDrain) + } else { + cleanup() + resolve() + } + }) + } +} + +export function canUseRustStreamingExport(options: RustStreamingExportOptions): boolean { + return canUseRustExportEngine(options) +} + +export async function exportSessionsWithRustStreaming(request: RustStreamingExportRequest): Promise> { + if (!canUseRustStreamingExport(request.options)) { + return { success: false, successCount: 0, failCount: request.sessionIds.length, error: `Rust streaming exporter does not support format: ${request.options.format}` } + } + + const opened = await request.source.open(request.accountDir, request.decryptKey) + if (!opened) { + return { success: false, successCount: 0, failCount: request.sessionIds.length, error: 'WCDB 打开失败' } + } + + const executablePath = resolveRustExporterPath({ resourcesPath: request.resourcesPath }) + const writer = new RustWriterProcess(executablePath, { + onProgress: request.onProgress, + onCreatedFile: request.control?.recordCreatedFile, + onCreatedDir: request.control?.recordCreatedDir + }) + const successSessionIds: string[] = [] + let activeSessionIndex = 0 + + try { + await writer.start({ outputDir: request.outputDir, options: request.options }) + const sessionNames = await getDisplayNameMap(request.source, request.sessionIds) + const senderNameCache = new Map() + + for (let index = 0; index < request.sessionIds.length; index++) { + activeSessionIndex = index + throwIfMessageStreamControlRequested(request.control) + const sessionId = request.sessionIds[index] + const sessionName = sessionNames.get(sessionId) || sessionId + request.onProgress?.({ + current: index, + total: request.sessionIds.length, + currentSession: sessionName, + currentSessionId: sessionId, + phase: 'preparing', + phaseLabel: 'Rust 写入器准备导出' + }) + const detailedJsonContext = request.options.format === 'json' + ? await createDetailedJsonContext(request.source, sessionId, sessionName, request.cleanedMyWxid, request.options.displayNamePreference) + : null + await writer.beginSession(sessionId, sessionName, detailedJsonContext?.sessionPayload) + + const stream = createMessageStream({ + source: request.source, + sessionId, + cleanedMyWxid: request.cleanedMyWxid, + dateRange: request.options.dateRange, + senderUsername: request.options.senderUsername, + control: request.control, + decodeContent: decodeMessageContent + }) + + let exportedMessages = 0 + for await (const row of stream) { + throwIfMessageStreamControlRequested(request.control) + const senderName = await resolveSenderName(request.source, row, sessionId, sessionName, senderNameCache) + const messageIndex = exportedMessages + 1 + const jsonMessage = detailedJsonContext + ? await buildDetailedJsonMessage(row, messageIndex, detailedJsonContext) + : undefined + await writer.writeMessage(formatRustWriterRow(row), senderName, jsonMessage) + exportedMessages = messageIndex + if (exportedMessages % 1000 === 0) { + request.onProgress?.({ + current: index, + total: request.sessionIds.length, + currentSession: sessionName, + currentSessionId: sessionId, + phase: 'exporting', + exportedMessages + }) + } + } + + await writer.endSession() + successSessionIds.push(sessionId) + request.onProgress?.({ + current: index + 1, + total: request.sessionIds.length, + currentSession: sessionName, + currentSessionId: sessionId, + phase: 'complete', + exportedMessages, + writtenFiles: 1 + }) + } + + return await writer.finish() + } catch (error) { + writer.cancel() + if (isMessageStreamStopError(error) || isMessageStreamPauseError(error)) { + const stopped = isMessageStreamStopError(error) + const paused = isMessageStreamPauseError(error) + return { + success: true, + successCount: successSessionIds.length, + failCount: 0, + stopped: stopped || undefined, + paused: paused || undefined, + pendingSessionIds: request.sessionIds.slice(activeSessionIndex), + successSessionIds, + failedSessionIds: [], + failedSessionErrors: {}, + sessionOutputPaths: {} + } + } + throw error + } +} + +interface DetailedJsonContext { + sessionId: string + sessionName: string + isGroup: boolean + cleanedMyWxid: string + displayNamePreference: 'group-nickname' | 'remark' | 'nickname' + groupNicknamesMap: Map + contactCache: Map> + source: RustStreamingExportRequest['source'] + sessionPayload: Record +} + +async function createDetailedJsonContext( + source: RustStreamingExportRequest['source'], + sessionId: string, + sessionName: string, + cleanedMyWxid: string, + displayNamePreference?: RustStreamingExportOptions['displayNamePreference'] +): Promise { + const preference = displayNamePreference || 'remark' + const isGroup = sessionId.includes('@chatroom') + const contactCache = new Map>() + const groupNicknamesMap = isGroup ? await getGroupNicknamesMap(source, sessionId) : new Map() + const sessionContact = await getContactCached(source, contactCache, sessionId) + const sessionNickname = getContactNickname(sessionContact.contact) || sessionName + const sessionRemark = getContactRemark(sessionContact.contact) + const sessionGroupNickname = isGroup + ? callExportHelper('resolveGroupNicknameByCandidates', groupNicknamesMap, [sessionId]) || '' + : '' + const sessionDisplayName = getPreferredDisplayName(sessionId, sessionNickname, sessionRemark, sessionGroupNickname, preference) + + return { + sessionId, + sessionName, + isGroup, + cleanedMyWxid, + displayNamePreference: preference, + groupNicknamesMap, + contactCache, + source, + sessionPayload: { + wxid: sessionId, + nickname: sessionNickname, + remark: sessionRemark, + displayName: sessionDisplayName, + type: isGroup ? '群聊' : '私聊', + lastTimestamp: null, + messageCount: 0 + } + } +} + +async function buildDetailedJsonMessage( + row: MessageStreamRow, + localId: number, + context: DetailedJsonContext +): Promise> { + const sourceMatch = /[\s\S]*?<\/msgsource>/i.exec(row.content || '') + const source = sourceMatch ? sourceMatch[0] : '' + let content: string | null = parseMessageContent(row, context) + if (callExportHelper('isReadableSystemMessage', row.localType, row.content)) { + content = callExportHelper('extractReadableSystemMessageText', row.content) || content + } + + const quotedReplyDisplay = await resolveQuotedReplyDisplay(row, context) + if (quotedReplyDisplay) { + content = callExportHelper('buildQuotedReplyText', quotedReplyDisplay) || content + } + const appendedLinkContent = quotedReplyDisplay + ? null + : callExportHelper('formatLinkCardExportText', row.content, row.localType, 'append-url') + if (appendedLinkContent) { + content = appendedLinkContent + } + + const senderDisplayName = await resolveDetailedSenderDisplayName(row, context) + const message: Record = { + localId, + createTime: row.createTime, + formattedTime: formatTimestamp(row.createTime), + type: getMessageTypeName(row.localType, row.content), + localType: row.localType, + content, + isSend: row.isSend ? 1 : 0, + senderUsername: row.senderUsername, + senderDisplayName, + source, + senderAvatarKey: row.senderUsername + } + + if (row.localType === 47) { + if (row.emojiMd5) message.emojiMd5 = row.emojiMd5 + if (row.emojiCdnUrl) message.emojiCdnUrl = row.emojiCdnUrl + if (row.emojiCaption) message.emojiCaption = row.emojiCaption + } + + const platformMessageId = normalizeUnsignedIntToken(row.serverIdRaw ?? row.serverId) + if (platformMessageId !== '0') message.platformMessageId = platformMessageId + + const replyToMessageId = callExportHelper('getExportReplyToMessageId', row.content) + if (replyToMessageId) message.replyToMessageId = replyToMessageId + + const appMsgMeta = callExportHelper | null>('extractArkmeAppMessageMeta', row.content, row.localType) + if (appMsgMeta && (appMsgMeta.appMsgKind === 'quote' || appMsgMeta.appMsgKind === 'link')) { + Object.assign(message, appMsgMeta) + } + if (quotedReplyDisplay) { + if (quotedReplyDisplay.quotedSender) message.quotedSender = quotedReplyDisplay.quotedSender + if (quotedReplyDisplay.quotedPreview) message.quotedContent = quotedReplyDisplay.quotedPreview + } + + if (typeof message.content === 'string' && callExportHelper('isTransferExportContent', message.content) && row.content) { + const transferDesc = await resolveTransferDesc(row.content, context) + if (transferDesc) { + message.content = callExportHelper('appendTransferDesc', message.content, transferDesc) || message.content + } + } + + if (row.localType === 48) { + if (row.locationLat != null) message.locationLat = row.locationLat + if (row.locationLng != null) message.locationLng = row.locationLng + if (row.locationPoiname) message.locationPoiname = row.locationPoiname + if (row.locationLabel) message.locationLabel = row.locationLabel + } + + return message +} + +function parseMessageContent(row: MessageStreamRow, context: DetailedJsonContext): string | null { + const parsed = callExportHelper( + 'parseMessageContent', + row.content, + row.localType, + undefined, + undefined, + context.cleanedMyWxid, + row.senderUsername, + row.isSend, + row.emojiCaption + ) + return parsed ?? row.content ?? '' +} + +async function resolveQuotedReplyDisplay(row: MessageStreamRow, context: DetailedJsonContext): Promise { + if (!row.content || !/(|<refermsg>|('resolveQuotedReplyDisplayWithNames', { + content: row.content, + isGroup: context.isGroup, + displayNamePreference: context.displayNamePreference, + getContact: (username: string) => getContactCached(context.source, context.contactCache, username), + groupNicknamesMap: context.groupNicknamesMap, + cleanedMyWxid: context.cleanedMyWxid, + rawMyWxid: context.cleanedMyWxid, + myDisplayName: context.cleanedMyWxid + }) +} + +async function resolveTransferDesc(content: string, context: DetailedJsonContext): Promise { + return await callExportHelperAsync('resolveTransferDesc', content, context.cleanedMyWxid, context.groupNicknamesMap, async (username: string) => { + const contactResult = await getContactCached(context.source, context.contactCache, username) + return getContactRemark(contactResult.contact) || getContactNickname(contactResult.contact) || username + }) || '' +} + +async function resolveDetailedSenderDisplayName(row: MessageStreamRow, context: DetailedJsonContext): Promise { + const senderWxid = row.senderUsername || '' + const contactResult = senderWxid + ? await getContactCached(context.source, context.contactCache, senderWxid) + : { success: false as const } + const senderNickname = getContactNickname(contactResult.contact) || senderWxid + const senderRemark = getContactRemark(contactResult.contact) + const senderGroupNickname = context.isGroup + ? callExportHelper('resolveGroupNicknameByCandidates', context.groupNicknamesMap, [senderWxid]) || '' + : '' + return getPreferredDisplayName(senderWxid, senderNickname, senderRemark, senderGroupNickname, context.displayNamePreference) +} + +function getPreferredDisplayName( + wxid: string, + nickname: string, + remark: string, + groupNickname: string, + preference: 'group-nickname' | 'remark' | 'nickname' +): string { + return callExportHelper('getPreferredDisplayName', wxid, nickname, remark, groupNickname, preference) + || groupNickname + || remark + || nickname + || wxid +} + +async function getContactCached( + source: RustStreamingExportRequest['source'], + cache: Map>, + username: string +): Promise<{ success: boolean; contact?: any; error?: string }> { + const normalized = String(username || '').trim() + if (!normalized || !source.getContact) return { success: false } + const cached = cache.get(normalized) + if (cached) return await cached + const pending = source.getContact(normalized).catch((error) => ({ success: false as const, error: String(error) })) + cache.set(normalized, pending) + return await pending +} + +async function getGroupNicknamesMap( + source: RustStreamingExportRequest['source'], + sessionId: string +): Promise> { + if (!source.getGroupNicknames) return new Map() + try { + const result = await source.getGroupNicknames(sessionId) + if (!result.success || !result.nicknames) return new Map() + return new Map(Object.entries(result.nicknames).map(([key, value]) => [key, String(value || '')]).filter(([, value]) => value)) + } catch { + return new Map() + } +} + +function getContactNickname(contact: any): string { + return String(contact?.nickName || contact?.nick_name || contact?.nickname || contact?.displayName || '').trim() +} + +function getContactRemark(contact: any): string { + return String(contact?.remark || '').trim() +} + +function getMessageTypeName(localType: number, content: string): string { + return callExportHelper('getMessageTypeName', localType, content) || fallbackMessageTypeName(localType) +} + +function formatTimestamp(timestamp: number): string { + return callExportHelper('formatTimestamp', timestamp) || fallbackFormatTimestamp(timestamp) +} + +function normalizeUnsignedIntToken(value: unknown): string { + const text = String(value ?? '').trim() + if (!text) return '0' + if (/^\d+$/.test(text)) return text.replace(/^0+(?=\d)/, '') || '0' + const parsed = Number(text) + if (!Number.isFinite(parsed) || parsed < 0) return '0' + return String(Math.floor(parsed)) +} + +function fallbackMessageTypeName(localType: number): string { + const names: Record = { + 1: '文本消息', + 3: '图片消息', + 34: '语音消息', + 42: '名片消息', + 43: '视频消息', + 47: '动画表情', + 48: '位置消息', + 49: '链接消息', + 50: '通话消息', + 10000: '系统消息', + 244813135921: '引用消息' + } + return names[localType] || '其他消息' +} + +function fallbackFormatTimestamp(timestamp: number): string { + const date = new Date(timestamp * 1000) + const pad = (value: number) => String(value).padStart(2, '0') + return `${date.getFullYear()}-${pad(date.getMonth() + 1)}-${pad(date.getDate())} ${pad(date.getHours())}:${pad(date.getMinutes())}:${pad(date.getSeconds())}` +} + +function callExportHelper(name: string, ...args: any[]): T | undefined { + const helper = (exportService as any)[name] + if (typeof helper !== 'function') return undefined + try { + return helper.apply(exportService, args) as T + } catch { + return undefined + } +} + +async function callExportHelperAsync(name: string, ...args: any[]): Promise { + const helper = (exportService as any)[name] + if (typeof helper !== 'function') return undefined + try { + return await helper.apply(exportService, args) as T + } catch { + return undefined + } +} + +async function resolveSenderName( + source: RustStreamingExportRequest['source'], + row: MessageStreamRow, + sessionId: string, + sessionName: string, + senderNameCache: Map +): Promise { + if (row.isSend) return '我' + if (!sessionId.includes('@chatroom')) return sessionName + if (senderNameCache.has(row.senderUsername)) return senderNameCache.get(row.senderUsername)! + const nameMap = await getDisplayNameMap(source, [row.senderUsername]) + const name = nameMap.get(row.senderUsername) || row.senderUsername + senderNameCache.set(row.senderUsername, name) + return name +} + +function formatRustWriterRow(row: MessageStreamRow): MessageStreamRow { + if (row.localType !== 10000 && !/> { + const result = await source.getDisplayNames(usernames) + const map = new Map() + for (const username of usernames) { + map.set(username, result.success && result.map ? (result.map[username] || username) : username) + } + return map +} + +function decodeMessageContent(row: any): string { + const compressed = decodeMaybeEncoded(row?.compress_content ?? row?.compressContent) + if (compressed) return compressed + return decodeMaybeEncoded(row?.message_content ?? row?.messageContent ?? row?.content ?? '') +} + +function decodeMaybeEncoded(raw: unknown): string { + if (!raw || typeof raw !== 'string') return '' + const value = raw.trim() + if (!value) return '' + if (/^[0-9]+$/.test(value)) return value + if (value.length > 16 && /^[0-9a-fA-F]+$/.test(value) && value.length % 2 === 0) { + try { + return decodeBinaryContent(Buffer.from(value, 'hex')) + } catch { + return '' + } + } + if (value.length > 16 && /^[A-Za-z0-9+/]+={0,2}$/.test(value) && value.length % 4 === 0) { + try { + return decodeBinaryContent(Buffer.from(value, 'base64')) + } catch { + return value + } + } + return value +} + +function decodeBinaryContent(data: Buffer): string { + if (data.length >= 4 && data.readUInt32LE(0) === 0xFD2FB528) { + try { + const fzstd = require('fzstd') + const decompressed = fzstd.decompress(data) + return Buffer.from(decompressed).toString('utf-8') + } catch { + return '' + } + } + return data.toString('utf-8').replace(/\uFFFD/g, '') +} diff --git a/electron/services/export/streamingWriters.test.ts b/electron/services/export/streamingWriters.test.ts new file mode 100644 index 00000000..41a7b048 --- /dev/null +++ b/electron/services/export/streamingWriters.test.ts @@ -0,0 +1,109 @@ +import { describe, expect, it } from 'vitest' +import { writeChatLabJsonlStream, writeHtmlStream, writeTxtStream } from './streamingWriters' +import type { MessageStreamRow } from './messageStream' + +class MemorySink { + chunks: string[] = [] + async write(chunk: string): Promise { + this.chunks.push(chunk) + } + async end(): Promise {} + text(): string { + return this.chunks.join('') + } +} + +async function* rows(): AsyncGenerator { + yield { localId: 1, serverId: 11, createTime: 1, localType: 1, content: 'hello', senderUsername: 'me', isSend: true } + yield { localId: 2, serverId: 12, createTime: 2, localType: 1, content: '&"', senderUsername: 'you', isSend: false } +} + +async function* systemRows(): AsyncGenerator { + yield { + localId: 3, + serverId: 13, + createTime: 3, + localType: 10000, + content: '', + senderUsername: 'room', + isSend: false + } +} + +async function* qrcodeSystemRows(): AsyncGenerator { + yield { + localId: 4, + serverId: 14, + createTime: 4, + localType: 10000, + content: '', + senderUsername: 'room', + isSend: false + } +} + +describe('streaming writers', () => { + it('writes txt without buffering the full stream', async () => { + const sink = new MemorySink() + await writeTxtStream(rows(), sink, { + getSenderName: (row) => row.isSend ? '我' : '对方', + formatTimestamp: (ts) => `t${ts}`, + flushEvery: 1 + }) + expect(sink.chunks.length).toBeGreaterThan(1) + expect(sink.text()).toContain("t1 '我'\nhello\n\n") + expect(sink.text()).toContain("t2 '对方'\n&\"\n\n") + }) + + it('escapes html while writing message chunks', async () => { + const sink = new MemorySink() + await writeHtmlStream(rows(), sink, { + sessionName: 'A&B', + getSenderName: (row) => row.senderUsername, + formatTimestamp: (ts) => `t${ts}`, + flushEvery: 1 + }) + const text = sink.text() + expect(text).toContain('A&B') + expect(text).toContain('<b>&"') + expect(text).not.toContain('&"') + }) + + it('writes valid jsonl records split across chunks', async () => { + const sink = new MemorySink() + await writeChatLabJsonlStream(rows(), sink, { + sessionName: 'room', + getSenderName: (row) => row.senderUsername, + flushEvery: 1 + }) + const lines = sink.text().trim().split(/\r?\n/) + expect(lines.length).toBe(4) + expect(JSON.parse(lines[0])._type).toBe('chatlab') + expect(JSON.parse(lines[2]).content).toBe('hello') + expect(JSON.parse(lines[3]).content).toBe('&"') + }) + + it('expands system message templates while streaming', async () => { + const sink = new MemorySink() + await writeChatLabJsonlStream(systemRows(), sink, { + sessionName: 'room', + getSenderName: (row) => row.senderUsername, + flushEvery: 1 + }) + const message = JSON.parse(sink.text().trim().split(/\r?\n/)[2]) + expect(message.content).toBe('"张三"邀请"李四、王五"加入了群聊') + expect(message.content).not.toContain('$username$') + expect(message.content).not.toContain('$names$') + }) + + it('expands QR code join templates while streaming', async () => { + const sink = new MemorySink() + await writeTxtStream(qrcodeSystemRows(), sink, { + getSenderName: (row) => row.senderUsername, + flushEvery: 1 + }) + expect(sink.text()).toContain('"新成员"通过扫描"分享者"分享的二维码加入群聊') + expect(sink.text()).not.toContain('$adder$') + expect(sink.text()).not.toContain('$from$') + }) +}) diff --git a/electron/services/export/streamingWriters.ts b/electron/services/export/streamingWriters.ts new file mode 100644 index 00000000..cbebef60 --- /dev/null +++ b/electron/services/export/streamingWriters.ts @@ -0,0 +1,159 @@ +import type { MessageStreamRow } from './messageStream' +import { extractReadableSystemMessageText } from '../systemMessageFormatter' + +export interface TextSink { + write: (chunk: string) => Promise + end?: () => Promise +} + +export interface StreamingWriterOptions { + flushEvery?: number + getSenderName: (row: MessageStreamRow) => string | Promise + formatTimestamp?: (timestamp: number) => string +} + +export interface HtmlStreamingWriterOptions extends StreamingWriterOptions { + sessionName: string +} + +export interface ChatLabJsonlStreamingWriterOptions extends StreamingWriterOptions { + sessionName: string +} + +export async function writeTxtStream( + rows: AsyncIterable, + sink: TextSink, + options: StreamingWriterOptions +): Promise<{ messageCount: number }> { + let buffer: string[] = [] + let messageCount = 0 + const flushEvery = Math.max(1, Math.floor(options.flushEvery || 120)) + const flush = async () => { + if (buffer.length === 0) return + await sink.write(buffer.join('')) + buffer = [] + } + + for await (const row of rows) { + const senderName = await options.getSenderName(row) + const timestamp = formatTimestamp(row.createTime, options) + const content = formatStreamingContent(row) + buffer.push(`${timestamp} '${senderName}'\n${content}\n\n`) + messageCount++ + if (buffer.length >= flushEvery) { + await flush() + } + } + + await flush() + await sink.end?.() + return { messageCount } +} + +export async function writeHtmlStream( + rows: AsyncIterable, + sink: TextSink, + options: HtmlStreamingWriterOptions +): Promise<{ messageCount: number }> { + const flushEvery = Math.max(1, Math.floor(options.flushEvery || 100)) + let buffer: string[] = [] + let messageCount = 0 + const flush = async () => { + if (buffer.length === 0) return + await sink.write(buffer.join('')) + buffer = [] + } + + await sink.write(`${escapeHtml(options.sessionName)}
\n`) + for await (const row of rows) { + const senderName = await options.getSenderName(row) + const timestamp = formatTimestamp(row.createTime, options) + const content = formatStreamingContent(row) + buffer.push( + `
` + + `` + + `${escapeHtml(senderName)}` + + `

${escapeHtml(content).replace(/\r?\n/g, '
')}

` + + `
\n` + ) + messageCount++ + if (buffer.length >= flushEvery) { + await flush() + } + } + await flush() + await sink.write('
\n') + await sink.end?.() + return { messageCount } +} + +export async function writeChatLabJsonlStream( + rows: AsyncIterable, + sink: TextSink, + options: ChatLabJsonlStreamingWriterOptions +): Promise<{ messageCount: number }> { + const flushEvery = Math.max(1, Math.floor(options.flushEvery || 200)) + let buffer: string[] = [] + let messageCount = 0 + const flush = async () => { + if (buffer.length === 0) return + await sink.write(buffer.join('')) + buffer = [] + } + + await sink.write(`${JSON.stringify({ _type: 'chatlab', version: '1.0', generator: 'WeFlow' })}\n`) + await sink.write(`${JSON.stringify({ _type: 'meta', name: options.sessionName, platform: 'wechat' })}\n`) + + for await (const row of rows) { + const accountName = await options.getSenderName(row) + const content = formatStreamingContent(row) + buffer.push(`${JSON.stringify({ + _type: 'message', + sender: row.senderUsername, + accountName, + timestamp: row.createTime, + type: toChatLabType(row.localType), + content, + platformMessageId: row.serverIdRaw || String(row.serverId || row.localId) + })}\n`) + messageCount++ + if (buffer.length >= flushEvery) { + await flush() + } + } + + await flush() + await sink.end?.() + return { messageCount } +} + +function formatTimestamp(timestamp: number, options: Pick): string { + return options.formatTimestamp ? options.formatTimestamp(timestamp) : String(timestamp) +} + +function formatStreamingContent(row: MessageStreamRow): string { + if (row.localType === 10000 || //g, '>') + .replace(/"/g, '"') + .replace(/'/g, ''') +} + +function toChatLabType(localType: number): number { + if (localType === 1) return 0 + if (localType === 3) return 1 + if (localType === 34) return 2 + if (localType === 43) return 3 + if (localType === 47) return 5 + if (localType === 48) return 8 + if (localType === 10000) return 80 + return 99 +} diff --git a/electron/services/export/syntheticLargeExport.test.ts b/electron/services/export/syntheticLargeExport.test.ts new file mode 100644 index 00000000..471586db --- /dev/null +++ b/electron/services/export/syntheticLargeExport.test.ts @@ -0,0 +1,86 @@ +import * as fs from 'fs' +import * as os from 'os' +import * as path from 'path' +import { afterEach, describe, expect, it } from 'vitest' +import type { MessageStreamRow } from './messageStream' +import { writeTxtStream } from './streamingWriters' + +class FileSink { + private readonly stream: fs.WriteStream + + constructor(filePath: string) { + this.stream = fs.createWriteStream(filePath, { encoding: 'utf-8' }) + this.stream.setMaxListeners(0) + } + + async write(chunk: string): Promise { + await new Promise((resolve, reject) => { + this.stream.once('error', reject) + if (!this.stream.write(chunk)) { + this.stream.once('drain', resolve) + } else { + resolve() + } + }) + } + + async end(): Promise { + await new Promise((resolve, reject) => { + this.stream.once('error', reject) + this.stream.end(() => resolve()) + }) + } +} + +const createdFiles: string[] = [] + +afterEach(() => { + for (const filePath of createdFiles.splice(0)) { + try { fs.rmSync(filePath, { force: true }) } catch {} + } +}) + +describe('synthetic large streaming export', () => { + it('writes 550k txt messages with bounded heap growth', async () => { + const total = 550_000 + const outputPath = path.join(os.tmpdir(), `weflow-stream-${Date.now()}-${process.pid}.txt`) + createdFiles.push(outputPath) + + let peakHeap = process.memoryUsage().heapUsed + async function* generateRows(): AsyncGenerator { + for (let i = 1; i <= total; i++) { + if ((i % 5000) === 0) { + peakHeap = Math.max(peakHeap, process.memoryUsage().heapUsed) + await new Promise(resolve => setImmediate(resolve)) + } + yield { + localId: i, + serverId: i, + createTime: 1700000000 + i, + localType: 1, + content: `message ${i}`, + senderUsername: i % 2 === 0 ? 'me' : 'friend', + isSend: i % 2 === 0 + } + } + } + + const startHeap = process.memoryUsage().heapUsed + const startedAt = Date.now() + const result = await writeTxtStream(generateRows(), new FileSink(outputPath), { + getSenderName: row => row.isSend ? '我' : 'friend', + formatTimestamp: ts => String(ts), + flushEvery: 512 + }) + peakHeap = Math.max(peakHeap, process.memoryUsage().heapUsed) + + const stat = fs.statSync(outputPath) + const heapGrowthMb = (peakHeap - startHeap) / 1024 / 1024 + const durationMs = Date.now() - startedAt + console.info(`[syntheticLargeExport] messages=${result.messageCount} bytes=${stat.size} durationMs=${durationMs} heapGrowthMb=${heapGrowthMb.toFixed(1)}`) + + expect(result.messageCount).toBe(total) + expect(stat.size).toBeGreaterThan(10 * 1024 * 1024) + expect(heapGrowthMb).toBeLessThan(128) + }, 120_000) +}) diff --git a/electron/services/export/typescriptStreamingExporter.test.ts b/electron/services/export/typescriptStreamingExporter.test.ts new file mode 100644 index 00000000..81645ede --- /dev/null +++ b/electron/services/export/typescriptStreamingExporter.test.ts @@ -0,0 +1,113 @@ +import * as fs from 'fs' +import * as os from 'os' +import * as path from 'path' +import { afterEach, describe, expect, it } from 'vitest' +import { canUseTypeScriptStreamingExport, exportSessionsWithTypeScriptStreaming } from './typescriptStreamingExporter' + +const cleanupPaths: string[] = [] + +afterEach(() => { + for (const item of cleanupPaths.splice(0).sort((a, b) => b.length - a.length)) { + try { fs.rmSync(item, { recursive: true, force: true }) } catch {} + } +}) + +describe('typescript streaming exporter', () => { + it('gates streaming to text-only supported formats', () => { + expect(canUseTypeScriptStreamingExport({ format: 'txt' })).toBe(true) + expect(canUseTypeScriptStreamingExport({ format: 'html', contentType: 'text' })).toBe(true) + expect(canUseTypeScriptStreamingExport({ format: 'weclone' })).toBe(false) + expect(canUseTypeScriptStreamingExport({ format: 'txt', exportMedia: true })).toBe(false) + }) + + it('exports a session through cursor batches and records created paths', async () => { + const outputDir = path.join(os.tmpdir(), `weflow-ts-stream-${Date.now()}-${process.pid}`) + cleanupPaths.push(outputDir) + const createdFiles: string[] = [] + const createdDirs: string[] = [] + let fetchCount = 0 + let closedCursor = 0 + const source = { + open: async () => true, + getDisplayNames: async (usernames: string[]) => ({ + success: true, + map: Object.fromEntries(usernames.map(username => [username, username === 'room' ? '测试会话' : `name-${username}`])) + }), + openMessageCursor: async () => ({ success: true, cursor: 1 }), + openMessageCursorLite: async () => ({ success: true, cursor: 1 }), + fetchMessageBatch: async () => { + fetchCount++ + if (fetchCount === 1) { + return { + success: true, + rows: [ + { local_id: 1, server_id: 11, create_time: 1, local_type: 1, message_content: 'hello', is_send: 1 }, + { local_id: 2, server_id: 12, create_time: 2, local_type: 1, message_content: 'world', is_send: 0, sender_username: 'friend' } + ], + hasMore: false + } + } + return { success: true, rows: [], hasMore: false } + }, + closeMessageCursor: async (cursor: number) => { + closedCursor = cursor + } + } + + const result = await exportSessionsWithTypeScriptStreaming({ + source, + sessionIds: ['room'], + outputDir, + options: { format: 'txt' }, + accountDir: 'account', + decryptKey: 'key', + cleanedMyWxid: 'me', + control: { + recordCreatedFile: filePath => createdFiles.push(filePath), + recordCreatedDir: dirPath => createdDirs.push(dirPath) + } + }) + + expect(result.success).toBe(true) + expect(fetchCount).toBe(1) + expect(closedCursor).toBe(1) + expect(createdDirs).toEqual([outputDir]) + expect(createdFiles.length).toBe(1) + expect(fs.readFileSync(createdFiles[0], 'utf-8')).toContain('hello') + expect(fs.readFileSync(createdFiles[0], 'utf-8')).toContain('world') + }) + + it('returns a resumable paused result instead of failing the session', async () => { + const outputDir = path.join(os.tmpdir(), `weflow-ts-stream-paused-${Date.now()}-${process.pid}`) + cleanupPaths.push(outputDir) + const source = { + open: async () => true, + getDisplayNames: async (usernames: string[]) => ({ + success: true, + map: Object.fromEntries(usernames.map(username => [username, username])) + }), + openMessageCursor: async () => ({ success: true, cursor: 1 }), + openMessageCursorLite: async () => ({ success: true, cursor: 1 }), + fetchMessageBatch: async () => ({ success: true, rows: [], hasMore: false }), + closeMessageCursor: async () => {} + } + + const result = await exportSessionsWithTypeScriptStreaming({ + source, + sessionIds: ['room', 'next'], + outputDir, + options: { format: 'txt' }, + accountDir: 'account', + decryptKey: 'key', + cleanedMyWxid: 'me', + control: { + shouldPause: () => true + } + }) + + expect(result.success).toBe(true) + expect(result.paused).toBe(true) + expect(result.failedSessionIds).toEqual([]) + expect(result.pendingSessionIds).toEqual(['room', 'next']) + }) +}) diff --git a/electron/services/export/typescriptStreamingExporter.ts b/electron/services/export/typescriptStreamingExporter.ts new file mode 100644 index 00000000..4fd5089f --- /dev/null +++ b/electron/services/export/typescriptStreamingExporter.ts @@ -0,0 +1,309 @@ +import * as fs from 'fs' +import * as path from 'path' +import { canUseTypeScriptStreamingEngine } from './exportEngineRouter' +import { + createMessageStream, + isMessageStreamPauseError, + isMessageStreamStopError, + throwIfMessageStreamControlRequested, + type MessageCursorSource, + type MessageStreamControl, + type MessageStreamRow +} from './messageStream' +import { writeChatLabJsonlStream, writeHtmlStream, writeTxtStream, type TextSink } from './streamingWriters' + +export interface TypeScriptStreamingExportOptions { + format: 'txt' | 'html' | 'chatlab-jsonl' | string + dateRange?: { start: number; end: number } | null + senderUsername?: string + fileNameSuffix?: string + contentType?: string + exportMedia?: boolean + exportAvatars?: boolean + exportImages?: boolean + exportVoices?: boolean + exportVideos?: boolean + exportEmojis?: boolean + exportFiles?: boolean + exportVoiceAsText?: boolean +} + +export interface TypeScriptStreamingExportRequest { + source: MessageCursorSource & { + open: (accountDir: string, decryptKey: string) => Promise + getDisplayNames: (usernames: string[]) => Promise<{ success: boolean; map?: Record; error?: string }> + } + sessionIds: string[] + outputDir: string + options: TypeScriptStreamingExportOptions + accountDir: string + decryptKey: string + cleanedMyWxid: string + onProgress?: (progress: Record) => void + control?: MessageStreamControl & { + recordCreatedFile?: (filePath: string) => void + recordCreatedDir?: (dirPath: string) => void + } +} + +class FileSink implements TextSink { + private readonly stream: fs.WriteStream + + constructor(filePath: string) { + this.stream = fs.createWriteStream(filePath, { encoding: 'utf-8' }) + } + + async write(chunk: string): Promise { + await new Promise((resolve, reject) => { + const onError = (error: Error) => { + cleanup() + reject(error) + } + const onDrain = () => { + cleanup() + resolve() + } + const cleanup = () => { + this.stream.off('error', onError) + this.stream.off('drain', onDrain) + } + this.stream.once('error', onError) + if (!this.stream.write(chunk)) { + this.stream.once('drain', onDrain) + } else { + cleanup() + resolve() + } + }) + } + + async end(): Promise { + await new Promise((resolve, reject) => { + this.stream.once('error', reject) + this.stream.end(() => resolve()) + }) + } +} + +export function canUseTypeScriptStreamingExport(options: TypeScriptStreamingExportOptions): boolean { + return canUseTypeScriptStreamingEngine(options) +} + +export async function exportSessionsWithTypeScriptStreaming(request: TypeScriptStreamingExportRequest): Promise> { + if (!canUseTypeScriptStreamingExport(request.options)) { + return { success: false, successCount: 0, failCount: request.sessionIds.length, error: `streaming exporter does not support format: ${request.options.format}` } + } + + const opened = await request.source.open(request.accountDir, request.decryptKey) + if (!opened) { + return { success: false, successCount: 0, failCount: request.sessionIds.length, error: 'WCDB 打开失败' } + } + + await fs.promises.mkdir(request.outputDir, { recursive: true }) + request.control?.recordCreatedDir?.(request.outputDir) + const sessionNames = await getDisplayNameMap(request.source, request.sessionIds) + const senderNameCache = new Map() + const successSessionIds: string[] = [] + const failedSessionIds: string[] = [] + const failedSessionErrors: Record = {} + const sessionOutputPaths: Record = {} + + for (let index = 0; index < request.sessionIds.length; index++) { + const sessionId = request.sessionIds[index] + const sessionName = sessionNames.get(sessionId) || sessionId + request.onProgress?.({ + current: index, + total: request.sessionIds.length, + currentSession: sessionName, + currentSessionId: sessionId, + phase: 'preparing' + }) + + try { + throwIfMessageStreamControlRequested(request.control) + const outputPath = reserveOutputPath( + request.outputDir, + `${sanitizeFileName(sessionName)}${request.options.fileNameSuffix || ''}`, + extensionForFormat(request.options.format) + ) + request.control?.recordCreatedFile?.(outputPath) + const stream = createMessageStream({ + source: request.source, + sessionId, + cleanedMyWxid: request.cleanedMyWxid, + dateRange: request.options.dateRange, + senderUsername: request.options.senderUsername, + control: request.control, + decodeContent: decodeMessageContent + }) + const getSenderName = async (row: MessageStreamRow) => { + if (row.isSend) return '我' + if (!sessionId.includes('@chatroom')) return sessionName + if (senderNameCache.has(row.senderUsername)) return senderNameCache.get(row.senderUsername)! + const nameMap = await getDisplayNameMap(request.source, [row.senderUsername]) + const name = nameMap.get(row.senderUsername) || row.senderUsername + senderNameCache.set(row.senderUsername, name) + return name + } + const sink = new FileSink(outputPath) + const writerOptions = { + sessionName, + getSenderName, + formatTimestamp: formatTimestamp, + flushEvery: 256 + } + const result = request.options.format === 'txt' + ? await writeTxtStream(stream, sink, writerOptions) + : request.options.format === 'html' + ? await writeHtmlStream(stream, sink, writerOptions) + : await writeChatLabJsonlStream(stream, sink, writerOptions) + + successSessionIds.push(sessionId) + sessionOutputPaths[sessionId] = outputPath + request.onProgress?.({ + current: index + 1, + total: request.sessionIds.length, + currentSession: sessionName, + currentSessionId: sessionId, + phase: 'complete', + exportedMessages: result.messageCount, + writtenFiles: 1 + }) + } catch (error) { + const controlResult = buildControlResult( + error, + request.sessionIds.slice(index), + successSessionIds, + failedSessionIds, + failedSessionErrors, + sessionOutputPaths + ) + if (controlResult) return controlResult + failedSessionIds.push(sessionId) + failedSessionErrors[sessionId] = error instanceof Error ? error.message : String(error) + } + } + + const successCount = successSessionIds.length + const failCount = failedSessionIds.length + return { + success: successCount > 0 || failCount === 0, + successCount, + failCount, + successSessionIds, + failedSessionIds, + failedSessionErrors, + sessionOutputPaths, + error: successCount === 0 && failCount > 0 ? Object.values(failedSessionErrors).slice(0, 3).join('；') : undefined + } +} + +function buildControlResult( + error: unknown, + pendingSessionIds: string[], + successSessionIds: string[], + failedSessionIds: string[], + failedSessionErrors: Record, + sessionOutputPaths: Record +): Record | null { + const stopped = isMessageStreamStopError(error) + const paused = isMessageStreamPauseError(error) + if (!stopped && !paused) return null + + return { + success: true, + successCount: successSessionIds.length, + failCount: failedSessionIds.length, + stopped: stopped || undefined, + paused: paused || undefined, + pendingSessionIds, + successSessionIds, + failedSessionIds, + failedSessionErrors, + sessionOutputPaths + } +} + +async function getDisplayNameMap( + source: TypeScriptStreamingExportRequest['source'], + usernames: string[] +): Promise> { + const result = await source.getDisplayNames(usernames) + const map = new Map() + for (const username of usernames) { + map.set(username, result.success && result.map ? (result.map[username] || username) : username) + } + return map +} + +function extensionForFormat(format: string): string { + if (format === 'html') return '.html' + if (format === 'chatlab-jsonl') return '.jsonl' + return '.txt' +} + +function reserveOutputPath(outputDir: string, baseName: string, ext: string): string { + let candidate = path.join(outputDir, `${baseName}${ext}`) + let index = 1 + while (fs.existsSync(candidate)) { + candidate = path.join(outputDir, `${baseName} (${index})${ext}`) + index++ + } + return candidate +} + +function sanitizeFileName(value: string): string { + const cleaned = String(value || '') + .replace(/[<>:"/\\|?*\x00-\x1F]/g, '_') + .trim() + .replace(/^\.+|\.+$/g, '') + return cleaned || 'session' +} + +function formatTimestamp(timestamp: number): string { + const date = new Date(timestamp * 1000) + const pad = (value: number) => String(value).padStart(2, '0') + return `${date.getFullYear()}-${pad(date.getMonth() + 1)}-${pad(date.getDate())} ${pad(date.getHours())}:${pad(date.getMinutes())}:${pad(date.getSeconds())}` +} + +function decodeMessageContent(row: any): string { + const compressed = decodeMaybeEncoded(row?.compress_content ?? row?.compressContent) + if (compressed) return compressed + return decodeMaybeEncoded(row?.message_content ?? row?.messageContent ?? row?.content ?? '') +} + +function decodeMaybeEncoded(raw: unknown): string { + if (!raw) return '' + if (typeof raw !== 'string') return '' + const value = raw.trim() + if (!value) return '' + if (/^[0-9]+$/.test(value)) return value + if (value.length > 16 && /^[0-9a-fA-F]+$/.test(value) && value.length % 2 === 0) { + try { + return decodeBinaryContent(Buffer.from(value, 'hex')) + } catch { + return '' + } + } + if (value.length > 16 && /^[A-Za-z0-9+/]+={0,2}$/.test(value) && value.length % 4 === 0) { + try { + return decodeBinaryContent(Buffer.from(value, 'base64')) + } catch { + return value + } + } + return value +} + +function decodeBinaryContent(data: Buffer): string { + if (data.length >= 4 && data.readUInt32LE(0) === 0xFD2FB528) { + try { + const fzstd = require('fzstd') + const decompressed = fzstd.decompress(data) + return Buffer.from(decompressed).toString('utf-8') + } catch { + return '' + } + } + return data.toString('utf-8').replace(/\uFFFD/g, '') +} diff --git a/electron/services/exportService.ts b/electron/services/exportService.ts index 72198df9..90dbb004 100644 --- a/electron/services/exportService.ts +++ b/electron/services/exportService.ts @@ -15,6 +15,10 @@ import { voiceTranscribeService } from './voiceTranscribeService' import { exportRecordService } from './exportRecordService' import { EXPORT_HTML_STYLES } from './exportHtmlStyles' import { LRUCache } from '../utils/LRUCache.js' +import { + cleanSystemMessageContent, + extractReadableSystemMessageText as extractReadableSystemMessageTextValue +} from './systemMessageFormatter' // ChatLab 格式类型定义 interface ChatLabHeader { @@ -95,6 +99,7 @@ const FILE_APP_LOCAL_TYPE_SET = new Set(FILE_APP_LOCAL_TYPES) export interface ExportOptions { format: 'chatlab' | 'chatlab-jsonl' | 'json' | 'arkme-json' | 'html' | 'txt' | 'excel' | 'weclone' | 'sql' + engine?: 'auto' | 'typescript' | 'rust' contentType?: 'text' | 'voice' | 'image' | 'video' | 'emoji' | 'file' dateRange?: { start: number; end: number } | null senderUsername?: string @@ -165,6 +170,8 @@ export interface ExportProgress { total: number currentSession: string currentSessionId?: string + exportEngine?: 'rust' | 'typescript' + exportEngineLabel?: string phase: 'preparing' | 'exporting' | 'exporting-media' | 'exporting-voice' | 'writing' | 'complete' phaseProgress?: number phaseTotal?: number @@ -527,6 +534,25 @@ class ExportService { } } + private isExportPerfLogEnabled(): boolean { + return this.configService.get('logEnabled') === true + } + + private getMemorySnapshotMb(): { rss: number; heapUsed: number; external: number } { + const memory = process.memoryUsage() + const toMb = (value: number) => Math.round(value / 1024 / 1024) + return { + rss: toMb(memory.rss), + heapUsed: toMb(memory.heapUsed), + external: toMb(memory.external) + } + } + + private logExportPerf(label: string, fields: Record): void { + if (!this.isExportPerfLogEnabled()) return + console.info(`[ExportPerf] ${label}`, fields) + } + private async ensureExportDir(dirPath: string, control?: ExportTaskControl, dirCache?: Set): Promise { if (dirCache?.has(dirPath)) return const existed = await this.pathExists(dirPath) @@ -3043,64 +3069,11 @@ class ExportService { } private cleanSystemMessage(content: string): string { - if (!content) return '[系统消息]' - - // 先尝试提取特定的系统消息内容 - // 1. 提取 sysmsg 中的文本内容 - const sysmsgTextMatch = /]*>([\s\S]*?)<\/sysmsg>/i.exec(content) - if (sysmsgTextMatch) { - content = sysmsgTextMatch[1] - } - - // 2. 提取 revokemsg 撤回消息 - const revokeMatch = /<\/replacemsg>/i.exec(content) - if (revokeMatch) { - return revokeMatch[1].trim() - } - - // 3. 提取 pat 拍一拍消息（sysmsg 内的 template 格式） - const patMatch = /