diff --git a/.github/labeler.yml b/.github/labeler.yml new file mode 100644 index 00000000..de326de0 --- /dev/null +++ b/.github/labeler.yml @@ -0,0 +1,17 @@ +# Add 'vmind' label to any change within the 'vmind' package +vmind: + - packages/vmind/** + + +# Add 'calculator' label to any change within the 'calculator' package +calculator: + - packages/calculator/** + + +# Add 'chart-advisor' label to any change within the 'calculator' package +chart-advisor: + - packages/chart-advisor/** + + + + diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 69f606c3..530d4f8f 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -48,6 +48,9 @@ jobs: release_version: ${{ steps.semver_parser.outputs.full }} write_next_bump: true + - name: Generate changelog by rush version + run: node common/scripts/install-run-rush.js version --bump + - name: Update version run: node common/scripts/install-run-rush.js version --bump diff --git a/.gitignore b/.gitignore index 1222cc77..279a800e 100644 --- a/.gitignore +++ b/.gitignore @@ -34,6 +34,7 @@ dist esm es cjs +lib build build-es5 *.zip @@ -103,3 +104,6 @@ docs/public/documents .env.local packages/calculator/tsconfig.cjs.tsbuildinfo packages/calculator/tsconfig.esm.tsbuildinfo +packages/chart-advisor/tsconfig.esm.tsbuildinfo +packages/chart-advisor/tsconfig.cjs.tsbuildinfo +packages/vmind/__tests__/browser/src/pages/mockData.ts diff --git a/common/config/rush/pnpm-lock.yaml b/common/config/rush/pnpm-lock.yaml index ab7d6f55..1a6b9227 100644 --- a/common/config/rush/pnpm-lock.yaml +++ b/common/config/rush/pnpm-lock.yaml @@ -85,6 +85,43 @@ importers: ts-node: 10.9.0_4lhcgfu2tqlb5z5dwdvf5srjjq typescript: 4.9.5 + ../../packages/chart-advisor: + specifiers: + '@internal/bundler': workspace:* + '@internal/eslint-config': workspace:* + '@internal/ts-config': workspace:* + '@rushstack/eslint-patch': ~1.1.4 + '@types/jest': ^26.0.0 + '@types/lodash': 4.14.182 + '@types/node': '*' + '@typescript-eslint/eslint-plugin': 5.30.0 + '@typescript-eslint/parser': 5.30.0 + jest: ^26.0.0 + lodash: 4.17.21 + npm-run-all: ^4.1.5 + rimraf: ^3.0.2 + ts-jest: ^26.0.0 + typescript: 4.9.5 + undici-types: ^5.27.2 + dependencies: + lodash: 4.17.21 + devDependencies: + '@internal/bundler': link:../../tools/bundler + '@internal/eslint-config': link:../../share/eslint-config + '@internal/ts-config': link:../../share/ts-config + '@rushstack/eslint-patch': 1.1.4 + '@types/jest': 26.0.24 + '@types/lodash': 4.14.182 + '@types/node': 20.11.16 + '@typescript-eslint/eslint-plugin': 5.30.0_dwvk2jxdxtyxgoclhbqbxvd22a + '@typescript-eslint/parser': 5.30.0_typescript@4.9.5 + jest: 26.6.3 + npm-run-all: 4.1.5 + rimraf: 3.0.2 + ts-jest: 26.5.6_xuote2qreek47x2di7kesslrai + typescript: 4.9.5 + undici-types: 5.28.3 + ../../packages/vmind: specifiers: '@arco-design/web-react': 2.46.1 @@ -103,12 +140,13 @@ importers: '@typescript-eslint/eslint-plugin': 5.30.0 '@typescript-eslint/parser': 5.30.0 '@visactor/calculator': workspace:* - '@visactor/chart-advisor': 0.1.10 - '@visactor/vchart': ^1.9.0 + '@visactor/chart-advisor': workspace:* + '@visactor/vchart': ^1.10.2 '@visactor/vdataset': ~0.17.4 '@visactor/vrender-core': ^0.17.23 '@visactor/vutils': ~0.17.4 '@vitejs/plugin-react': 3.1.0 + alasql: ~4.3.2 axios: ^1.4.0 canvas: ^2.11.2 dayjs: ~1.11.10 @@ -136,9 +174,10 @@ importers: vite-plugin-libcss: ~1.1.1 dependencies: '@visactor/calculator': link:../calculator - '@visactor/chart-advisor': 0.1.10 + '@visactor/chart-advisor': link:../chart-advisor '@visactor/vdataset': 0.17.4 '@visactor/vutils': 0.17.4 + alasql: 4.3.2 axios: 1.6.7 dayjs: 1.11.10 exceljs: 4.4.0 @@ -162,7 +201,7 @@ importers: '@types/react-dom': 18.2.18 '@typescript-eslint/eslint-plugin': 5.30.0_cow5zg7tx6c3eisi5a4ud5kwia '@typescript-eslint/parser': 5.30.0_vwud3sodsb5zxmzckoj7rdwdbq - '@visactor/vchart': 1.9.2 + '@visactor/vchart': 1.10.2 '@visactor/vrender-core': 0.17.23 '@vitejs/plugin-react': 3.1.0_vite@3.2.6 canvas: 2.11.2 @@ -1876,6 +1915,46 @@ packages: slash: 3.0.0 dev: true + /@jest/core/26.6.3: + resolution: {integrity: sha512-xvV1kKbhfUqFVuZ8Cyo+JPpipAHHAV3kcDBftiduK8EICXmTFddryy3P7NfZt8Pv37rA9nEJBKCCkglCPt/Xjw==} + engines: {node: '>= 10.14.2'} + dependencies: + '@jest/console': 26.6.2 + '@jest/reporters': 26.6.2 + '@jest/test-result': 26.6.2 + '@jest/transform': 26.6.2 + '@jest/types': 26.6.2 + '@types/node': 20.11.16 + ansi-escapes: 4.3.2 + chalk: 4.1.2 + exit: 0.1.2 + graceful-fs: 4.2.11 + jest-changed-files: 26.6.2 + jest-config: 26.6.3 + jest-haste-map: 26.6.2 + jest-message-util: 26.6.2 + jest-regex-util: 26.0.0 + jest-resolve: 26.6.2 + jest-resolve-dependencies: 26.6.3 + jest-runner: 26.6.3 + jest-runtime: 26.6.3 + jest-snapshot: 26.6.2 + jest-util: 26.6.2 + jest-validate: 26.6.2 + jest-watcher: 26.6.2 + micromatch: 4.0.5 + p-each-series: 2.2.0 + rimraf: 3.0.2 + slash: 3.0.0 + strip-ansi: 6.0.1 + transitivePeerDependencies: + - bufferutil + - canvas + - supports-color + - ts-node + - utf-8-validate + dev: true + /@jest/core/26.6.3_canvas@2.11.2: resolution: {integrity: sha512-xvV1kKbhfUqFVuZ8Cyo+JPpipAHHAV3kcDBftiduK8EICXmTFddryy3P7NfZt8Pv37rA9nEJBKCCkglCPt/Xjw==} engines: {node: '>= 10.14.2'} @@ -2091,6 +2170,23 @@ packages: - supports-color dev: true + /@jest/test-sequencer/26.6.3: + resolution: {integrity: sha512-YHlVIjP5nfEyjlrSr8t/YdNfU/1XEt7c5b4OxcXCjyRhjzLYu/rO69/WHPuYcbCWkz8kAeZVZp2N2+IOLLEPGw==} + engines: {node: '>= 10.14.2'} + dependencies: + '@jest/test-result': 26.6.2 + graceful-fs: 4.2.11 + jest-haste-map: 26.6.2 + jest-runner: 26.6.3 + jest-runtime: 26.6.3 + transitivePeerDependencies: + - bufferutil + - canvas + - supports-color + - ts-node + - utf-8-validate + dev: true + /@jest/test-sequencer/26.6.3_canvas@2.11.2: resolution: {integrity: sha512-YHlVIjP5nfEyjlrSr8t/YdNfU/1XEt7c5b4OxcXCjyRhjzLYu/rO69/WHPuYcbCWkz8kAeZVZp2N2+IOLLEPGw==} engines: {node: '>= 10.14.2'} @@ -3033,6 +3129,51 @@ packages: transitivePeerDependencies: - supports-color + /@typescript-eslint/eslint-plugin/5.30.0_dwvk2jxdxtyxgoclhbqbxvd22a: + resolution: {integrity: sha512-lvhRJ2pGe2V9MEU46ELTdiHgiAFZPKtLhiU5wlnaYpMc2+c1R8fh8i80ZAa665drvjHKUJyRRGg3gEm1If54ow==} + engines: {node: ^12.22.0 || ^14.17.0 || >=16.0.0} + peerDependencies: + '@typescript-eslint/parser': ^5.0.0 + eslint: ^6.0.0 || ^7.0.0 || ^8.0.0 + typescript: '*' + peerDependenciesMeta: + typescript: + optional: true + dependencies: + '@typescript-eslint/parser': 5.30.0_typescript@4.9.5 + '@typescript-eslint/scope-manager': 5.30.0 + '@typescript-eslint/type-utils': 5.30.0_typescript@4.9.5 + '@typescript-eslint/utils': 5.30.0_typescript@4.9.5 + debug: 4.3.4 + functional-red-black-tree: 1.0.1 + ignore: 5.3.1 + regexpp: 3.2.0 + semver: 7.5.4 + tsutils: 3.21.0_typescript@4.9.5 + typescript: 4.9.5 + transitivePeerDependencies: + - supports-color + dev: true + + /@typescript-eslint/parser/5.30.0_typescript@4.9.5: + resolution: {integrity: sha512-2oYYUws5o2liX6SrFQ5RB88+PuRymaM2EU02/9Ppoyu70vllPnHVO7ioxDdq/ypXHA277R04SVjxvwI8HmZpzA==} + engines: {node: ^12.22.0 || ^14.17.0 || >=16.0.0} + peerDependencies: + eslint: ^6.0.0 || ^7.0.0 || ^8.0.0 + typescript: '*' + peerDependenciesMeta: + typescript: + optional: true + dependencies: + '@typescript-eslint/scope-manager': 5.30.0 + '@typescript-eslint/types': 5.30.0 + '@typescript-eslint/typescript-estree': 5.30.0_typescript@4.9.5 + debug: 4.3.4 + typescript: 4.9.5 + transitivePeerDependencies: + - supports-color + dev: true + /@typescript-eslint/parser/5.30.0_vwud3sodsb5zxmzckoj7rdwdbq: resolution: {integrity: sha512-2oYYUws5o2liX6SrFQ5RB88+PuRymaM2EU02/9Ppoyu70vllPnHVO7ioxDdq/ypXHA277R04SVjxvwI8HmZpzA==} engines: {node: ^12.22.0 || ^14.17.0 || >=16.0.0} @@ -3059,6 +3200,24 @@ packages: '@typescript-eslint/types': 5.30.0 '@typescript-eslint/visitor-keys': 5.30.0 + /@typescript-eslint/type-utils/5.30.0_typescript@4.9.5: + resolution: {integrity: sha512-GF8JZbZqSS+azehzlv/lmQQ3EU3VfWYzCczdZjJRxSEeXDQkqFhCBgFhallLDbPwQOEQ4MHpiPfkjKk7zlmeNg==} + engines: {node: ^12.22.0 || ^14.17.0 || >=16.0.0} + peerDependencies: + eslint: '*' + typescript: '*' + peerDependenciesMeta: + typescript: + optional: true + dependencies: + '@typescript-eslint/utils': 5.30.0_typescript@4.9.5 + debug: 4.3.4 + tsutils: 3.21.0_typescript@4.9.5 + typescript: 4.9.5 + transitivePeerDependencies: + - supports-color + dev: true + /@typescript-eslint/type-utils/5.30.0_vwud3sodsb5zxmzckoj7rdwdbq: resolution: {integrity: sha512-GF8JZbZqSS+azehzlv/lmQQ3EU3VfWYzCczdZjJRxSEeXDQkqFhCBgFhallLDbPwQOEQ4MHpiPfkjKk7zlmeNg==} engines: {node: ^12.22.0 || ^14.17.0 || >=16.0.0} @@ -3101,6 +3260,23 @@ packages: transitivePeerDependencies: - supports-color + /@typescript-eslint/utils/5.30.0_typescript@4.9.5: + resolution: {integrity: sha512-0bIgOgZflLKIcZsWvfklsaQTM3ZUbmtH0rJ1hKyV3raoUYyeZwcjQ8ZUJTzS7KnhNcsVT1Rxs7zeeMHEhGlltw==} + engines: {node: ^12.22.0 || ^14.17.0 || >=16.0.0} + peerDependencies: + eslint: ^6.0.0 || ^7.0.0 || ^8.0.0 + dependencies: + '@types/json-schema': 7.0.15 + '@typescript-eslint/scope-manager': 5.30.0 + '@typescript-eslint/types': 5.30.0 + '@typescript-eslint/typescript-estree': 5.30.0_typescript@4.9.5 + eslint-scope: 5.1.1 + eslint-utils: 3.0.0 + transitivePeerDependencies: + - supports-color + - typescript + dev: true + /@typescript-eslint/utils/5.30.0_vwud3sodsb5zxmzckoj7rdwdbq: resolution: {integrity: sha512-0bIgOgZflLKIcZsWvfklsaQTM3ZUbmtH0rJ1hKyV3raoUYyeZwcjQ8ZUJTzS7KnhNcsVT1Rxs7zeeMHEhGlltw==} engines: {node: ^12.22.0 || ^14.17.0 || >=16.0.0} @@ -3129,29 +3305,23 @@ packages: resolution: {integrity: sha512-sL/cEvJWAnClXw0wHk85/2L0G6Sj8UB0Ctc1TEMbKSsmpRosqhwj9gWgFRZSrBr2f9tiXISwNhCPmlfqUqyb9Q==} dev: true - /@visactor/chart-advisor/0.1.10: - resolution: {integrity: sha512-br8EoI6uz2GwxjkHTUNOGkVhnTzNrf96h4LIeAz0IojXLxplqTp4USmKW8IRkgk9PJs5sDcB+/9cgZBpz5vR4A==} - dependencies: - lodash: 4.17.21 - dev: false - - /@visactor/vchart/1.9.2: - resolution: {integrity: sha512-wvzskhploTKzsspPBlx3s+m6tAMM1OU6Fyk96fRy/4WP5udjZ7X6gXFbZ2f2C4NmmleirN1nF8psVB2Q+VVk+Q==} + /@visactor/vchart/1.10.2: + resolution: {integrity: sha512-xFMe6MnpOHRyAcG9LNVrYbik4gt0HGgr/8+Phf6Bke1fwpJa1se/w2ofzlTVL7eMPmxYo7KKrmQyG0eLGWTkkA==} dependencies: - '@visactor/vdataset': 0.17.4 - '@visactor/vgrammar-core': 0.11.11 - '@visactor/vgrammar-hierarchy': 0.11.11 - '@visactor/vgrammar-projection': 0.11.11 - '@visactor/vgrammar-sankey': 0.11.11 - '@visactor/vgrammar-util': 0.11.11 - '@visactor/vgrammar-wordcloud': 0.11.11 - '@visactor/vgrammar-wordcloud-shape': 0.11.11 - '@visactor/vrender-components': 0.17.23 - '@visactor/vrender-core': 0.17.23 - '@visactor/vrender-kits': 0.17.23 - '@visactor/vscale': 0.17.4 - '@visactor/vutils': 0.17.4 - '@visactor/vutils-extension': 1.9.2 + '@visactor/vdataset': 0.18.1 + '@visactor/vgrammar-core': 0.12.7 + '@visactor/vgrammar-hierarchy': 0.12.7 + '@visactor/vgrammar-projection': 0.12.7 + '@visactor/vgrammar-sankey': 0.12.7 + '@visactor/vgrammar-util': 0.12.7 + '@visactor/vgrammar-wordcloud': 0.12.7 + '@visactor/vgrammar-wordcloud-shape': 0.12.7 + '@visactor/vrender-components': 0.18.7 + '@visactor/vrender-core': 0.18.7 + '@visactor/vrender-kits': 0.18.7 + '@visactor/vscale': 0.18.1 + '@visactor/vutils': 0.18.1 + '@visactor/vutils-extension': 1.10.2 dev: true /@visactor/vdataset/0.17.4: @@ -3174,90 +3344,113 @@ packages: simple-statistics: 7.8.3 simplify-geojson: 1.0.5 topojson-client: 3.1.0 + dev: false - /@visactor/vgrammar-coordinate/0.11.11: - resolution: {integrity: sha512-2gwXXfi+NrkdcoQABL2heYs31qfzIWuv3AqYm/qO5pFQ78+eGsB9ZGo/8DldjvfMH8zw50QzB+tzgLrlEcUenA==} + /@visactor/vdataset/0.18.1: + resolution: {integrity: sha512-ByrBt2kgLvYRve+Q+9oo3Ibav5WVSyWPuxdDJHK7kDTJGtTuV8z4qKcqArB86PcAOJS1s5L0TtHlV4Femm2xoA==} dependencies: - '@visactor/vgrammar-util': 0.11.11 - '@visactor/vutils': 0.17.4 + '@turf/flatten': 6.5.0 + '@turf/helpers': 6.5.0 + '@turf/rewind': 6.5.0 + '@visactor/vutils': 0.18.1 + d3-dsv: 2.0.0 + d3-geo: 1.12.1 + d3-hexbin: 0.2.2 + d3-hierarchy: 3.1.2 + eventemitter3: 4.0.7 + geobuf: 3.0.2 + geojson-dissolve: 3.1.0 + path-browserify: 1.0.1 + pbf: 3.2.1 + point-at-length: 1.1.0 + simple-statistics: 7.8.3 + simplify-geojson: 1.0.5 + topojson-client: 3.1.0 dev: true - /@visactor/vgrammar-core/0.11.11: - resolution: {integrity: sha512-3y1EPhWn95FaqA6UnNR+LNHzTM3zaJRUPLl29a9oUXVV6OgOTo05dEFyETuaybrxIDpPodqroBGU2yemikvgjA==} + /@visactor/vgrammar-coordinate/0.12.7: + resolution: {integrity: sha512-nJR506XRUnci4i/URc/6QIk4M0chjmhXX3nb/8DtpnpfKItFA976J6UuG1RJxVgYpcOHk3qIionX+oZotQnvIg==} dependencies: - '@visactor/vdataset': 0.17.4 - '@visactor/vgrammar-coordinate': 0.11.11 - '@visactor/vgrammar-util': 0.11.11 - '@visactor/vrender-components': 0.17.23 - '@visactor/vrender-core': 0.17.23 - '@visactor/vrender-kits': 0.17.23 - '@visactor/vscale': 0.17.4 - '@visactor/vutils': 0.17.4 + '@visactor/vgrammar-util': 0.12.7 + '@visactor/vutils': 0.18.1 dev: true - /@visactor/vgrammar-hierarchy/0.11.11: - resolution: {integrity: sha512-iJmUYV++GxguZgpeoH3g2C6b1Es25hXdSr7wvZyckQBEMhfXoLj8zDV00BNkMKO46gPNUGBq6lH84NHDmXfCAw==} + /@visactor/vgrammar-core/0.12.7: + resolution: {integrity: sha512-oaCqiCM/JYyDyh603oehaP7QATS9Ur9AkEvC6rmyxqjTXE1HHDXIIUtvPOgTuGJCi47mf23QCyrf+rAgefX5vQ==} dependencies: - '@visactor/vgrammar-core': 0.11.11 - '@visactor/vgrammar-util': 0.11.11 - '@visactor/vrender-core': 0.17.23 - '@visactor/vrender-kits': 0.17.23 - '@visactor/vutils': 0.17.4 + '@visactor/vdataset': 0.18.1 + '@visactor/vgrammar-coordinate': 0.12.7 + '@visactor/vgrammar-util': 0.12.7 + '@visactor/vrender-components': 0.18.7 + '@visactor/vrender-core': 0.18.7 + '@visactor/vrender-kits': 0.18.7 + '@visactor/vscale': 0.18.1 + '@visactor/vutils': 0.18.1 dev: true - /@visactor/vgrammar-projection/0.11.11: - resolution: {integrity: sha512-fop112v6EROgZQA4jOpkRa5MFymo7JrfUybIt1d0qZzW2TDHNP1BCDZl6GmODMflBNXs03zalwgzj/GnEBC6bw==} + /@visactor/vgrammar-hierarchy/0.12.7: + resolution: {integrity: sha512-jpBQplqsheyFU0RmiyCGXm0ppCPZyganfR3Z/IudP3VG4UaRKkY6TxYw76wTcqqcIvRDfo4rWwTq/TnWI7f4Gg==} dependencies: - '@visactor/vgrammar-core': 0.11.11 - '@visactor/vgrammar-util': 0.11.11 - '@visactor/vutils': 0.17.4 + '@visactor/vgrammar-core': 0.12.7 + '@visactor/vgrammar-util': 0.12.7 + '@visactor/vrender-core': 0.18.7 + '@visactor/vrender-kits': 0.18.7 + '@visactor/vutils': 0.18.1 + dev: true + + /@visactor/vgrammar-projection/0.12.7: + resolution: {integrity: sha512-MMpMa4nd+gSFsyWjCMYZEpATYLYIMai+z7/G2yeZ102d0B+tXLLkgrPQkmxewRrbaGxmEpLqMxuNMMB8g1CKYw==} + dependencies: + '@visactor/vgrammar-core': 0.12.7 + '@visactor/vgrammar-util': 0.12.7 + '@visactor/vutils': 0.18.1 d3-geo: 1.12.1 dev: true - /@visactor/vgrammar-sankey/0.11.11: - resolution: {integrity: sha512-J98xvPW6aJXCJ/AIb0Mw5RU7HVCH63k7ZoQXyphbT850AZXN7s+cztY6TQk9/CHe2Fv536Wf+bHe8mdbty5jlw==} + /@visactor/vgrammar-sankey/0.12.7: + resolution: {integrity: sha512-Ax5qeeuLhBgKv9JcRjM/1+vA3mGqK/LlcTWEA0yN31KiHwse9THH2n93MlDhU/HkX0yQ7tFeMAKXxwUFZsVT+A==} dependencies: - '@visactor/vgrammar-core': 0.11.11 - '@visactor/vgrammar-util': 0.11.11 - '@visactor/vrender-core': 0.17.23 - '@visactor/vrender-kits': 0.17.23 - '@visactor/vutils': 0.17.4 + '@visactor/vgrammar-core': 0.12.7 + '@visactor/vgrammar-util': 0.12.7 + '@visactor/vrender-core': 0.18.7 + '@visactor/vrender-kits': 0.18.7 + '@visactor/vutils': 0.18.1 dev: true - /@visactor/vgrammar-util/0.11.11: - resolution: {integrity: sha512-3R4KdQ2j0JH0ZeM8eMSoa8RM/RXHWNv3AFiO5ebzBZDALzkgBvcYmKmyMEINTkakRlojEFpaP27LqkCL0CpYPg==} + /@visactor/vgrammar-util/0.12.7: + resolution: {integrity: sha512-rxKy9Y2PMGxJX9B+bMOdQqQ5604tWvXiDxSTnPt2HtzBKGLnTj7j7dUfKdxBlVvaQnlMjaeLr4OdKnA3Yh9KhA==} dependencies: - '@visactor/vutils': 0.17.4 + '@visactor/vutils': 0.18.1 dev: true - /@visactor/vgrammar-wordcloud-shape/0.11.11: - resolution: {integrity: sha512-kzPJgfZonxWUVsKNn0ov81KIrzJAV3d6MQm6A/yYWfJvqF4T44iB38FRo9Ox+uBZ19wLqnx8ueE2WgId3Far5w==} + /@visactor/vgrammar-wordcloud-shape/0.12.7: + resolution: {integrity: sha512-9QjyxiVxNdP9tddCyI1W20aWSfMHkigNsz7/J4njCLJlSE3AN3+bZa6llMparNGWTFldHg3Hxlsbp5/40g+F3A==} dependencies: - '@visactor/vgrammar-core': 0.11.11 - '@visactor/vgrammar-util': 0.11.11 - '@visactor/vrender-core': 0.17.23 - '@visactor/vrender-kits': 0.17.23 - '@visactor/vscale': 0.17.4 - '@visactor/vutils': 0.17.4 + '@visactor/vgrammar-core': 0.12.7 + '@visactor/vgrammar-util': 0.12.7 + '@visactor/vrender-core': 0.18.7 + '@visactor/vrender-kits': 0.18.7 + '@visactor/vscale': 0.18.1 + '@visactor/vutils': 0.18.1 dev: true - /@visactor/vgrammar-wordcloud/0.11.11: - resolution: {integrity: sha512-OPThALDHaXKBw9EadNuFhUoOBD+vvtxTvdV+YxJrZ7B1dDfr8lJ896ACNqacQ9lUikHJdO8DhutdcS5FQw1/gg==} + /@visactor/vgrammar-wordcloud/0.12.7: + resolution: {integrity: sha512-kx3/GZ3xL7B1oPOJf25iC+c3GeArFFhi8eUaRCQYrSkCkVhZ7EEVJfKe2kxgdgXyj6Di/FgLh8g3hcBSaIlAnA==} dependencies: - '@visactor/vgrammar-core': 0.11.11 - '@visactor/vgrammar-util': 0.11.11 - '@visactor/vrender-core': 0.17.23 - '@visactor/vrender-kits': 0.17.23 - '@visactor/vutils': 0.17.4 + '@visactor/vgrammar-core': 0.12.7 + '@visactor/vgrammar-util': 0.12.7 + '@visactor/vrender-core': 0.18.7 + '@visactor/vrender-kits': 0.18.7 + '@visactor/vutils': 0.18.1 dev: true - /@visactor/vrender-components/0.17.23: - resolution: {integrity: sha512-unFzLVodABDVh5RQA3ls/eR6mLHoy/nx6yyoMm6VibuHZUYORI7W3x30vgFHn+POo7yPV3aVjB56Y2IUH7FrtQ==} + /@visactor/vrender-components/0.18.7: + resolution: {integrity: sha512-nQVW8skqklijPqLJb14wtyrUYqdCDvicfPF40rXszpmwF8SZrlN5ZMgor+afr+7/mxLYHHYlI9k9ecvTSfyHEw==} dependencies: - '@visactor/vrender-core': 0.17.23 - '@visactor/vrender-kits': 0.17.23 + '@visactor/vrender-core': 0.18.7 + '@visactor/vrender-kits': 0.18.7 '@visactor/vscale': 0.17.4 - '@visactor/vutils': 0.17.4 + '@visactor/vutils': 0.18.1 dev: true /@visactor/vrender-core/0.17.23: @@ -3267,12 +3460,19 @@ packages: color-convert: 2.0.1 dev: true - /@visactor/vrender-kits/0.17.23: - resolution: {integrity: sha512-3Mnr5y62ZTeZweaEGNLMM8Wf3vkf2K8aTiW4kinXkk0Am0k78v/0kdkwaYEnNM89uJ4zRVXPkkLG6h+/4KMo8Q==} + /@visactor/vrender-core/0.18.7: + resolution: {integrity: sha512-mjqJpgsQo+sbsF9UNkFNQNFaO0AqPX+6uxnwoMZ1q3LJoQ8p/cRvDAez/dYT4Z1ZqdaB8GgRXYLp+vy2+P2lUA==} + dependencies: + '@visactor/vutils': 0.18.1 + color-convert: 2.0.1 + dev: true + + /@visactor/vrender-kits/0.18.7: + resolution: {integrity: sha512-UG3WKMBufnQa2cPGhJnpd6/e5NdH3FQgDfE/FA/UKxTBqfG1bSS2KkK7Y+bJJwueCMRZQx3d4ksPeW1H3vOnMg==} dependencies: '@resvg/resvg-js': 2.4.1 - '@visactor/vrender-core': 0.17.23 - '@visactor/vutils': 0.17.4 + '@visactor/vrender-core': 0.18.7 + '@visactor/vutils': 0.18.1 roughjs: 4.5.2 dev: true @@ -3282,13 +3482,19 @@ packages: '@visactor/vutils': 0.17.4 dev: true - /@visactor/vutils-extension/1.9.2: - resolution: {integrity: sha512-qOL5lsUx+RSM3/A/kEJB3+50QPC+wQs2tEclrfa7EXWdXQzVPNlcU2evDikTJska6b/WlMJIdtdNvbAhaTz51A==} + /@visactor/vscale/0.18.1: + resolution: {integrity: sha512-0wpd0avbFLvuDKNHt2PxdKdqLSU9+zUkM6GJYWbXsUUYOiKaFkt2xTkdwUHKq66v23C7Iy14Pm7VVr0wVgflbA==} dependencies: - '@visactor/vrender-core': 0.17.23 - '@visactor/vrender-kits': 0.17.23 - '@visactor/vscale': 0.17.4 - '@visactor/vutils': 0.17.4 + '@visactor/vutils': 0.18.1 + dev: true + + /@visactor/vutils-extension/1.10.2: + resolution: {integrity: sha512-xX4SAAg9YG8iSCWH4tnaWYOZCtqVHtZ6bzX1u9255qN0FbEEll12kWBZEiSn1kX4vQCMbbbs/jjX/WDOLPwuIw==} + dependencies: + '@visactor/vrender-core': 0.18.7 + '@visactor/vrender-kits': 0.18.7 + '@visactor/vscale': 0.18.1 + '@visactor/vutils': 0.18.1 dev: true /@visactor/vutils/0.17.4: @@ -3298,6 +3504,14 @@ packages: '@turf/invariant': 6.5.0 eventemitter3: 4.0.7 + /@visactor/vutils/0.18.1: + resolution: {integrity: sha512-XGq9a85HrVP3Rbby1qO2/JS9GewJtZv6y35Xujcb2ZGLEjnpCK61Y1OXwSC5SZOKmtsH4SjYMf5czlnNhQ3GeA==} + dependencies: + '@turf/helpers': 6.5.0 + '@turf/invariant': 6.5.0 + eventemitter3: 4.0.7 + dev: true + /@vitejs/plugin-react/3.1.0_vite@3.2.6: resolution: {integrity: sha512-AfgcRL8ZBhAlc3BFdigClmTUMISmmzHn7sB2h9U1odvc5U/MjWXsAaz18b/WoppUTDBzxOJwo2VdClfUcItu9g==} engines: {node: ^14.18.0 || >=16.0.0} @@ -3459,6 +3673,17 @@ packages: uri-js: 4.4.1 dev: true + /alasql/4.3.2: + resolution: {integrity: sha512-eil4Y/GQx0LP7P2dOiG8+N+jESibN2t/8cnVcNW6kuatPLkW7bCxC/vKINxJNOfyReeisco9nW4kjWg75bTnDg==} + engines: {node: '>=15'} + hasBin: true + dependencies: + cross-fetch: 4.0.0 + yargs: 16.2.0 + transitivePeerDependencies: + - encoding + dev: false + /ansi-colors/1.1.0: resolution: {integrity: sha512-SFKX67auSNoVR38N3L+nvsPjOE0bybKTYbkf5tRvushrAPQ9V75huw0ZxBkKVeRU9kqH3d6HA4xTckbwZ4ixmA==} engines: {node: '>=0.10.0'} @@ -4536,7 +4761,6 @@ packages: string-width: 4.2.3 strip-ansi: 6.0.1 wrap-ansi: 7.0.0 - dev: true /cliui/8.0.1: resolution: {integrity: sha512-BSeNnyus75C4//NQ9gQt1/csTXyo/8Sb+afLAkzAptFuMsod9HFokGNudZpi/oQV73hnVK+sR+5PVRMd+Dr7YQ==} @@ -4773,6 +4997,14 @@ packages: /create-require/1.1.1: resolution: {integrity: sha512-dcKFX3jn0MpIaXjisoRvexIJVEKzaq7z2rZKxf+MSr9TkdmHmsU4m2lcLojrj/FHl8mk5VxMmYA+ftRkP/3oKQ==} + /cross-fetch/4.0.0: + resolution: {integrity: sha512-e4a5N8lVvuLgAWgnCrLr2PP0YyDOTHa9H/Rj54dirp61qXnNq46m82bRhNqIA5VccJtWBvPTFRV3TtvHUKPB1g==} + dependencies: + node-fetch: 2.7.0 + transitivePeerDependencies: + - encoding + dev: false + /cross-spawn/6.0.5: resolution: {integrity: sha512-eTVLrBSt7fjbDygz805pMnstIs2VTBNkRm0qxZd+M7A5XDdxVRWO5MxGBXZhjY4cqLYLdtrGqRf8mBPmzwSpWQ==} engines: {node: '>=4.8'} @@ -5812,6 +6044,15 @@ packages: esrecurse: 4.3.0 estraverse: 5.3.0 + /eslint-utils/3.0.0: + resolution: {integrity: sha512-uuQC43IGctw68pJA1RgbQS8/NP7rch6Cwd4j3ZBtgo4/8Flj4eGE7ZYSZRN3iq5pVUv6GPdW5Z1RFleo84uLDA==} + engines: {node: ^10.0.0 || ^12.0.0 || >= 14.0.0} + peerDependencies: + eslint: '>=5' + dependencies: + eslint-visitor-keys: 2.1.0 + dev: true + /eslint-utils/3.0.0_eslint@8.18.0: resolution: {integrity: sha512-uuQC43IGctw68pJA1RgbQS8/NP7rch6Cwd4j3ZBtgo4/8Flj4eGE7ZYSZRN3iq5pVUv6GPdW5Z1RFleo84uLDA==} engines: {node: ^10.0.0 || ^12.0.0 || >= 14.0.0} @@ -7576,6 +7817,32 @@ packages: throat: 5.0.0 dev: true + /jest-cli/26.6.3: + resolution: {integrity: sha512-GF9noBSa9t08pSyl3CY4frMrqp+aQXFGFkf5hEPbh/pIUFYWMK6ZLTfbmadxJVcJrdRoChlWQsA2VkJcDFK8hg==} + engines: {node: '>= 10.14.2'} + hasBin: true + dependencies: + '@jest/core': 26.6.3 + '@jest/test-result': 26.6.2 + '@jest/types': 26.6.2 + chalk: 4.1.2 + exit: 0.1.2 + graceful-fs: 4.2.11 + import-local: 3.1.0 + is-ci: 2.0.0 + jest-config: 26.6.3 + jest-util: 26.6.2 + jest-validate: 26.6.2 + prompts: 2.4.2 + yargs: 15.4.1 + transitivePeerDependencies: + - bufferutil + - canvas + - supports-color + - ts-node + - utf-8-validate + dev: true + /jest-cli/26.6.3_canvas@2.11.2: resolution: {integrity: sha512-GF9noBSa9t08pSyl3CY4frMrqp+aQXFGFkf5hEPbh/pIUFYWMK6ZLTfbmadxJVcJrdRoChlWQsA2VkJcDFK8hg==} engines: {node: '>= 10.14.2'} @@ -7653,6 +7920,40 @@ packages: - supports-color dev: true + /jest-config/26.6.3: + resolution: {integrity: sha512-t5qdIj/bCj2j7NFVHb2nFB4aUdfucDn3JRKgrZnplb8nieAirAzRSHP8uDEd+qV6ygzg9Pz4YG7UTJf94LPSyg==} + engines: {node: '>= 10.14.2'} + peerDependencies: + ts-node: '>=9.0.0' + peerDependenciesMeta: + ts-node: + optional: true + dependencies: + '@babel/core': 7.20.12 + '@jest/test-sequencer': 26.6.3 + '@jest/types': 26.6.2 + babel-jest: 26.6.3_@babel+core@7.20.12 + chalk: 4.1.2 + deepmerge: 4.3.1 + glob: 7.2.3 + graceful-fs: 4.2.11 + jest-environment-jsdom: 26.6.2 + jest-environment-node: 26.6.2 + jest-get-type: 26.3.0 + jest-jasmine2: 26.6.3 + jest-regex-util: 26.0.0 + jest-resolve: 26.6.2 + jest-util: 26.6.2 + jest-validate: 26.6.2 + micromatch: 4.0.5 + pretty-format: 26.6.2 + transitivePeerDependencies: + - bufferutil + - canvas + - supports-color + - utf-8-validate + dev: true + /jest-config/26.6.3_canvas@2.11.2: resolution: {integrity: sha512-t5qdIj/bCj2j7NFVHb2nFB4aUdfucDn3JRKgrZnplb8nieAirAzRSHP8uDEd+qV6ygzg9Pz4YG7UTJf94LPSyg==} engines: {node: '>= 10.14.2'} @@ -7947,6 +8248,36 @@ packages: - supports-color dev: true + /jest-jasmine2/26.6.3: + resolution: {integrity: sha512-kPKUrQtc8aYwBV7CqBg5pu+tmYXlvFlSFYn18ev4gPFtrRzB15N2gW/Roew3187q2w2eHuu0MU9TJz6w0/nPEg==} + engines: {node: '>= 10.14.2'} + dependencies: + '@babel/traverse': 7.23.9 + '@jest/environment': 26.6.2 + '@jest/source-map': 26.6.2 + '@jest/test-result': 26.6.2 + '@jest/types': 26.6.2 + '@types/node': 20.11.16 + chalk: 4.1.2 + co: 4.6.0 + expect: 26.6.2 + is-generator-fn: 2.1.0 + jest-each: 26.6.2 + jest-matcher-utils: 26.6.2 + jest-message-util: 26.6.2 + jest-runtime: 26.6.3 + jest-snapshot: 26.6.2 + jest-util: 26.6.2 + pretty-format: 26.6.2 + throat: 5.0.0 + transitivePeerDependencies: + - bufferutil + - canvas + - supports-color + - ts-node + - utf-8-validate + dev: true + /jest-jasmine2/26.6.3_canvas@2.11.2: resolution: {integrity: sha512-kPKUrQtc8aYwBV7CqBg5pu+tmYXlvFlSFYn18ev4gPFtrRzB15N2gW/Roew3187q2w2eHuu0MU9TJz6w0/nPEg==} engines: {node: '>= 10.14.2'} @@ -8182,6 +8513,38 @@ packages: - supports-color dev: true + /jest-runner/26.6.3: + resolution: {integrity: sha512-atgKpRHnaA2OvByG/HpGA4g6CSPS/1LK0jK3gATJAoptC1ojltpmVlYC3TYgdmGp+GLuhzpH30Gvs36szSL2JQ==} + engines: {node: '>= 10.14.2'} + dependencies: + '@jest/console': 26.6.2 + '@jest/environment': 26.6.2 + '@jest/test-result': 26.6.2 + '@jest/types': 26.6.2 + '@types/node': 20.11.16 + chalk: 4.1.2 + emittery: 0.7.2 + exit: 0.1.2 + graceful-fs: 4.2.11 + jest-config: 26.6.3 + jest-docblock: 26.0.0 + jest-haste-map: 26.6.2 + jest-leak-detector: 26.6.2 + jest-message-util: 26.6.2 + jest-resolve: 26.6.2 + jest-runtime: 26.6.3 + jest-util: 26.6.2 + jest-worker: 26.6.2 + source-map-support: 0.5.21 + throat: 5.0.0 + transitivePeerDependencies: + - bufferutil + - canvas + - supports-color + - ts-node + - utf-8-validate + dev: true + /jest-runner/26.6.3_canvas@2.11.2: resolution: {integrity: sha512-atgKpRHnaA2OvByG/HpGA4g6CSPS/1LK0jK3gATJAoptC1ojltpmVlYC3TYgdmGp+GLuhzpH30Gvs36szSL2JQ==} engines: {node: '>= 10.14.2'} @@ -8278,6 +8641,46 @@ packages: - supports-color dev: true + /jest-runtime/26.6.3: + resolution: {integrity: sha512-lrzyR3N8sacTAMeonbqpnSka1dHNux2uk0qqDXVkMv2c/A3wYnvQ4EXuI013Y6+gSKSCxdaczvf4HF0mVXHRdw==} + engines: {node: '>= 10.14.2'} + hasBin: true + dependencies: + '@jest/console': 26.6.2 + '@jest/environment': 26.6.2 + '@jest/fake-timers': 26.6.2 + '@jest/globals': 26.6.2 + '@jest/source-map': 26.6.2 + '@jest/test-result': 26.6.2 + '@jest/transform': 26.6.2 + '@jest/types': 26.6.2 + '@types/yargs': 15.0.19 + chalk: 4.1.2 + cjs-module-lexer: 0.6.0 + collect-v8-coverage: 1.0.2 + exit: 0.1.2 + glob: 7.2.3 + graceful-fs: 4.2.11 + jest-config: 26.6.3 + jest-haste-map: 26.6.2 + jest-message-util: 26.6.2 + jest-mock: 26.6.2 + jest-regex-util: 26.0.0 + jest-resolve: 26.6.2 + jest-snapshot: 26.6.2 + jest-util: 26.6.2 + jest-validate: 26.6.2 + slash: 3.0.0 + strip-bom: 4.0.0 + yargs: 15.4.1 + transitivePeerDependencies: + - bufferutil + - canvas + - supports-color + - ts-node + - utf-8-validate + dev: true + /jest-runtime/26.6.3_canvas@2.11.2: resolution: {integrity: sha512-lrzyR3N8sacTAMeonbqpnSka1dHNux2uk0qqDXVkMv2c/A3wYnvQ4EXuI013Y6+gSKSCxdaczvf4HF0mVXHRdw==} engines: {node: '>= 10.14.2'} @@ -8496,6 +8899,22 @@ packages: supports-color: 7.2.0 dev: true + /jest/26.6.3: + resolution: {integrity: sha512-lGS5PXGAzR4RF7V5+XObhqz2KZIDUA1yD0DG6pBVmy10eh0ZIXQImRuzocsI/N2XZ1GrLFwTS27In2i2jlpq1Q==} + engines: {node: '>= 10.14.2'} + hasBin: true + dependencies: + '@jest/core': 26.6.3 + import-local: 3.1.0 + jest-cli: 26.6.3 + transitivePeerDependencies: + - bufferutil + - canvas + - supports-color + - ts-node + - utf-8-validate + dev: true + /jest/26.6.3_canvas@2.11.2: resolution: {integrity: sha512-lGS5PXGAzR4RF7V5+XObhqz2KZIDUA1yD0DG6pBVmy10eh0ZIXQImRuzocsI/N2XZ1GrLFwTS27In2i2jlpq1Q==} engines: {node: '>= 10.14.2'} @@ -9476,7 +9895,6 @@ packages: optional: true dependencies: whatwg-url: 5.0.0 - dev: true /node-int64/0.4.0: resolution: {integrity: sha512-O5lz91xSOeoXP6DulyHfllpq+Eg00MWitZIbtPfoSEvqIHdl5gfcY6hYzDWnj0qD5tz52PI08u9qUvSVeUBeHw==} @@ -11735,7 +12153,6 @@ packages: /tr46/0.0.3: resolution: {integrity: sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==} - dev: true /tr46/1.0.1: resolution: {integrity: sha512-dTpowEjclQ7Kgx5SdBkqRzVhERQXov8/l9Ft9dVM9fmg0W0KQSVaXX9T4i6twCPNtYiZM53lpSSUAwJbFPOHxA==} @@ -11997,6 +12414,10 @@ packages: /undici-types/5.26.5: resolution: {integrity: sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==} + /undici-types/5.28.3: + resolution: {integrity: sha512-VJD0un4i6M1/lFOJPhacHdq6FadtlkdhKBed2W6yBqmrAr/W58oqENaOIX031stDVFwz9AemOLkIj/2AXAMLCg==} + dev: true + /unicode-canonical-property-names-ecmascript/2.0.0: resolution: {integrity: sha512-yY5PpDlfVIU5+y/BSCxAJRBIS1Zc2dDG3Ujq+sR0U+JjUevW2JhocOF+soROYDSaAezOzOKuyyixhD6mBknSmQ==} engines: {node: '>=4'} @@ -12484,7 +12905,6 @@ packages: /webidl-conversions/3.0.1: resolution: {integrity: sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==} - dev: true /webidl-conversions/4.0.2: resolution: {integrity: sha512-YQ+BmxuTgd6UXZW3+ICGfyqRyHXVlD5GtQr5+qjiNW7bF0cqrzX500HVXPBOvgXb5YnzDd+h0zqyv61KUD7+Sg==} @@ -12520,7 +12940,6 @@ packages: dependencies: tr46: 0.0.3 webidl-conversions: 3.0.1 - dev: true /whatwg-url/6.5.0: resolution: {integrity: sha512-rhRZRqx/TLJQWUpQ6bmrt2UV4f0HCQ463yQuONJqC6fO2VoEb1pTYddbe59SkYq87aoM5A3bdhMZiUiVws+fzQ==} @@ -12764,7 +13183,6 @@ packages: /yargs-parser/20.2.9: resolution: {integrity: sha512-y11nGElTIV+CT3Zv9t7VKl+Q3hTQoT9a1Qzezhhl6Rp21gJ/IVTW7Z3y9EWXhuUBC2Shnf+DX0antecpAwSP8w==} engines: {node: '>=10'} - dev: true /yargs-parser/21.1.1: resolution: {integrity: sha512-tVpsJW7DdjecAiFpbIB1e3qxIQsE6NoPc5/eTdrbbIC4h0LVsWhnoa3g+m2HclBIujHzsxZ4VJVA+GUuc2/LBw==} @@ -12830,7 +13248,6 @@ packages: string-width: 4.2.3 y18n: 5.0.8 yargs-parser: 20.2.9 - dev: true /yargs/17.7.2: resolution: {integrity: sha512-7dSzzRQ++CKnNI/krKnYRV7JKKPUXMEh61soaHKg9mrWEhzFWhFnxPxGl+69cD1Ou63C13NUPCnmIcrvqCuM6w==} diff --git a/common/config/rush/version-policies.json b/common/config/rush/version-policies.json index 66cc743f..8bdf9a65 100644 --- a/common/config/rush/version-policies.json +++ b/common/config/rush/version-policies.json @@ -2,7 +2,7 @@ { "definitionName": "lockStepVersion", "policyName": "vmindMin", - "version": "1.2.4", + "version": "1.2.5", "mainProject": "@visactor/vmind", "nextBump": "patch" } diff --git a/docs/assets/changelog/en/changelog.md b/docs/assets/changelog/en/changelog.md index e6e922f6..e69de29b 100644 --- a/docs/assets/changelog/en/changelog.md +++ b/docs/assets/changelog/en/changelog.md @@ -1,542 +0,0 @@ -# v0.17.18 - -2024-01-24 - - -**🆕 New feature** - -- **@visactor/vrender-components**: adjust the timing for label customLayoutFunc invocation -- **@visactor/vrender-components**: label component will sync maxLineWidth to maxWidth in richText -- **@visactor/vrender-kits**: compatible canvas in lynx env -- **@visactor/vrender-core**: support backgroundCornerRadius - -**🐛 Bug fix** - -- **@visactor/vrender-components**: event pos error when interactive in site -- **@visactor/vrender-kits**: fix issue with interface -- **@visactor/vrender-core**: fix issue with multiline text textBaseline, closed [#886](https://github.com/VisActor/VRender/issues/886) -- **@visactor/vrender-core**: fix issue with union empty bounds -- **@visactor/vrender-core**: richtext.textConfig supports number type text - - - -[more detail about v0.17.18](https://github.com/VisActor/VRender/releases/tag/v0.17.18) - -# v0.17.17 - -2024-01-22 - - -**🆕 New feature** - -- **@visactor/vrender-core**: html only append dom inside body -- **@visactor/vrender-core**: color support str gradient color -- **@visactor/vrender**: color support str gradient color - -**🐛 Bug fix** - -- **@visactor/vrender-components**: title support multiline -- **@visactor/vrender-kits**: fix issue with loaded tree-shaking -- **@visactor/vrender-core**: fix issue with rerun getTextBounds -- **@visactor/vrender-core**: fix issue with set image -- **@visactor/vrender-core**: fix issue with loaded tree-shaking - - - -[more detail about v0.17.17](https://github.com/VisActor/VRender/releases/tag/v0.17.17) - -# v0.17.16 - -2024-01-18 - - -**🆕 New feature** - -- **@visactor/vrender-core**: enable pass supportsPointerEvents and supportsTouchEvents - -**🐛 Bug fix** - -- **@visactor/vrender-components**: when no brush is active, brush should not call stopPropagation() - -[more detail about v0.17.16](https://github.com/VisActor/VRender/releases/tag/v0.17.16) - -# v0.17.15 - -2024-01-17 - - -**🆕 New feature** - -- **@visactor/vrender-components**: support boolean config in label -- **@visactor/vrender-core**: add supportsTouchEvents and supportsPointerEvents params - -**🐛 Bug fix** - -- **@visactor/vrender-components**: fix the flush of axis when axis label has rotate angle -- **@visactor/vrender-components**: arc label line not shown -- **@visactor/vrender-components**: error happens in line-label when line has no points -- **@visactor/vrender-core**: fix issue with html attribute -- **@visactor/vrender-core**: fix issue with env-check -- **@visactor/vrender-core**: fix issue with text background opacity - - - -[more detail about v0.17.15](https://github.com/VisActor/VRender/releases/tag/v0.17.15) - -# v0.17.14 - -2024-01-12 - -**🐛 Bug fix** -- **@visactor/vrender-core**: fix `splitRect` when rect has `x1` or `y1` -- **@visactor/vrender**: fix `splitRect` when rect has `x1` or `y1` - - -[more detail about v0.17.14](https://github.com/VisActor/VRender/releases/tag/v0.17.14) - -# v0.17.13 - -2024-01-10 - -**🆕 New feature** -- **@visactor/vrender-core**: background support opacity -**🐛 Bug fix** -- **@visactor/vrender-components**: filter out invisible indicator spec -- **@visactor/vrender-components**: `measureTextSize` needs to take into account the fonts configured on the stage theme -- **@visactor/vrender-core**: fix issue with incremental draw -- **@visactor/vrender-core**: supply the `getTheme()` api for `IStage` - - - -[more detail about v0.17.13](https://github.com/VisActor/VRender/releases/tag/v0.17.13) - -# v0.17.12 - -2024-01-10 - -**🆕 New feature** -- **@visactor/vrender-components**: support fit strategy for indicator -- **marker**: mark point support confine. fix @Visactor/VChart[#1573](https://github.com/VisActor/VRender/issues/1573) -**🐛 Bug fix** -- **marker**: fix problem of no render when set visible attr and add valid judgment logic. fix@Visactor/Vchart[#1901](https://github.com/VisActor/VRender/issues/1901) -- **datazoom**: adaptive handler text layout. fix@Visactor/VChart[#1809](https://github.com/VisActor/VRender/issues/1809) -- **datazoom**: set pickable false when zoomLock. fix @Visactor/VChart[#1565](https://github.com/VisActor/VRender/issues/1565) -- **datazoom**: handler not follow mouse after resize. fix@Visactor/Vchart[#1490](https://github.com/VisActor/VRender/issues/1490) -- **@visactor/vrender-components**: arc outside label invisible with visible label line - - - -[more detail about v0.17.12](https://github.com/VisActor/VRender/releases/tag/v0.17.12) - -# v0.17.11 - -2024-01-05 - -**🆕 New feature** -- **@visactor/vrender-core**: add backgroundFit attribute -**🐛 Bug fix** -- **@visactor/vrender-core**: fix issue with position in html attribute -- fix: label invisible when baseMark visible is false - -**Full Changelog**: https://github.com/VisActor/VRender/compare/v0.17.10...v0.17.11 - -[more detail about v0.17.11](https://github.com/VisActor/VRender/releases/tag/v0.17.11) - -# v0.17.10 - -2024-01-03 - -**🆕 New feature** -- **@visactor/vrender-components**: support `lastVisible` of LineAxis label -- **@visactor/vrender-kits**: support fillPickable and strokePickable for area, closed [#792](https://github.com/VisActor/VRender/issues/792) -- **@visactor/vrender-core**: support fillPickable and strokePickable for area, closed [#792](https://github.com/VisActor/VRender/issues/792) -- **@visactor/vrender-core**: support `lastVisible` of LineAxis label -- **@visactor/vrender**: support `lastVisible` of LineAxis label -**🐛 Bug fix** -- **@visactor/vrender-components**: fix the auto limit width of label when the label has vertical `direction` in orient top or bottom -- **@visactor/vrender-components**: fix issue with legend symbol size -- **@visactor/vrender-components**: fixed height calculation issue after multi-layer axis text rotation -- **@visactor/vrender-core**: fix issue with area-line highperformance draw -- **@visactor/vrender-core**: fix the auto limit width of label when the label has vertical `direction` in orient top or bottom -- **@visactor/vrender-core**: disable layer picker in interactive layer -- **@visactor/vrender**: fix the auto limit width of label when the label has vertical `direction` in orient top or bottom -**🔖 other** -- **@visactor/vrender-components**: 'feat: support label line in label component' - - - -[more detail about v0.17.10](https://github.com/VisActor/VRender/releases/tag/v0.17.10) - -# v0.17.9 - -2024-01-03 - -**🐛 Bug fix** -- **@visactor/vrender-components**: fix label position when offset is 0 -- **@visactor/vrender-core**: fix issue with conical cache not work as expect - - - -[more detail about v0.17.9](https://github.com/VisActor/VRender/releases/tag/v0.17.9) - -# v0.17.8 - -2023-12-29 - -**🆕 New feature** -- **@visactor/vrender-components**: optimize outer label layout in tangential direction -- **@visactor/vrender-core**: support drawGraphicToCanvas -- **@visactor/vrender**: support drawGraphicToCanvas -**🐛 Bug fix** -- **@visactor/vrender-components**: when axis label space is 0, and axis tick' inside is true, the axis label's position is not correct -- **@visactor/vrender-components**: fix morphing of rect -- **@visactor/vrender-kits**: fix issue with mapToCanvasPoint in miniapp, closed [#828](https://github.com/VisActor/VRender/issues/828) -- **@visactor/vrender-core**: fix issue with rect.toCustomPath -- **@visactor/vrender-core**: fix issue with area segment with single point, closed [#801](https://github.com/VisActor/VRender/issues/801) -- **@visactor/vrender-core**: fix issue with new Function in miniapp -- **@visactor/vrender-core**: fix morphing of rect -- **@visactor/vrender-core**: fix issue with side-effect in some env -- **@visactor/vrender-core**: fix issue with check tt env -- **@visactor/vrender-core**: fix issue with cliped attribute in vertical text, closed [#827](https://github.com/VisActor/VRender/issues/827) -- **@visactor/vrender**: fix issue with area segment with single point, closed [#801](https://github.com/VisActor/VRender/issues/801) -- **@visactor/vrender**: fix morphing of rect -- **@visactor/vrender**: fix issue with side-effect in some env - - - -[more detail about v0.17.8](https://github.com/VisActor/VRender/releases/tag/v0.17.8) - -# v0.17.7 - -2023-12-21 - -**🐛 Bug fix** -- **@visactor/vrender-kits**: fix issue with create layer in miniapp env -- **@visactor/vrender-core**: fix issue with create layer in miniapp env - - - -[more detail about v0.17.7](https://github.com/VisActor/VRender/releases/tag/v0.17.7) - -# v0.17.6 - -2023-12-20 - -**What's Changed** -* Main by @neuqzxy in https://github.com/VisActor/VRender/pull/813 -* fix: fix issue with rect stroke contribution by @neuqzxy in https://github.com/VisActor/VRender/pull/814 -* [Auto release] release 0.17.6 by @github-actions in https://github.com/VisActor/VRender/pull/815 - - -**Full Changelog**: https://github.com/VisActor/VRender/compare/v0.17.5...v0.17.6 - -[more detail about v0.17.6](https://github.com/VisActor/VRender/releases/tag/v0.17.6) - -# v0.17.5 - -2023-12-19 - -**🆕 New feature** -- **scrollbar**: dispatch scrollDown event -- **@visactor/vrender-components**: labelLine support animate -- **@visactor/vrender-components**: label don't create enter animate animationEnter while duration less than 0 -- **@visactor/vrender**: add disableAutoClipedPoptip attribute in text graphic -**🐛 Bug fix** -- **@visactor/vrender-components**: fix issue with arc animate with delayafter -- **@visactor/vrender-components**: fix issue with poptip circular dependencies -- **@visactor/vrender-core**: fix issue with plugin unregister -- **@visactor/vrender-core**: fix issue with text while whitespace is normal -- **@visactor/vrender**: fix cursor update error in multi-stage - - - -[more detail about v0.17.5](https://github.com/VisActor/VRender/releases/tag/v0.17.5) - -# v0.17.4 - -2023-12-15 - -**🐛 Bug fix** -- **datazoom**: symbol size problem -- **@visactor/vrender-core**: fix issue with arc imprecise bounds, closed [#728](https://github.com/VisActor/VRender/issues/728) - - - -[more detail about v0.17.4](https://github.com/VisActor/VRender/releases/tag/v0.17.4) - -# v0.17.3 - -2023-12-14 - -**🐛 Bug fix** -- **datazoom**: handler zindex to interaction error - - - -[more detail about v0.17.3](https://github.com/VisActor/VRender/releases/tag/v0.17.3) - -# v0.17.2 - -2023-12-14 - -**🆕 New feature** -- **dataZoom**: add mask to modify hot zone. feat @visactor/vchart[#1415](https://github.com/VisActor/VRender/issues/1415)' -- **dataZoom**: add mask to modify hot zone. feat @visactor/vchart[#1415](https://github.com/VisActor/VRender/issues/1415)' -- **dataZoom**: add mask to modify hot zone. feat @visactor/vchart[#1415](https://github.com/VisActor/VRender/issues/1415)' -- **dataZoom**: add mask to modify hot zone. feat @visactor/vchart[#1415](https://github.com/VisActor/VRender/issues/1415)' -- **dataZoom**: add mask to modify hot zone. feat @visactor/vchart[#1415](https://github.com/VisActor/VRender/issues/1415)' -- **@visactor/vrender-core**: rect3d support x1y1, fix -radius issue with rect -- **dataZoom**: add mask to modify hot zone. feat @visactor/vchart[#1415](https://github.com/VisActor/VRender/issues/1415)' -**🐛 Bug fix** -- **@visactor/vrender-components**: scrollbar slider width/height should not be negative -- **@visactor/vrender-components**: datazoom event block window event. fix @visactor/vchart[#1686](https://github.com/VisActor/VRender/issues/1686) -- **@visactor/vrender-components**: fix the issue of brushEnd trigger multiple times, related https://github.com/VisActor/VChart/issues/1694 -- **@visactor/vrender-core**: fix shadow pick issue -**⚡ Performance optimization** -- **@visactor/vrender-components**: optimize the `_handleStyle()` in legend - - - -[more detail about v0.17.2](https://github.com/VisActor/VRender/releases/tag/v0.17.2) - -# v0.17.1 - -2023-12-06 - -**🆕 New feature** -- **@visactor/vrender-kits**: support pickStrokeBuffer, closed [#758](https://github.com/VisActor/VRender/issues/758) -- **@visactor/vrender-core**: support pickStrokeBuffer, closed [#758](https://github.com/VisActor/VRender/issues/758) -**🐛 Bug fix** -- **@visactor/vrender-kits**: fix issue with rebind pick-contribution -- **@visactor/vrender-core**: fix issue in area chart with special points -- **@visactor/vrender-core**: fix issue with rebind pick-contribution -- **@visactor/vrender-core**: fix error with wrap text and normal whiteSpace text - - - -[more detail about v0.17.1](https://github.com/VisActor/VRender/releases/tag/v0.17.1) - -# v0.17.0 - -2023-11-30 - -**🆕 New feature** -- **@visactor/vrender-components**: optmize bounds performance -- **@visactor/vrender-kits**: rect support x1 and y1 -- **@visactor/vrender-kits**: optmize bounds performance -- **@visactor/vrender-core**: support disableCheckGraphicWidthOutRange to skip check if graphic out of range -- **@visactor/vrender-core**: rect support x1 and y1 -- **@visactor/vrender-core**: don't rewrite global reflect -- **@visactor/vrender-core**: text support background, closed [#711](https://github.com/VisActor/VRender/issues/711) -- **@visactor/vrender-core**: optmize bounds performance -- **@visactor/vrender**: don't rewrite global reflect -- **@visactor/vrender**: skip update bounds while render small node-tree, closed [#660](https://github.com/VisActor/VRender/issues/660) -- **@visactor/vrender**: optmize bounds performance -**🔨 Refactor** -- **@visactor/vrender-kits**: refact inversify completely, closed [#657](https://github.com/VisActor/VRender/issues/657) -- **@visactor/vrender-core**: refact inversify completely, closed [#657](https://github.com/VisActor/VRender/issues/657) -- **@visactor/vrender**: refact inversify completely, closed [#657](https://github.com/VisActor/VRender/issues/657) -**⚡ Performance optimization** -- **@visactor/vrender-components**: add option `skipDefault` to vrender-components -- **@visactor/vrender-core**: area support drawLinearAreaHighPerformance, closed [#672](https://github.com/VisActor/VRender/issues/672) - - - -[more detail about v0.17.0](https://github.com/VisActor/VRender/releases/tag/v0.17.0) - -# v0.16.18 - -2023-11-30 - -**🆕 New feature** -- **@visactor/vrender-components**: discrete legend's pager support position property -- **@visactor/vrender-core**: support suffixPosition, closed [#625](https://github.com/VisActor/VRender/issues/625) -- **@visactor/vrender**: support suffixPosition, closed [#625](https://github.com/VisActor/VRender/issues/625) -**🐛 Bug fix** -- **@visactor/vrender-kits**: doubletap should not be triggered when the target is different twice before and after -- **@visactor/vrender-core**: fix issue with attribute interpolate, closed [#741](https://github.com/VisActor/VRender/issues/741) -- **@visactor/vrender-core**: fix issue about calcuate bounds with shadow, closed [#474](https://github.com/VisActor/VRender/issues/474) -- **@visactor/vrender-core**: fix issue with white line in some dpr device, closed [#666](https://github.com/VisActor/VRender/issues/666) -**🔨 Refactor** -- **@visactor/vrender-components**: move getSizeHandlerPath out of sizlegend -- **@visactor/vrender-core**: event-related coordinate points do not require complex Point classes - - - -[more detail about v0.16.18](https://github.com/VisActor/VRender/releases/tag/v0.16.18) - -# v0.16.17 - -2023-11-23 - -**🆕 New feature** -- **@visactor/vrender-components**: support rich text for label, axis, marker,tooltip, indicator and title -- **@visactor/vrender-components**: add mode type of smartInvert -- **@visactor/vrender-components**: place more label for overlapPadding case -- **@visactor/vrender-kits**: support 'tap' gesture for Gesture plugin -- **@visactor/vrender-core**: add `event` config for Stage params, which can configure `clickInterval` and some other options in eventSystem -- **@visactor/vrender-core**: support fill and stroke while svg don't support, closed [#710](https://github.com/VisActor/VRender/issues/710) -**🐛 Bug fix** -- **@visactor/vrender-kits**: \`pickMode: 'imprecise'\` not work in polygon -- **@visactor/vrender-core**: richtext may throw error when textConfig is null -- **@visactor/vrender-core**: fix issue with image repeat, closed [#712](https://github.com/VisActor/VRender/issues/712) -- **@visactor/vrender-core**: fix issue with restore and save count not equal -**⚡ Performance optimization** -- **@visactor/vrender-core**: not setAttribute while background is not url, closed [#696](https://github.com/VisActor/VRender/issues/696) - - - -[more detail about v0.16.17](https://github.com/VisActor/VRender/releases/tag/v0.16.17) - -# v0.16.16 - -2023-11-17 - -**🐛 Bug fix** -- **@visactor/vrender-components**: fix the issue of legend item.shape can not set visible, related https://github.com/VisActor/VChart/issues/1508 -- **@visactor/vrender-core**: assign symbol rect function to old - - - -[more detail about v0.16.16](https://github.com/VisActor/VRender/releases/tag/v0.16.16) - -# v0.16.15 - -2023-11-16 - -**🐛 Bug fix** -- **@visactor/vrender-compoments**: legendItemHover and legendItemUnHover should trigger once - - - -[more detail about v0.16.15](https://github.com/VisActor/VRender/releases/tag/v0.16.15) - -# v0.16.14 - -2023-11-15 - -**🆕 New feature** -- **@visactor/vrender-components**: datazoom update callback supports new trigger tag param -- **@visactor/vrender-components**: support line/area label -- **@visactor/vrender-components**: lineHeight support string, which means percent -- **@visactor/vrender-core**: add round line symbol, closed [#1458](https://github.com/VisActor/VRender/issues/1458) -- **@visactor/vrender-core**: lineHeight support string, which means percent -**🐛 Bug fix** -- **@visactor/vrender-core**: fix issue with render while in scale transform - - - -[more detail about v0.16.14](https://github.com/VisActor/VRender/releases/tag/v0.16.14) - -# v0.16.13 - -2023-11-15 - -**🆕 New feature** -- **@visactor/vrender-core**: add preventRender function -- **@visactor/vrender-core**: merge wrap text function to text -**🐛 Bug fix** -- **@visactor/vrender-kits**: temp fix issue with lynx measuretext - - - -[more detail about v0.16.13](https://github.com/VisActor/VRender/releases/tag/v0.16.13) - -# v0.16.12 - -2023-11-07 - -**🆕 New feature** -- **@visactor/vrender-core**: optimize text increase animation -**🐛 Bug fix** -- **@visactor/vrender-components**: padding of title component -- **@visactor/vrender-components**: padding offset of AABBbounds -- **@visactor/vrender-kits**: fix node-canvas max count issue -- **@visactor/vrender-core**: fix node-canvas max count issue - - - -[more detail about v0.16.12](https://github.com/VisActor/VRender/releases/tag/v0.16.12) - -# v0.16.11 - -2023-11-07 - -**🐛 Bug fix** -- **@visactor/vrender-components**: optimize the auto-overlap of axis label, which use rotateBounds when text rotated, relate https://github.com/VisActor/VChart/issues/133 -- **@visactor/vrender-components**: flush should not sue width height -- **@visactor/vrender-components**: fix the lastvisible logic of axis's auto-hide -- **@visactor/vrender-kits**: fix issue with xul and html attribute, closed [#634](https://github.com/VisActor/VRender/issues/634) -- **@visactor/vrender-core**: fix issue with xul and html attribute, closed [#634](https://github.com/VisActor/VRender/issues/634) - - - -[more detail about v0.16.11](https://github.com/VisActor/VRender/releases/tag/v0.16.11) - -# v0.16.10 - -2023-11-02 - -**What's Changed** -* Sync main by @neuqzxy in https://github.com/VisActor/VRender/pull/640 -* fix: fix issue with xul and html attribute, closed [#634](https://github.com/VisActor/VRender/issues/634) by @neuqzxy in https://github.com/VisActor/VRender/pull/635 -* Echance/axis auto rotate by @kkxxkk2019 in https://github.com/VisActor/VRender/pull/633 -* [Auto release] release 0.16.9 by @github-actions in https://github.com/VisActor/VRender/pull/641 - - -**Full Changelog**: https://github.com/VisActor/VRender/compare/v0.16.9...v0.16.10 - -[more detail about v0.16.10](https://github.com/VisActor/VRender/releases/tag/v0.16.10) - -# v0.16.9 - -2023-10-27 - -**🆕 New feature** -- **@visactor/vrender-components**: add checkbox indeterminate state -- **label**: rect label support position `top-right`|`top-left`|`bottom-righ`|`bottom-left` -- **@visactor/vrender-core**: stage background support image -**🐛 Bug fix** -- **@visactor/vrender-components**: all the group container of marker do not trigger event -- **datazoom**: text bounds when visible is false. fix VisActor/VChart[#1281](https://github.com/VisActor/VRender/issues/1281) - - - -[more detail about v0.16.9](https://github.com/VisActor/VRender/releases/tag/v0.16.9) - -# v0.16.8 - -2023-10-23 - -**🐛 Bug fix** -- **@visactor/vrender-components**: fix the issue of error position of focus when legend item just has label - - - -[more detail about v0.16.8](https://github.com/VisActor/VRender/releases/tag/v0.16.8) - -# v0.16.7 - -2023-10-23 - -**🐛 Bug fix** -- **label**: fix the issue that `clampForce` does not work when`overlapPadding` is configured -- **@visactor/vrender-core**: fix issue with creating multi chart in miniapp - - - -[more detail about v0.16.7](https://github.com/VisActor/VRender/releases/tag/v0.16.7) - -# v0.16.6 - -2023-10-23 - -**🆕 New feature** -- **@visactor/vrender-components**: optimize the layout method of circle axis label -**🐛 Bug fix** -- **@visactor/vrender-components**: fix the layout issue of legend item because of the error logic of `focusStartX` - - - -[more detail about v0.16.6](https://github.com/VisActor/VRender/releases/tag/v0.16.6) - diff --git a/docs/assets/changelog/zh/changelog.md b/docs/assets/changelog/zh/changelog.md index bdf0cc2e..e69de29b 100644 --- a/docs/assets/changelog/zh/changelog.md +++ b/docs/assets/changelog/zh/changelog.md @@ -1,542 +0,0 @@ -# v0.17.18 - -2024-01-24 - - -**🆕 新增功能** - -- **@visactor/vrender-components**: adjust the timing for label customLayoutFunc invocation -- **@visactor/vrender-components**: label component will sync maxLineWidth to maxWidth in richText -- **@visactor/vrender-kits**: compatible canvas in lynx env -- **@visactor/vrender-core**: support backgroundCornerRadius - -**🐛 功能修复** - -- **@visactor/vrender-components**: event pos error when interactive in site -- **@visactor/vrender-kits**: fix issue with interface -- **@visactor/vrender-core**: fix issue with multiline text textBaseline, closed [#886](https://github.com/VisActor/VRender/issues/886) -- **@visactor/vrender-core**: fix issue with union empty bounds -- **@visactor/vrender-core**: richtext.textConfig supports number type text - - - -[更多详情请查看 v0.17.18](https://github.com/VisActor/VRender/releases/tag/v0.17.18) - -# v0.17.17 - -2024-01-22 - - -**🆕 新增功能** - -- **@visactor/vrender-core**: html only append dom inside body -- **@visactor/vrender-core**: color support str gradient color -- **@visactor/vrender**: color support str gradient color - -**🐛 功能修复** - -- **@visactor/vrender-components**: title support multiline -- **@visactor/vrender-kits**: fix issue with loaded tree-shaking -- **@visactor/vrender-core**: fix issue with rerun getTextBounds -- **@visactor/vrender-core**: fix issue with set image -- **@visactor/vrender-core**: fix issue with loaded tree-shaking - - - -[更多详情请查看 v0.17.17](https://github.com/VisActor/VRender/releases/tag/v0.17.17) - -# v0.17.16 - -2024-01-18 - - -**🆕 新增功能** - -- **@visactor/vrender-core**: enable pass supportsPointerEvents and supportsTouchEvents - -**🐛 功能修复** - -- **@visactor/vrender-components**: when no brush is active, brush should not call stopPropagation() - -[更多详情请查看 v0.17.16](https://github.com/VisActor/VRender/releases/tag/v0.17.16) - -# v0.17.15 - -2024-01-17 - - -**🆕 新增功能** - -- **@visactor/vrender-components**: support boolean config in label -- **@visactor/vrender-core**: add supportsTouchEvents and supportsPointerEvents params - -**🐛 功能修复** - -- **@visactor/vrender-components**: fix the flush of axis when axis label has rotate angle -- **@visactor/vrender-components**: arc label line not shown -- **@visactor/vrender-components**: error happens in line-label when line has no points -- **@visactor/vrender-core**: fix issue with html attribute -- **@visactor/vrender-core**: fix issue with env-check -- **@visactor/vrender-core**: fix issue with text background opacity - - - -[更多详情请查看 v0.17.15](https://github.com/VisActor/VRender/releases/tag/v0.17.15) - -# v0.17.14 - -2024-01-12 - -**🐛 功能修复** -- **@visactor/vrender-core**: fix `splitRect` when rect has `x1` or `y1` -- **@visactor/vrender**: fix `splitRect` when rect has `x1` or `y1` - - -[更多详情请查看 v0.17.14](https://github.com/VisActor/VRender/releases/tag/v0.17.14) - -# v0.17.13 - -2024-01-10 - -**🆕 新增功能** -- **@visactor/vrender-core**: background support opacity -**🐛 功能修复** -- **@visactor/vrender-components**: filter out invisible indicator spec -- **@visactor/vrender-components**: `measureTextSize` needs to take into account the fonts configured on the stage theme -- **@visactor/vrender-core**: fix issue with incremental draw -- **@visactor/vrender-core**: supply the `getTheme()` api for `IStage` - - - -[更多详情请查看 v0.17.13](https://github.com/VisActor/VRender/releases/tag/v0.17.13) - -# v0.17.12 - -2024-01-10 - -**🆕 新增功能** -- **@visactor/vrender-components**: support fit strategy for indicator -- **marker**: mark point support confine. fix @Visactor/VChart[#1573](https://github.com/VisActor/VRender/issues/1573) -**🐛 功能修复** -- **marker**: fix problem of no render when set visible attr and add valid judgment logic. fix@Visactor/Vchart[#1901](https://github.com/VisActor/VRender/issues/1901) -- **datazoom**: adaptive handler text layout. fix@Visactor/VChart[#1809](https://github.com/VisActor/VRender/issues/1809) -- **datazoom**: set pickable false when zoomLock. fix @Visactor/VChart[#1565](https://github.com/VisActor/VRender/issues/1565) -- **datazoom**: handler not follow mouse after resize. fix@Visactor/Vchart[#1490](https://github.com/VisActor/VRender/issues/1490) -- **@visactor/vrender-components**: arc outside label invisible with visible label line - - - -[更多详情请查看 v0.17.12](https://github.com/VisActor/VRender/releases/tag/v0.17.12) - -# v0.17.11 - -2024-01-05 - -**🆕 新增功能** -- **@visactor/vrender-core**: add backgroundFit attribute -**🐛 功能修复** -- **@visactor/vrender-core**: fix issue with position in html attribute -- fix: label invisible when baseMark visible is false - -**Full Changelog**: https://github.com/VisActor/VRender/compare/v0.17.10...v0.17.11 - -[更多详情请查看 v0.17.11](https://github.com/VisActor/VRender/releases/tag/v0.17.11) - -# v0.17.10 - -2024-01-03 - -**🆕 新增功能** -- **@visactor/vrender-components**: support `lastVisible` of LineAxis label -- **@visactor/vrender-kits**: support fillPickable and strokePickable for area, closed [#792](https://github.com/VisActor/VRender/issues/792) -- **@visactor/vrender-core**: support fillPickable and strokePickable for area, closed [#792](https://github.com/VisActor/VRender/issues/792) -- **@visactor/vrender-core**: support `lastVisible` of LineAxis label -- **@visactor/vrender**: support `lastVisible` of LineAxis label -**🐛 功能修复** -- **@visactor/vrender-components**: fix the auto limit width of label when the label has vertical `direction` in orient top or bottom -- **@visactor/vrender-components**: fix issue with legend symbol size -- **@visactor/vrender-components**: fixed height calculation issue after multi-layer axis text rotation -- **@visactor/vrender-core**: fix issue with area-line highperformance draw -- **@visactor/vrender-core**: fix the auto limit width of label when the label has vertical `direction` in orient top or bottom -- **@visactor/vrender-core**: disable layer picker in interactive layer -- **@visactor/vrender**: fix the auto limit width of label when the label has vertical `direction` in orient top or bottom -**🔖 其他** -- **@visactor/vrender-components**: 'feat: support label line in label component' - - - -[更多详情请查看 v0.17.10](https://github.com/VisActor/VRender/releases/tag/v0.17.10) - -# v0.17.9 - -2024-01-03 - -**🐛 功能修复** -- **@visactor/vrender-components**: fix label position when offset is 0 -- **@visactor/vrender-core**: fix issue with conical cache not work as expect - - - -[更多详情请查看 v0.17.9](https://github.com/VisActor/VRender/releases/tag/v0.17.9) - -# v0.17.8 - -2023-12-29 - -**🆕 新增功能** -- **@visactor/vrender-components**: optimize outer label layout in tangential direction -- **@visactor/vrender-core**: support drawGraphicToCanvas -- **@visactor/vrender**: support drawGraphicToCanvas -**🐛 功能修复** -- **@visactor/vrender-components**: when axis label space is 0, and axis tick' inside is true, the axis label's position is not correct -- **@visactor/vrender-components**: fix morphing of rect -- **@visactor/vrender-kits**: fix issue with mapToCanvasPoint in miniapp, closed [#828](https://github.com/VisActor/VRender/issues/828) -- **@visactor/vrender-core**: fix issue with rect.toCustomPath -- **@visactor/vrender-core**: fix issue with area segment with single point, closed [#801](https://github.com/VisActor/VRender/issues/801) -- **@visactor/vrender-core**: fix issue with new Function in miniapp -- **@visactor/vrender-core**: fix morphing of rect -- **@visactor/vrender-core**: fix issue with side-effect in some env -- **@visactor/vrender-core**: fix issue with check tt env -- **@visactor/vrender-core**: fix issue with cliped attribute in vertical text, closed [#827](https://github.com/VisActor/VRender/issues/827) -- **@visactor/vrender**: fix issue with area segment with single point, closed [#801](https://github.com/VisActor/VRender/issues/801) -- **@visactor/vrender**: fix morphing of rect -- **@visactor/vrender**: fix issue with side-effect in some env - - - -[更多详情请查看 v0.17.8](https://github.com/VisActor/VRender/releases/tag/v0.17.8) - -# v0.17.7 - -2023-12-21 - -**🐛 功能修复** -- **@visactor/vrender-kits**: fix issue with create layer in miniapp env -- **@visactor/vrender-core**: fix issue with create layer in miniapp env - - - -[更多详情请查看 v0.17.7](https://github.com/VisActor/VRender/releases/tag/v0.17.7) - -# v0.17.6 - -2023-12-20 - -**What's Changed** -* Main by @neuqzxy in https://github.com/VisActor/VRender/pull/813 -* fix: fix issue with rect stroke contribution by @neuqzxy in https://github.com/VisActor/VRender/pull/814 -* [Auto release] release 0.17.6 by @github-actions in https://github.com/VisActor/VRender/pull/815 - - -**Full Changelog**: https://github.com/VisActor/VRender/compare/v0.17.5...v0.17.6 - -[更多详情请查看 v0.17.6](https://github.com/VisActor/VRender/releases/tag/v0.17.6) - -# v0.17.5 - -2023-12-19 - -**🆕 新增功能** -- **scrollbar**: dispatch scrollDown event -- **@visactor/vrender-components**: labelLine support animate -- **@visactor/vrender-components**: label don't create enter animate animationEnter while duration less than 0 -- **@visactor/vrender**: add disableAutoClipedPoptip attribute in text graphic -**🐛 功能修复** -- **@visactor/vrender-components**: fix issue with arc animate with delayafter -- **@visactor/vrender-components**: fix issue with poptip circular dependencies -- **@visactor/vrender-core**: fix issue with plugin unregister -- **@visactor/vrender-core**: fix issue with text while whitespace is normal -- **@visactor/vrender**: fix cursor update error in multi-stage - - - -[更多详情请查看 v0.17.5](https://github.com/VisActor/VRender/releases/tag/v0.17.5) - -# v0.17.4 - -2023-12-15 - -**🐛 功能修复** -- **datazoom**: symbol size problem -- **@visactor/vrender-core**: fix issue with arc imprecise bounds, closed [#728](https://github.com/VisActor/VRender/issues/728) - - - -[更多详情请查看 v0.17.4](https://github.com/VisActor/VRender/releases/tag/v0.17.4) - -# v0.17.3 - -2023-12-14 - -**🐛 功能修复** -- **datazoom**: handler zindex to interaction error - - - -[更多详情请查看 v0.17.3](https://github.com/VisActor/VRender/releases/tag/v0.17.3) - -# v0.17.2 - -2023-12-14 - -**🆕 新增功能** -- **dataZoom**: add mask to modify hot zone. feat @visactor/vchart[#1415](https://github.com/VisActor/VRender/issues/1415)' -- **dataZoom**: add mask to modify hot zone. feat @visactor/vchart[#1415](https://github.com/VisActor/VRender/issues/1415)' -- **dataZoom**: add mask to modify hot zone. feat @visactor/vchart[#1415](https://github.com/VisActor/VRender/issues/1415)' -- **dataZoom**: add mask to modify hot zone. feat @visactor/vchart[#1415](https://github.com/VisActor/VRender/issues/1415)' -- **dataZoom**: add mask to modify hot zone. feat @visactor/vchart[#1415](https://github.com/VisActor/VRender/issues/1415)' -- **@visactor/vrender-core**: rect3d support x1y1, fix -radius issue with rect -- **dataZoom**: add mask to modify hot zone. feat @visactor/vchart[#1415](https://github.com/VisActor/VRender/issues/1415)' -**🐛 功能修复** -- **@visactor/vrender-components**: scrollbar slider width/height should not be negative -- **@visactor/vrender-components**: datazoom event block window event. fix @visactor/vchart[#1686](https://github.com/VisActor/VRender/issues/1686) -- **@visactor/vrender-components**: fix the issue of brushEnd trigger multiple times, related https://github.com/VisActor/VChart/issues/1694 -- **@visactor/vrender-core**: fix shadow pick issue -**⚡ 性能优化** -- **@visactor/vrender-components**: optimize the `_handleStyle()` in legend - - - -[更多详情请查看 v0.17.2](https://github.com/VisActor/VRender/releases/tag/v0.17.2) - -# v0.17.1 - -2023-12-06 - -**🆕 新增功能** -- **@visactor/vrender-kits**: support pickStrokeBuffer, closed [#758](https://github.com/VisActor/VRender/issues/758) -- **@visactor/vrender-core**: support pickStrokeBuffer, closed [#758](https://github.com/VisActor/VRender/issues/758) -**🐛 功能修复** -- **@visactor/vrender-kits**: fix issue with rebind pick-contribution -- **@visactor/vrender-core**: fix issue in area chart with special points -- **@visactor/vrender-core**: fix issue with rebind pick-contribution -- **@visactor/vrender-core**: fix error with wrap text and normal whiteSpace text - - - -[更多详情请查看 v0.17.1](https://github.com/VisActor/VRender/releases/tag/v0.17.1) - -# v0.17.0 - -2023-11-30 - -**🆕 新增功能** -- **@visactor/vrender-components**: optmize bounds performance -- **@visactor/vrender-kits**: rect support x1 and y1 -- **@visactor/vrender-kits**: optmize bounds performance -- **@visactor/vrender-core**: support disableCheckGraphicWidthOutRange to skip check if graphic out of range -- **@visactor/vrender-core**: rect support x1 and y1 -- **@visactor/vrender-core**: don't rewrite global reflect -- **@visactor/vrender-core**: text support background, closed [#711](https://github.com/VisActor/VRender/issues/711) -- **@visactor/vrender-core**: optmize bounds performance -- **@visactor/vrender**: don't rewrite global reflect -- **@visactor/vrender**: skip update bounds while render small node-tree, closed [#660](https://github.com/VisActor/VRender/issues/660) -- **@visactor/vrender**: optmize bounds performance -**🔨 功能重构** -- **@visactor/vrender-kits**: refact inversify completely, closed [#657](https://github.com/VisActor/VRender/issues/657) -- **@visactor/vrender-core**: refact inversify completely, closed [#657](https://github.com/VisActor/VRender/issues/657) -- **@visactor/vrender**: refact inversify completely, closed [#657](https://github.com/VisActor/VRender/issues/657) -**⚡ 性能优化** -- **@visactor/vrender-components**: add option `skipDefault` to vrender-components -- **@visactor/vrender-core**: area support drawLinearAreaHighPerformance, closed [#672](https://github.com/VisActor/VRender/issues/672) - - - -[更多详情请查看 v0.17.0](https://github.com/VisActor/VRender/releases/tag/v0.17.0) - -# v0.16.18 - -2023-11-30 - -**🆕 新增功能** -- **@visactor/vrender-components**: discrete legend's pager support position property -- **@visactor/vrender-core**: support suffixPosition, closed [#625](https://github.com/VisActor/VRender/issues/625) -- **@visactor/vrender**: support suffixPosition, closed [#625](https://github.com/VisActor/VRender/issues/625) -**🐛 功能修复** -- **@visactor/vrender-kits**: doubletap should not be triggered when the target is different twice before and after -- **@visactor/vrender-core**: fix issue with attribute interpolate, closed [#741](https://github.com/VisActor/VRender/issues/741) -- **@visactor/vrender-core**: fix issue about calcuate bounds with shadow, closed [#474](https://github.com/VisActor/VRender/issues/474) -- **@visactor/vrender-core**: fix issue with white line in some dpr device, closed [#666](https://github.com/VisActor/VRender/issues/666) -**🔨 功能重构** -- **@visactor/vrender-components**: move getSizeHandlerPath out of sizlegend -- **@visactor/vrender-core**: event-related coordinate points do not require complex Point classes - - - -[更多详情请查看 v0.16.18](https://github.com/VisActor/VRender/releases/tag/v0.16.18) - -# v0.16.17 - -2023-11-23 - -**🆕 新增功能** -- **@visactor/vrender-components**: support rich text for label, axis, marker,tooltip, indicator and title -- **@visactor/vrender-components**: add mode type of smartInvert -- **@visactor/vrender-components**: place more label for overlapPadding case -- **@visactor/vrender-kits**: support 'tap' gesture for Gesture plugin -- **@visactor/vrender-core**: add `event` config for Stage params, which can configure `clickInterval` and some other options in eventSystem -- **@visactor/vrender-core**: support fill and stroke while svg don't support, closed [#710](https://github.com/VisActor/VRender/issues/710) -**🐛 功能修复** -- **@visactor/vrender-kits**: \`pickMode: 'imprecise'\` not work in polygon -- **@visactor/vrender-core**: richtext may throw error when textConfig is null -- **@visactor/vrender-core**: fix issue with image repeat, closed [#712](https://github.com/VisActor/VRender/issues/712) -- **@visactor/vrender-core**: fix issue with restore and save count not equal -**⚡ 性能优化** -- **@visactor/vrender-core**: not setAttribute while background is not url, closed [#696](https://github.com/VisActor/VRender/issues/696) - - - -[更多详情请查看 v0.16.17](https://github.com/VisActor/VRender/releases/tag/v0.16.17) - -# v0.16.16 - -2023-11-17 - -**🐛 功能修复** -- **@visactor/vrender-components**: fix the issue of legend item.shape can not set visible, related https://github.com/VisActor/VChart/issues/1508 -- **@visactor/vrender-core**: assign symbol rect function to old - - - -[更多详情请查看 v0.16.16](https://github.com/VisActor/VRender/releases/tag/v0.16.16) - -# v0.16.15 - -2023-11-16 - -**🐛 功能修复** -- **@visactor/vrender-compoments**: legendItemHover and legendItemUnHover should trigger once - - - -[更多详情请查看 v0.16.15](https://github.com/VisActor/VRender/releases/tag/v0.16.15) - -# v0.16.14 - -2023-11-15 - -**🆕 新增功能** -- **@visactor/vrender-components**: datazoom update callback supports new trigger tag param -- **@visactor/vrender-components**: support line/area label -- **@visactor/vrender-components**: lineHeight support string, which means percent -- **@visactor/vrender-core**: add round line symbol, closed [#1458](https://github.com/VisActor/VRender/issues/1458) -- **@visactor/vrender-core**: lineHeight support string, which means percent -**🐛 功能修复** -- **@visactor/vrender-core**: fix issue with render while in scale transform - - - -[更多详情请查看 v0.16.14](https://github.com/VisActor/VRender/releases/tag/v0.16.14) - -# v0.16.13 - -2023-11-15 - -**🆕 新增功能** -- **@visactor/vrender-core**: add preventRender function -- **@visactor/vrender-core**: merge wrap text function to text -**🐛 功能修复** -- **@visactor/vrender-kits**: temp fix issue with lynx measuretext - - - -[更多详情请查看 v0.16.13](https://github.com/VisActor/VRender/releases/tag/v0.16.13) - -# v0.16.12 - -2023-11-07 - -**🆕 新增功能** -- **@visactor/vrender-core**: optimize text increase animation -**🐛 功能修复** -- **@visactor/vrender-components**: padding of title component -- **@visactor/vrender-components**: padding offset of AABBbounds -- **@visactor/vrender-kits**: fix node-canvas max count issue -- **@visactor/vrender-core**: fix node-canvas max count issue - - - -[更多详情请查看 v0.16.12](https://github.com/VisActor/VRender/releases/tag/v0.16.12) - -# v0.16.11 - -2023-11-07 - -**🐛 功能修复** -- **@visactor/vrender-components**: optimize the auto-overlap of axis label, which use rotateBounds when text rotated, relate https://github.com/VisActor/VChart/issues/133 -- **@visactor/vrender-components**: flush should not sue width height -- **@visactor/vrender-components**: fix the lastvisible logic of axis's auto-hide -- **@visactor/vrender-kits**: fix issue with xul and html attribute, closed [#634](https://github.com/VisActor/VRender/issues/634) -- **@visactor/vrender-core**: fix issue with xul and html attribute, closed [#634](https://github.com/VisActor/VRender/issues/634) - - - -[更多详情请查看 v0.16.11](https://github.com/VisActor/VRender/releases/tag/v0.16.11) - -# v0.16.10 - -2023-11-02 - -**What's Changed** -* Sync main by @neuqzxy in https://github.com/VisActor/VRender/pull/640 -* fix: fix issue with xul and html attribute, closed [#634](https://github.com/VisActor/VRender/issues/634) by @neuqzxy in https://github.com/VisActor/VRender/pull/635 -* Echance/axis auto rotate by @kkxxkk2019 in https://github.com/VisActor/VRender/pull/633 -* [Auto release] release 0.16.9 by @github-actions in https://github.com/VisActor/VRender/pull/641 - - -**Full Changelog**: https://github.com/VisActor/VRender/compare/v0.16.9...v0.16.10 - -[更多详情请查看 v0.16.10](https://github.com/VisActor/VRender/releases/tag/v0.16.10) - -# v0.16.9 - -2023-10-27 - -**🆕 新增功能** -- **@visactor/vrender-components**: add checkbox indeterminate state -- **label**: rect label support position `top-right`|`top-left`|`bottom-righ`|`bottom-left` -- **@visactor/vrender-core**: stage background support image -**🐛 功能修复** -- **@visactor/vrender-components**: all the group container of marker do not trigger event -- **datazoom**: text bounds when visible is false. fix VisActor/VChart[#1281](https://github.com/VisActor/VRender/issues/1281) - - - -[更多详情请查看 v0.16.9](https://github.com/VisActor/VRender/releases/tag/v0.16.9) - -# v0.16.8 - -2023-10-23 - -**🐛 功能修复** -- **@visactor/vrender-components**: fix the issue of error position of focus when legend item just has label - - - -[更多详情请查看 v0.16.8](https://github.com/VisActor/VRender/releases/tag/v0.16.8) - -# v0.16.7 - -2023-10-23 - -**🐛 功能修复** -- **label**: fix the issue that `clampForce` does not work when`overlapPadding` is configured -- **@visactor/vrender-core**: fix issue with creating multi chart in miniapp - - - -[更多详情请查看 v0.16.7](https://github.com/VisActor/VRender/releases/tag/v0.16.7) - -# v0.16.6 - -2023-10-23 - -**🆕 新增功能** -- **@visactor/vrender-components**: optimize the layout method of circle axis label -**🐛 功能修复** -- **@visactor/vrender-components**: fix the layout issue of legend item because of the error logic of `focusStartX` - - - -[更多详情请查看 v0.16.6](https://github.com/VisActor/VRender/releases/tag/v0.16.6) - diff --git a/packages/calculator/package.json b/packages/calculator/package.json index c9482560..6ec7aa1b 100644 --- a/packages/calculator/package.json +++ b/packages/calculator/package.json @@ -1,6 +1,6 @@ { "name": "@visactor/calculator", - "version": "1.2.4", + "version": "1.2.5", "description": "SQL-like query executor with DSL", "main": "lib", "module": "es", diff --git a/packages/chart-advisor/.eslintrc.cjs b/packages/chart-advisor/.eslintrc.cjs new file mode 100644 index 00000000..f3be945a --- /dev/null +++ b/packages/chart-advisor/.eslintrc.cjs @@ -0,0 +1,18 @@ +require('@rushstack/eslint-patch/modern-module-resolution'); + +module.exports = { + extends: ['@internal/eslint-config/profile/react'], + parserOptions: { tsconfigRootDir: __dirname, project: './tsconfig.eslint.json' }, + // ignorePatterns: [], + env: { + browser: true, + es2021: true, + node: true, + jest: true + }, + rules: { + '@typescript-eslint/no-unused-vars': 'warn', + 'react/display-name': 'off', + 'no-console': 'warn' + } +}; diff --git a/packages/chart-advisor/LICENSE b/packages/chart-advisor/LICENSE new file mode 100644 index 00000000..308f98cc --- /dev/null +++ b/packages/chart-advisor/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2023 Bytedance Ltd. and/or its affiliates. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/packages/chart-advisor/README.md b/packages/chart-advisor/README.md new file mode 100644 index 00000000..9c936565 --- /dev/null +++ b/packages/chart-advisor/README.md @@ -0,0 +1,154 @@ + +# Chart Advisor + +## How to use + +``` +chartAdvisor(originDataset: DataItem[], dimensionList:Field[], measureList:Field[], aliasMap: AliasMap) +``` + +Enter the dataset and the information about the dimension and indicator fields, it returns the current recommended chart type and field allocation. + +## Example + +``` +const dataset=[{ "210816110721021": "Furniture", "210816110721021": "14138" }, { "210816110721021": "Office supplies", "210816110721021": "34611" }, { "210816110721021": "Technology", "210816110721021": "12637" }] + +const dimensionList=[{ "uniqueId": 210816110721021, "type": "textual" }] + +const measureList=[{ "uniqueId": 210816110721022, "type": "numerical" }] + +const aliasMap={ "210816110721021": "Category", "210816110721022": "Number" } + +const result = advisor.calAdvisedChart(dataset, dimensionList, measureList, aliasMap) + +console.log(result) +``` + +The output of the above code is: + +``` +{ +chartType: 'column', +cell: [ +{ +x: ['210816110721021'], +y: ['210816110721022'], +row: [], +column: [], +color: [], +size: [], +angle: [] +} +], +colorItems: [], +dataset: [[[[{"210816110721021":"Furniture","210816110721022":"14138"},{"210816110721021":"Office supplies","210816110721022":"34611"},{"210816110721021":"Technology","210816110721022":"12637"}]]]], +aliasMap: { '210816110721021': 'Category', '210816110721022': 'Number' } +} +``` + +The above data can be used directly to generate vizData. + +The final generated chart is: + +![](exampleChart.png) + +## Parameter explanation: + +originDataset: The original data set, which is a list, each element is a DataItem, representing a row of data, the format is {uniqueID1: value1, uniqueID2: value2, ...}, where uniqueID is the id of the field, value is the value of this row data in that field. + +``` +type DataItem = { +[key: number]: string +} +``` + +dimensionList: Collection of dimension information, which is a list, each element is a Field. +measureList: The collection of indicator information, which is list, each element is a Field. + +``` +type Field={ +uniqueId: number; //id of the field +type: string; //type of the field (number, string, date) +} +``` + +alisMap: Field alias table, consistent with the aliasMap in vizData. + +``` +type AliasMap = { +[key: number]: string; +}; +``` + +## Return value: + +Returns AdviseResult. The currently supported chart types are listed in ChartType. + +``` +type AdviseResult={ +chartType: ChartType; //chartType in vizData +cell:Cell; //cell in vizData, +colorItems:[] //colorItems in vizData +dataset: DataItem[]; //The processed dataset used to generate vizData +aliasMap: AliasMap; //The processed field alias table used to generate vizData +} +``` + +Explanation of each type: + +``` +enum ChartType { +/** Table */ +TABLE = 'table', + +/** Bar chart */ +COLUMN = 'column', +/** Percentage Bar chart */ +COLUMN_PERCENT = 'column_percent', +/** Parallel Bar chart */ +COLUMN_PARALLEL = 'column_parallel', + +/** Line chart */ +LINE = 'line', + +/** Pie chart */ +PIE = 'pie', + +/** Scatter plot */ +SCATTER = 'scatter', + +/** Combined Bar chart */ +COMBINECOLUMN = 'combineColumn', //combined chart composed of multiple bar charts + +/** Combined z line chart */ +COMBINELINE = 'combineLine', //combined chart composed of multiple line charts + +/** Measure card */ +MEASURE_CARD = 'measure_card', + +/** Word cloud */ +WORD_CLOUD = 'word_cloud', +} + +interface Cell { +column?: UniqueId[]; +row?: UniqueId[]; +x?: UniqueId[]; +y?: UniqueId[]; +group?: UniqueId[]; +color?: UniqueId[]; +size?: UniqueId[]; +shape?: UniqueId[]; +angle?: UniqueId[]; +radius?: UniqueId[]; +text?: UniqueId[]; +value?: UniqueId[]; +tooltip?: UniqueId[]; + +// Dimension expansion information (Cartesian product) +cartesianInfo?: CartesianInfo; +// Indicator expansion information (Indicator flattening) +foldInfo?: FoldInfo; +} +``` diff --git a/packages/chart-advisor/exampleChart.png b/packages/chart-advisor/exampleChart.png new file mode 100644 index 00000000..702bb060 Binary files /dev/null and b/packages/chart-advisor/exampleChart.png differ diff --git a/packages/chart-advisor/package.json b/packages/chart-advisor/package.json new file mode 100644 index 00000000..930b30a7 --- /dev/null +++ b/packages/chart-advisor/package.json @@ -0,0 +1,52 @@ +{ + "name": "@visactor/chart-advisor", + "version": "1.2.5", + "description": "图表推荐模块", + "main": "lib", + "module": "es", + "types": "es", + "sideEffects": false, + "exports": { + ".": { + "import": "./es/index.js", + "require": "./lib/index.js" + } + }, + "files": [ + "README.md", + "lib", + "es", + "build" + ], + "scripts": { + "clean": "rimraf es lib dist build *.tsbuildinfo", + "lint": "eslint . --ext ts,tsx --report-unused-disable-directives --max-warnings 0", + "test": "jest --passWithNoTests", + "test:coverage": "jest --coverage", + "build:esm": "tsc -p tsconfig.esm.json", + "build:cjs": "tsc -p tsconfig.cjs.json", + "build": "npm-run-all clean --parallel build:esm build:cjs" + }, + "author": "", + "license": "MIT", + "dependencies": { + "lodash": "4.17.21" + }, + "devDependencies": { + "@types/jest": "^26.0.0", + "@types/lodash": "4.14.182", + "@types/node": "*", + "jest": "^26.0.0", + "ts-jest": "^26.0.0", + "typescript": "4.9.5", + "undici-types": "^5.27.2", + "@typescript-eslint/eslint-plugin": "5.30.0", + "@typescript-eslint/parser": "5.30.0", + "npm-run-all": "^4.1.5", + "rimraf": "^3.0.2", + "@internal/ts-config": "workspace:*", + "@internal/eslint-config": "workspace:*", + "@internal/bundler": "workspace:*", + "@rushstack/eslint-patch": "~1.1.4" + } +} diff --git a/packages/chart-advisor/src/constant.ts b/packages/chart-advisor/src/constant.ts new file mode 100644 index 00000000..3a46fa00 --- /dev/null +++ b/packages/chart-advisor/src/constant.ts @@ -0,0 +1,26 @@ +import { UniqueId } from './type'; +//后端传来的平坦化数据集中，和前端的设置不一致 +export const FOLD_NAME: UniqueId = 10001; +export const FOLD_VALUE: UniqueId = 10002; +export const FOLD_VALUE_MAIN: UniqueId = 10011; +export const FOLD_VALUE_SUB: UniqueId = 10012; + +export const COLOR_FIELD: UniqueId = 20001; +export const GROUP_FIELD: UniqueId = 30001; + +// export const MAX_PIVOT_ROW: number = 0 //允许的最大透视行数 +// export const MAX_PIVOT_COLUMN: number = 0 //允许的最大透视列数 + +// X轴刻度数 +export const X_MAX_COUNT = 5000; +// 数据点数 +export const MAX_POINT_COUNT = 100000; +// 图例个数 +export const LEGEND_MAX_COUNT = 1000; +// 图例个数超过时数据点的限制 +export const LEGEND_MAX_POINT_COUNT = 200; + +export const MIN_BAR_NUMBER = 2; +export const MAX_BAR_NUMBER = 30; + +export const APPLY_PIVOT = false; //透视开关 diff --git a/packages/chart-advisor/src/dataUtil.ts b/packages/chart-advisor/src/dataUtil.ts new file mode 100644 index 00000000..62f47097 --- /dev/null +++ b/packages/chart-advisor/src/dataUtil.ts @@ -0,0 +1,70 @@ +import { cloneDeep, isNil, uniq, isNaN } from 'lodash'; +import { DataTypeName } from './type'; +//将vizData中的dataset数组展开 后端版本可直接获取到dataSource，不用执行此方法 +const restoreDataItem = item => { + if (!Array.isArray(item)) { + return item; + } + + return item.reduce((prev, cur) => prev.concat(restoreDataItem(cur)), []); +}; +export const restoreDatasets = dataset => restoreDataItem(dataset); + +// + +//计算平均数 +export const calMean = dataset => { + const { data } = dataset; + const dataNotNull = data.filter(each => !isNil(each) && !isNaN(each)); + + const sum = dataNotNull.reduce((prev, cur) => prev + cur, 0); + const { length } = data; + return sum / length; +}; +//计算数据集的标准差 +export const calStandardDeviation = dataset => { + const { data } = dataset; + if (data.length === 1) return 0; + + const dataNotNull = data.filter(each => !isNil(each) && !isNaN(each)); + + const mean = dataset.mean ? dataset.mean : calMean(dataset); + const sumpow = dataNotNull.reduce((prev, cur) => prev + (cur - mean) ** 2, 0); + const { length } = data; + return Math.sqrt(sumpow / (length - 1)); +}; + +//计算变异系数 +export const calCoefficient = dataset => { + const mean = dataset.mean ? dataset.mean : calMean(dataset); + //平均数=0时，变异系数无意义 + const standardDev = dataset.standardDev ? dataset.standardDev : calStandardDeviation(dataset); + if (mean !== 0) { + return standardDev / mean; + } else { + return undefined; + } +}; + +// 升序排序 +const asc = arr => arr.sort((a, b) => a - b); + +export const calQuantile = (dataset, q) => { + const { data = [] } = dataset; + + // 取绝对值且过滤掉 0 的 + const sorted = asc(cloneDeep(data.map(Math.abs))).filter(each => each && each > 0); + const pos = (sorted.length - 1) * q; + const base = Math.floor(pos); + const rest = pos - base; + if (sorted[base + 1] !== undefined) { + return sorted[base] + rest * (sorted[base + 1] - sorted[base]); + } else { + return sorted[base]; + } +}; + +//数组去重 +export const unique = arr => uniq(arr); + +export const isTemporal = (type: DataTypeName) => type === 'date'; diff --git a/packages/chart-advisor/src/fieldAssign.ts b/packages/chart-advisor/src/fieldAssign.ts new file mode 100644 index 00000000..8d08f8e9 --- /dev/null +++ b/packages/chart-advisor/src/fieldAssign.ts @@ -0,0 +1,596 @@ +import { clone, cloneDeep } from 'lodash'; +import { AutoChartCell } from './type'; +import * as dataUtils from './dataUtil'; + +import { AliasMap, UniqueId, DataItem, Dataset } from './type'; +import { + productLength, + legendProduct, + getDomainFromDataset, + retainDatasetField, + getCartesianInfo, + getFoldInfo, + fold, + removeDatasetField +} from './fieldUtils'; + +import { + FOLD_NAME, + FOLD_VALUE, + COLOR_FIELD, + GROUP_FIELD, + FOLD_VALUE_MAIN, + FOLD_VALUE_SUB, + X_MAX_COUNT, + MAX_POINT_COUNT, + LEGEND_MAX_COUNT, + LEGEND_MAX_POINT_COUNT +} from './constant'; + +/*自动分配策略： +当维度数大于1小于3时，前面的字段先用来形成透视，余下字段中的第一个用来做x轴。 +当透视达到1x1后，第三个字段用来做x轴，其余字段按规则分配到视觉通道中， +最后多余的字段用来形成笛卡尔积 +这样做的目的是： +1. 从外往里进行透视更加自然 +2. 改变颜色的影响远大于改变x轴的影响 +这与tableau的策略是一致的 +目前先限制1x1的透视，后续性能提升后透视的层数可以增加 +*/ + +/* +为支持透视的图表分配字段(条形图、柱状图) +*/ +export const assignPivotCharts = ( + originDataset, + dimList: UniqueId[], + measureList: UniqueId[], + aliasMapOld: AliasMap, + MAX_PIVOT_ROW: number, + MAX_PIVOT_COLUMN: number +) => { + const aliasMap = cloneDeep(aliasMapOld); + let dataset = cloneDeep(originDataset); + const cell: AutoChartCell = { + x: [], + y: [], + row: [], + column: [], + color: [], + size: [], + angle: [] + }; + + if (dimList.length === 0 || measureList.length === 0) { + return { cell, dataset }; + } + + let nowDimIndex = 0; + + //首先分配透视字段 + while ( + (cell.row.length < MAX_PIVOT_ROW || cell.column.length < MAX_PIVOT_COLUMN) && + nowDimIndex + 1 < dimList.length //还有剩余字段，可以分配透视维度 + ) { + if (cell.row.length <= cell.column.length) { + cell.row.push(dimList[nowDimIndex]); + } else { + cell.column.push(dimList[nowDimIndex]); + } + nowDimIndex++; + } + + //剩余第一个维度分配到x轴 + cell.x.push(dimList[nowDimIndex]); + nowDimIndex++; + + const colorFields = []; + + //其余字段做笛卡尔积分配到颜色 + colorFields.push(...dimList.slice(nowDimIndex)); + + const measuresListLength: number = measureList.length; + // 提取 y 字段（度量值） + let colorFieldsIncludeMeasure = false; + if (measuresListLength > 1) { + // 多度量 + aliasMap[FOLD_NAME] = `指标名称`; + aliasMap[FOLD_VALUE] = `指标值`; + colorFields.push(FOLD_NAME); + colorFieldsIncludeMeasure = true; + cell.y.push(FOLD_VALUE); + } else if (measuresListLength === 1) { + // 单度量 + cell.y.push(measureList[0]); + } + + // 提取 color分组字段 （维度其他项 以及 度量名称） + const colorFieldsValues: string[][] = colorFields.map((uniqueId: UniqueId) => + getDomainFromDataset(dataset, uniqueId) + ); + + // 计算图例项笛卡尔积之前，限制条目数量 + const dimItemsLen: number = getDomainFromDataset(dataset, cell.x[0]).length; + const colorItemsLen: number = productLength(colorFieldsValues); + if ( + dimItemsLen > X_MAX_COUNT || + dataset.length > MAX_POINT_COUNT || + (colorItemsLen > LEGEND_MAX_COUNT && dataset.length > LEGEND_MAX_POINT_COUNT) + ) { + return { + error: true, + errMsg: `数据量或图例项过多，请使用表格展示。` + }; + } + + // 计算图例项 + const colorItemsList: string[][] = legendProduct(colorFieldsValues, colorFieldsIncludeMeasure); + let colorItems: string[] = colorItemsList.map(d => d.join('-')); + + // 使用笛卡尔积后的图例项替换原数据中对应的字段 + if (colorFields.length > 0) { + dataset = dataset.map((data: DataItem) => { + const colorItem = colorFields.map(field => data[field]).join('-'); + return { + ...data, + [COLOR_FIELD]: colorItem + }; + }); + // 将笛卡尔积后无效的 colorItem 移除 + const _colorItems: string[] = getDomainFromDataset(dataset, COLOR_FIELD); + colorItems = colorItems.filter((d: string) => _colorItems.includes(d)); + + aliasMap[COLOR_FIELD] = `图例项`; + cell.color.push(COLOR_FIELD); + cell.cartesianInfo = getCartesianInfo(colorFields, COLOR_FIELD); + + // 多度量时的平坦化字段 + if (measuresListLength > 1) { + cell.foldInfo = getFoldInfo(measureList, FOLD_NAME, FOLD_VALUE, aliasMap); + } + } + + return { cell, dataset, colorItems, aliasMap }; +}; + +export const processCombination = ( + originDataset, + dimList: UniqueId[], + measureList: UniqueId[], + aliasMapOld: AliasMap, + MAX_PIVOT_ROW: number, + MAX_PIVOT_COLUMN: number +) => { + const aliasMap = cloneDeep(aliasMapOld); + //为组合图单独配置cell + const dataset = cloneDeep(originDataset); + const metaDatas = []; + + measureList.forEach(measure => { + const _allPillsIdList: UniqueId[] = [].concat(dimList, [measure]); + const _dataset: Dataset = retainDatasetField(dataset, _allPillsIdList); + const metaData = assignPivotCharts(_dataset, dimList, [measure], aliasMap, MAX_PIVOT_ROW, MAX_PIVOT_COLUMN); + metaDatas.push(metaData); + }); + + return metaDatas; +}; + +export const assignScatterPlot = ( + originDataset, + dimList: UniqueId[], + measureList: UniqueId[], + aliasMapOld: AliasMap +) => { + const aliasMap = cloneDeep(aliasMapOld); + let dataset = cloneDeep(originDataset); + const scatterCell: AutoChartCell = { + x: [], + y: [], + row: [], + column: [], + color: [], + size: [], + angle: [], + group: [GROUP_FIELD] + }; + + let remainMeasure = measureList.length; //剩余未分配的度量数量 + + if (dimList.length === 0 || measureList.length < 2) { + return { scatterCell, dataset }; + } + + //首先分配两个轴 + scatterCell.x.push(measureList[0]); + scatterCell.y.push(measureList[1]); + + remainMeasure -= 2; + + //将第三个度量分配到尺寸 + if (measureList.length > 2) { + scatterCell.size.push(measureList[2]); + } + + remainMeasure -= 1; + + const groupDimensions: UniqueId[] = []; + + //将第一个维度分配到group，优先将第二个维度分配到颜色 + if (dimList.length > 1) { + scatterCell.color.push(dimList[1]); + groupDimensions.concat(dimList.slice(2)); + } else { + groupDimensions.concat(dimList.slice(1)); + //将度量分配到颜色 + if (measureList.length > 3) { + scatterCell.color.push(measureList[3]); + remainMeasure -= 1; + } + } + + if (remainMeasure > 0) { + //不支持此情况 + const voidCell: AutoChartCell = { + x: [], + y: [], + row: [], + column: [], + color: [], + size: [], + angle: [] + }; + return { scatterCell: voidCell, dataset }; + } + + // 提取color分组字段 + const colorFieldsValues: string[][] = scatterCell.color.map((uniqueId: UniqueId) => + getDomainFromDataset(dataset, uniqueId) + ); + + // 计算图例项笛卡尔积之前，限制条目数量 + const colorItemsLen: number = productLength(colorFieldsValues); + if ( + dataset.length > MAX_POINT_COUNT || + (colorItemsLen > LEGEND_MAX_COUNT && dataset.length > LEGEND_MAX_POINT_COUNT) + ) { + return { + error: true, + errMsg: `数据量或图例项过多，请使用表格展示。` + }; + } + // 计算图例项 + const colorItemsList: string[][] = legendProduct(colorFieldsValues, false); + const colorItems: string[] = colorItemsList.map(d => d.join('-')); + + dataset = dataset.map((data: DataItem) => { + const groupItem: string = groupDimensions.map((field: UniqueId) => data[field]).join('-'); + return { + ...data, + [GROUP_FIELD]: groupItem + }; + }); + aliasMap[GROUP_FIELD] = `细分`; + + return { + scatterCell, + dataset: [[[dataset]]], + colorItems, + aliasMap + }; +}; + +export const sortTimeDim = (dimList, MAX_PIVOT_ROW, MAX_PIVOT_COLUMN): UniqueId[] => { + //对于lineChart,先进行预排序，将第一个时间维度放到能分配到x轴的位置上 + const dimListLength = dimList.length; + + //第一个时间维度 + const firstTimeIndex = dimList.findIndex(dim => { + const isDateType = dataUtils.isTemporal(dim.dataType); + return isDateType; + }); + + let targetPosition; + if (MAX_PIVOT_COLUMN + MAX_PIVOT_ROW > dimListLength - 1) { + //维度数量不够形成MAX_PIVOT_COLUMN*MAX_PIVOT_ROW的透视，此时会将最后一个维度分配到x轴 + targetPosition = dimListLength - 1; + } else { + //维度数量可以形成MAX_PIVOT_COLUMN*MAX_PIVOT_ROW的透视，此时会将透视字段后第一个字段分配到x轴 + targetPosition = MAX_PIVOT_ROW + MAX_PIVOT_COLUMN; + } + + const idList = dimList.map(dim => dim.uniqueID); + + //将时间维度挪到前面 + const timeItem = idList[firstTimeIndex]; + + idList.splice(firstTimeIndex, 1); + idList.splice(targetPosition, 0, timeItem); + + return idList; +}; + +export const assignPieChart = (originDataset, dimList: UniqueId[], measureList: UniqueId[], aliasMapOld: AliasMap) => { + const aliasMap = cloneDeep(aliasMapOld); + let dataset = cloneDeep(originDataset); + + const pieCell: AutoChartCell = { + x: [], + y: [], + row: [], + column: [], + color: [], + size: [], + angle: [] + }; + + //不满足自动图表下饼图字段要求 + if (!(dimList.length === 0 && measureList.length >= 3)) { + return { pieCell, dataset }; + } + + // 饼图的所有维度都参与颜色计算 + const colorFields: UniqueId[] = [...dimList]; + // 图例项的字段中是否包含指标名称（包含的话需要将指标名称放到最后计算笛卡尔积） + let colorFieldsIncludeMeasure = false; + + const measuresListLength: number = measureList.length; + if (measuresListLength > 1) { + // 多度量 + aliasMap[FOLD_NAME] = `指标名称`; + aliasMap[FOLD_VALUE] = `指标值`; + colorFields.push(FOLD_NAME); + colorFieldsIncludeMeasure = true; + pieCell.angle.push(FOLD_VALUE); + } else if (measuresListLength === 1) { + // 单度量 + pieCell.angle.push(measureList[0]); + } + + // 提取 color分组字段 （维度 以及 度量名称） + const colorFieldsValues: string[][] = colorFields.map((uniqueId: UniqueId) => + getDomainFromDataset(dataset, uniqueId) + ); + + // 计算图例项笛卡尔积之前，限制条目数量 + const colorItemsLen: number = productLength(colorFieldsValues); + if ( + dataset.length > MAX_POINT_COUNT || + (colorItemsLen > LEGEND_MAX_COUNT && dataset.length > LEGEND_MAX_POINT_COUNT) + ) { + return { + error: true, + errMsg: `数据量或图例项过多，请使用表格展示。` + }; + } + + // 计算图例项 + const colorItemsList: string[][] = legendProduct(colorFieldsValues, colorFieldsIncludeMeasure); + let colorItems: string[] = colorItemsList.map(d => d.join('-')); + + // 使用笛卡尔积后的图例项替换原数据中对应的字段 + if (colorFields.length > 0) { + dataset = dataset.map((data: DataItem) => { + const colorItem = colorFields.map(field => data[field]).join('-'); + return { + ...data, + [COLOR_FIELD]: colorItem + }; + }); + // 将笛卡尔积后无效的 colorItem 移除 + const _colorItems: string[] = getDomainFromDataset(dataset, COLOR_FIELD); + colorItems = colorItems.filter((d: string) => _colorItems.includes(d)); + + aliasMap[COLOR_FIELD] = `图例项`; + pieCell.color.push(COLOR_FIELD); + pieCell.cartesianInfo = getCartesianInfo(colorFields, COLOR_FIELD); + + // 多度量时的平坦化字段 + if (measuresListLength > 1) { + pieCell.foldInfo = getFoldInfo(measureList, FOLD_NAME, FOLD_VALUE, aliasMap); + } + } + + return { pieCell, dataset, colorItems, aliasMap }; +}; + +export const assignMeasureCard = ( + originDataset, + dimList: UniqueId[], + measureList: UniqueId[], + aliasMapOld: AliasMap +) => { + // eslint-disable-line + const aliasMap = cloneDeep(aliasMapOld); + const dataset = cloneDeep(originDataset); + + const cardDataset: Dataset = fold(dataset, measureList, FOLD_NAME, FOLD_VALUE, aliasMap); + + const cardCell: AutoChartCell = { + x: [], + y: [], + row: [], + column: [], + color: [FOLD_NAME], + size: [FOLD_VALUE], + angle: [], + value: [FOLD_VALUE], + text: [FOLD_NAME] + }; + + return { cardCell, dataset: cardDataset }; +}; + +export const assignFunnelChart = ( + originDataset, + dimList: UniqueId[], + measureList: UniqueId[], + aliasMapOld: AliasMap +) => { + const aliasMap = cloneDeep(aliasMapOld); + const dataset = cloneDeep(originDataset); + + const funnelCell: AutoChartCell = { + x: [], + y: [], + row: [], + column: [], + color: [], + size: [], + angle: [] + }; + + //不符合最低要求 + if (!((dimList.length === 1 && measureList.length === 1) || (dimList.length === 0 && measureList.length >= 2))) { + return { funnelCell, dataset }; + } + + const measuresListLength: number = measureList.length; + if (measuresListLength > 1) { + // 多度量 + aliasMap[FOLD_NAME] = `指标名称`; + aliasMap[FOLD_VALUE] = `指标值`; + funnelCell.size.push(FOLD_VALUE); + funnelCell.foldInfo = getFoldInfo(measureList, FOLD_NAME, FOLD_VALUE, aliasMap); + } else if (measuresListLength === 1) { + // 单度量 + funnelCell.size.push(measureList[0]); + } + + // 提取color分组字段 + const dimensionIdListLength: number = dimList.length; + if (dimensionIdListLength > 0) { + funnelCell.color.push(dimList[0]); + } else if (dimensionIdListLength === 0 && measuresListLength > 1) { + funnelCell.size.push(FOLD_NAME); + } + + return { funnelCell, dataset }; +}; + +export const assignDualAxis = ( + originDataset, + dimList: UniqueId[], + measureList: UniqueId[], + subMeasureList: UniqueId[], + aliasMapOld: AliasMap +) => { + const aliasMap = cloneDeep(aliasMapOld); + let dataset = cloneDeep(originDataset); + const cell: AutoChartCell = { + x: [], + y: [], + row: [], + column: [], + color: [], + cartesianInfo: null, + foldInfo: null + }; + + const singleSide = isMain => { + let _measuresIdList: UniqueId[]; + let removeIdList: UniqueId[]; + let FOLD_VALUE_FIELD: UniqueId; + let aliasFoldValue: string; + + if (isMain) { + _measuresIdList = measureList; + removeIdList = subMeasureList; + FOLD_VALUE_FIELD = FOLD_VALUE_MAIN; + aliasFoldValue = `指标值(主轴)`; + } else { + _measuresIdList = subMeasureList; + removeIdList = measureList; + FOLD_VALUE_FIELD = FOLD_VALUE_SUB; + aliasFoldValue = `指标值(次轴)`; + } + + if (_measuresIdList.length === 0) { + return []; + } + + // 只保留属于该轴的指标字段 + let sideDataset: Dataset = removeDatasetField(dataset, removeIdList); + + // 无论是否多度量都需要进行平坦化 + sideDataset = fold(sideDataset, _measuresIdList, FOLD_NAME, FOLD_VALUE_FIELD, aliasMap, false); + aliasMap[FOLD_VALUE_FIELD] = aliasFoldValue; + cell.y.push(FOLD_VALUE_FIELD); + return sideDataset; + }; + + // 左右轴依次进行平坦化 + const datasetMain: Dataset = singleSide(true); + const datasetSub: Dataset = singleSide(false); + + dataset = [].concat(datasetMain, datasetSub); + + const dimensionIdList: UniqueId[] = dimList; + const measureIdList: UniqueId[] = measureList.concat(subMeasureList); + + // 维度中参与图例项笛卡尔积计算的维度 + const colorFields: UniqueId[] = []; + + if (dimensionIdList.length > 0) { + // 提取 x 字段（维度第一项） + cell.x.push(String(dimensionIdList[0])); + // 其余项参与笛卡尔积计算 + colorFields.push(...dimensionIdList.slice(1)); + } + + const measuresListLength: number = measureIdList.length; + // 提取 y 字段（度量值） + if (measuresListLength > 0) { + aliasMap[FOLD_NAME] = `指标名称`; + colorFields.push(FOLD_NAME); + } + + // 提取 color分组字段 （维度其他项 以及 度量名称） + const colorFieldsValues: string[][] = colorFields.map((uniqueId: UniqueId) => + getDomainFromDataset(dataset, uniqueId) + ); + + // 计算图例项笛卡尔积之前，限制条目数量 + const dimItemsLen: number = getDomainFromDataset(dataset, cell.x[0]).length; + const colorItemsLen: number = productLength(colorFieldsValues); + if ( + dimItemsLen > X_MAX_COUNT || + dataset.length > MAX_POINT_COUNT || + (colorItemsLen > LEGEND_MAX_COUNT && dataset.length > LEGEND_MAX_POINT_COUNT) + ) { + return { + error: true, + errorMsg: `数据量或图例项过多，请使用表格展示。` + }; + } + + // 计算图例项 + const colorItemsList: string[][] = legendProduct(colorFieldsValues, true); + let colorItems: string[] = colorItemsList.map(d => d.join('-')); + + // 使用笛卡尔积后的图例项替换原数据中对应的字段 + if (colorFields.length > 0) { + dataset = dataset.map((data: DataItem) => { + const colorItem = colorFields.map(field => data[field]).join('-'); + return { + ...data, + [COLOR_FIELD]: colorItem + }; + }); + // 将笛卡尔积后无效的 colorItem 移除 + const _colorItems: string[] = getDomainFromDataset(dataset, COLOR_FIELD); + colorItems = colorItems.filter((d: string) => _colorItems.includes(d)); + + aliasMap[COLOR_FIELD] = `图例项`; + cell.color.push(COLOR_FIELD); + cell.cartesianInfo = getCartesianInfo(colorFields, COLOR_FIELD); + cell.foldInfo = getFoldInfo(measureIdList, FOLD_NAME, [FOLD_VALUE_MAIN, FOLD_VALUE_SUB], aliasMap); + } + + return { + dataset, + cell, + colorItems, + aliasMap + }; +}; diff --git a/packages/chart-advisor/src/fieldUtils.ts b/packages/chart-advisor/src/fieldUtils.ts new file mode 100644 index 00000000..804f8bd3 --- /dev/null +++ b/packages/chart-advisor/src/fieldUtils.ts @@ -0,0 +1,127 @@ +import { uniq, pick, omit } from 'lodash'; +import { FoldInfo } from './type'; +import { Dataset, UniqueId, DataItem, AliasMap } from './type'; + +/** + * 输入二维数组，输出笛卡尔积长度 + * @param list + */ +export const productLength = (list: any[][]) => + list.length === 0 ? 0 : list.map(d => d.length).reduce((pre, cur) => pre * cur, 1); + +/** + * 输入二维数组，输出笛卡尔积 + * 新增了计算笛卡尔积时将指标名称放在最后计算的特殊处理 + * @param list + */ +export const legendProduct = (list: any[][], hasMeasureName = false) => { + if (hasMeasureName && list.length > 1) { + const _list = [...list]; + const measureNames = _list.pop(); + const productResult = product(_list); + return measureNames + .map((measureName: any[]) => productResult.map((d: any[]) => d.concat(measureName))) + .reduce((pre, cur) => pre.concat(cur), []); + } + return product(list); +}; + +// 计算笛卡尔积 +const product = (list: any[][]) => + list.length === 0 + ? [] + : list.reduce( + function (a, b) { + return a + .map(function (x) { + return b.map(function (y) { + return x.concat(y); + }); + }) + .reduce(function (a, b) { + return a.concat(b); + }, []); + }, + [[]] + ); + +// 从数据集中获取 domain map +export const getDomainFromDataset = (dataset: Dataset, dim: UniqueId): string[] => { + const values: string[] = dataset.map((d: DataItem) => String(d[dim])); + return uniq(values); +}; + +// 保留dataset中的某些字段 +export const retainDatasetField = (dataset: Dataset, fields: UniqueId[]): Dataset => + dataset.map((data: DataItem) => pick(data, fields)) as Dataset; + +// 移除dataset中的某些字段 +export const removeDatasetField = (dataset: Dataset, fields: UniqueId[]): Dataset => + dataset.map((data: DataItem) => omit(data, fields)) as Dataset; + +/** + * 维度笛卡尔积的原始信息，因此可以通过cartesianInfo匹配原始的维度字段 + */ +export const getCartesianInfo = (fieldList: UniqueId[], key: UniqueId) => ({ + key, + fieldList +}); + +/** + * 由于多度量的情况下数据会被平坦化，会导致原本的字段信息丢失，因此可以通过foldInfo匹配原始的指标字段 + */ +export const getFoldInfo = ( + measuresId: UniqueId[], + foldName: UniqueId, + foldValue: UniqueId | UniqueId[], + aliasMap: AliasMap +) => + ({ + key: foldName, + value: foldValue, + // foldMap: Object.fromEntries(new Map( + foldMap: strMap2Obj(new Map(measuresId.map(id => [id, aliasMap[id]]))) + } as FoldInfo); +/** + * 临时替代fromEntries 以兼容73以前的chrome + */ +const strMap2Obj = strMap => { + const obj = Object.create(null); + for (const [k, v] of strMap) { + obj[k] = v; + } + return obj; +}; + +/** + * Transform fold 字段展开 + * @param dataset 原始数据集 + * @param fields 待展开的字段集 + * @param foldName foldName字段 + * @param foldValue foldValue字段 + * @param aliasMap 别名表 + * @param retains 是否保留被展开的字段 + * @return dataset + */ +export const fold = ( + dataset: Dataset, + fields: UniqueId[], + foldName: UniqueId, + foldValue: UniqueId, + aliasMap?: AliasMap, + retains = true +) => { + const _dataset = []; + dataset.forEach((data: DataItem) => { + fields.forEach((field: UniqueId) => { + // 是否保留被展开的字段 + const _data = retains ? data : omit(data, fields); + _dataset.push({ + ..._data, + [foldName]: aliasMap ? aliasMap[field] : field, + [foldValue]: data[field] + }); + }); + }); + return _dataset; +}; diff --git a/packages/chart-advisor/src/index.ts b/packages/chart-advisor/src/index.ts new file mode 100644 index 00000000..c0770f2a --- /dev/null +++ b/packages/chart-advisor/src/index.ts @@ -0,0 +1,140 @@ +import { DimensionDataset, MeasureDataset, ChartType, ScreenSize, UserPurpose } from './type'; +import type { + AdviseResult, + Scorer, + AdviserParams, + ScoreResult, + DataTypeName, + MeasureField, + DimensionField +} from './type'; +import { scorer as defaultScorer } from './score'; +import * as dataUtils from './dataUtil'; +import { isNil, isNaN } from 'lodash'; + +export { fold } from './fieldUtils'; +export { FOLD_NAME, FOLD_VALUE } from './constant'; + +export function chartAdvisor(params: AdviserParams): AdviseResult { + const { + originDataset, + dimensionList, + measureList, + aliasMap = {}, + maxPivotRow = 0, + maxPivotColumn = 0, + purpose = UserPurpose.NONE, + screen = ScreenSize.LARGE, + scorer = defaultScorer + } = params; + + const measureDatasets: MeasureDataset[] = []; + + /* + 计算measure的统计特征。 + 由于服务端返回的vizData.dataset做了平坦化(fold)处理，且remain=false， + 因此当多度量时需要先判断dataset中是否有该uniqueID对应的值. + */ + measureList.forEach(measure => { + const uniqueID = measure.uniqueId; + const measureSet: MeasureDataset = { + data: [] + }; + measureSet.uniqueID = uniqueID; + originDataset.forEach(row => { + if (row.hasOwnProperty(uniqueID)) { + measureSet.data.push(parseFloat(row[uniqueID])); + } + }); + const dataNotNull = measureSet.data.filter(each => !isNil(each) && !isNaN(each)); + measureSet.min = Math.min(...dataNotNull); + measureSet.max = Math.max(...dataNotNull); + measureSet.mean = dataUtils.calMean(measureSet); + measureSet.standardDev = dataUtils.calStandardDeviation(measureSet); + measureSet.coefficient = dataUtils.calCoefficient(measureSet); + measureSet.Q1 = dataUtils.calQuantile(measureSet, 0.25); + + measureDatasets.push(measureSet); + }); + + const dimensionDatasets: DimensionDataset[] = []; + + /* + 计算dimension的字段特征（字段名、基数、基数/数据条数）。 + */ + dimensionList.forEach(dimension => { + const uniqueID = dimension.uniqueId; + const dimensionSet: DimensionDataset = { + data: [] + }; + dimensionSet.uniqueID = uniqueID; + originDataset.forEach(row => { + dimensionSet.data.push(row[uniqueID]); + }); + dimensionSet.dataType = dimension.type; + dimensionSet.dimensionName = aliasMap[uniqueID]; + dimensionSet.cardinal = dataUtils.unique(dimensionSet.data).length; + dimensionSet.ratio = dimensionSet.cardinal / dimensionSet.data.length; + dimensionSet.isGeoField = !!dimension.isGeoField; + dimensionDatasets.push(dimensionSet); + }); + + try { + const scores = scorer({ + inputDataSet: originDataset, + dimList: dimensionDatasets, + measureList: measureDatasets, + aliasMap, + maxRowNum: maxPivotRow, + maxColNum: maxPivotColumn, + purpose, + screen + }).map(calculator => { + const score = calculator(); + return score; + }); + + scores.sort((chart1, chart2) => chart2.score - chart1.score); + // console.log(scores) + if (scores[0].score === 0) { + return { + chartType: ChartType.TABLE, + scores: [] + }; + } + // console.log(scores) + + // scores.forEach(score => { + // let cell = score.cell + // if (!Array.isArray(cell)) { + // cell = [cell] + // } + // //将所有的key转换为string + // cell.forEach(cl => { + // Object.entries(cl).forEach(([k, v]) => { + // if (k === 'cartesianInfo' || k === 'foldInfo') { + // cl[k] = null + // } + // else { + // cl[k] = v.map(value => String(value)) + // } + // }) + // }) + + // score.cell = cell + // }) + + return { + chartType: scores[0].chartType, + scores + }; + } catch (exception: any) { + return { + chartType: ChartType.TABLE, + scores: [], + error: exception.message ?? exception + }; + } +} + +export { Scorer, AdviserParams, ScoreResult, ChartType, AdviseResult, DataTypeName, MeasureField, DimensionField }; diff --git a/packages/chart-advisor/src/pivot.ts b/packages/chart-advisor/src/pivot.ts new file mode 100644 index 00000000..c2136a72 --- /dev/null +++ b/packages/chart-advisor/src/pivot.ts @@ -0,0 +1,203 @@ +import { isNil, range, uniq } from 'lodash'; + +import { getDomainFromDataset } from './fieldUtils'; +import { UniqueId, Dataset, Datasets, DataItem, PivotTree } from './type'; + +// 透视分析 +export const pivot = ( + dataset: Dataset, + colList: UniqueId[], + rowList: UniqueId[], + measureList: UniqueId[], + paginationInfo?: any +): { + datasets: Datasets; + colPivotTree: PivotTree; + rowPivotTree: PivotTree; + length: number; +} => { + // 行列的透视的 pathList + const rowPathList = getPathList(rowList, dataset); + const colPathList = getPathList(colList, dataset); + + // 按行分页 + const rowPathListPage = paginationDataset(rowPathList, paginationInfo); + + // 按行维度透视 + const rowGroups = groupByDims(dataset, rowList, rowPathListPage); + + // 按列维度透视 + const colGroups = rowGroups.map((row: Dataset) => groupByDims(row, colList, colPathList)); + + // 按多指标透视 (透视表中实际不存在此场景，因为多指标已经被平坦化) + const groups: Datasets = colGroups.map((pane: Dataset[]) => + pane.map((cell: Dataset) => groupByMeas(cell, measureList)) + ); + + // 根据分页后的 pathList 生成 header tree + const rowPivotTree = pivotTree(rowList, rowPathListPage); + const colPivotTree = pivotTree(colList, colPathList); + + return { + datasets: groups, + colPivotTree, + rowPivotTree, + length: rowGroups.length + }; +}; + +// 组合图透视分析 +export const pivotCombination = ( + dataset: Dataset[], + colList: UniqueId[], + rowList: UniqueId[] +): { + datasets: Datasets; + colPivotTree: PivotTree; + rowPivotTree: PivotTree; + length: number; +} => { + // 组合图 meta 数量 + const metaLength: number = dataset.length; + + // 先为数据打上 组合图 mate index tag， 然后将数据平坦为一个集合 + const datasetWithTag: Dataset[] = dataset.map((dataList: Dataset, index: number) => + dataList.map((data: DataItem) => addTag(data, index)) + ); + const flatDataset = datasetWithTag.flat(); + + // 行列的透视的 path_list + const rowPathList = getPathList(rowList, flatDataset); + const colPathList = getPathList(colList, flatDataset); + + // 按行维度透视 + const rowGroups = groupByDims(flatDataset, rowList, rowPathList); + + // 按列维度透视 + const colGroups = rowGroups.map((row: Dataset) => groupByDims(row, colList, colPathList)); + + // 按多指标透视 + const groups: Datasets = colGroups.map((pane: Dataset[]) => + pane.map((cell: Dataset) => groupByMeta(cell, metaLength)) + ); + // console.log(groups) + + // 根据分页后的 pathList 生成 header tree + const rowPivotTree = pivotTree(rowList, rowPathList); + const colPivotTree = pivotTree(colList, colPathList); + + return { + datasets: groups, + colPivotTree, + rowPivotTree, + length: rowGroups.length + }; +}; + +// 通过 domain_map 和 sort_service 生成透视路径集合 +type Path = string[]; +const getPathList = (keys: UniqueId[], dataset: Dataset, filterList: Path = []) => { + const pathList: Path[] = []; + if (keys.length > 0) { + const key: UniqueId = keys[0]; + const valueList: string[] = getDomainFromDataset(dataset, key); + + valueList.forEach((value: string) => { + if (keys.length > 1) { + const _dataset: Dataset = filterDataItem(dataset, [key], [value]); + const _filterList: Path = [...filterList, value]; + pathList.push(...getPathList(keys.slice(1), _dataset, _filterList)); + } else { + pathList.push([...filterList, value]); + } + }); + } + return pathList; +}; + +// 分组 dimension +const groupByDims = (source: Dataset, keys: UniqueId[], pathList: Path[]): Dataset[] => { + if (pathList.length === 0) { + return [source]; + } + const groups: Dataset[] = pathList.map((path: Path) => filterDataItem(source, keys, path)); + return groups; +}; + +// 分组 measure +const groupByMeas = (source: Dataset, measures: UniqueId[]): Dataset[] => { + if (measures.length <= 1) { + return [source]; + } + return measures.map((measure: UniqueId) => source.filter((dataItem: DataItem) => !isNil(dataItem[measure]))); +}; + +// 分组 meta +const groupByMeta = (source: Dataset, length: number): Dataset[] => { + const group: Dataset[] = range(length).map((index: number) => + source.filter((data: DataItem) => data[COMBINATION_INDEX] === index).map((data: DataItem) => removeTag(data)) + ); + return group; +}; + +// 分页逻辑 +const paginationDataset = (pathList: Path[], paginationInfo?: any) => { + if (isNil(paginationInfo)) { + return pathList; + } + const pageOffset: number = paginationInfo.offset; + const pageSize: number = paginationInfo.size; + return pathList.slice(pageOffset, pageOffset + pageSize); +}; + +// 生成 pivot header tree 递归创建 header node +const pivotTree = (keys: UniqueId[], pathList: Path[], deep = 0): PivotTree => { + if (keys.length < deep + 1) { + return null; + } + const getDomainFromPath = () => { + const nodes: string[] = pathList.map((path: Path) => path[deep]); + return uniq(nodes); + }; + const field: UniqueId = keys[deep]; + const domain: string[] = getDomainFromPath(); + return { + field, + values: domain.map((value: string) => ({ + value, + child: pivotTree( + keys, + pathList.filter((path: Path) => path[deep] === value), + deep + 1 + ) + })) + }; +}; + +// 工具函数 + +// 通过分组好的 kv 数据 在数据集中过滤出满足条件的数据项 +const filterDataItem = (sourceList: Dataset, keyList: UniqueId[], valueList: string[]): Dataset => + sourceList.filter(dataItem => checkKeysValues(dataItem, keyList, valueList)); + +// 通过指定的 kv 信息判断当前的数据项是否匹配 +const checkKeysValues = (dataItem: any, keyList: UniqueId[], valueList: string[]): boolean => + range(keyList.length) + .map((idx: number) => String(dataItem[keyList[idx]]) === valueList[idx]) + .reduce((pre: boolean, cur: boolean) => pre && cur, true); + +// 组合图透视分析所依赖的tag +const COMBINATION_INDEX = '__combination_index__'; + +// 增加tag +const addTag = (dataItem: DataItem, i: number) => ({ + ...dataItem, + [COMBINATION_INDEX]: i +}); + +// 移除tag +const removeTag = (dataItem: DataItem) => { + const _dataItem = { ...dataItem }; + delete _dataItem[COMBINATION_INDEX]; + return _dataItem; +}; diff --git a/packages/chart-advisor/src/score.ts b/packages/chart-advisor/src/score.ts new file mode 100644 index 00000000..8833a1a2 --- /dev/null +++ b/packages/chart-advisor/src/score.ts @@ -0,0 +1,1200 @@ +import { isTemporal } from './dataUtil'; +import { cloneDeep, uniq } from 'lodash'; +import { + ChartType, + MeasureDataset, + ScreenSize, + UserPurpose, + AutoChartCell, + Dataset, + UniqueId, + ScoreResult, + Scorer +} from './type'; +import * as dataUtils from './dataUtil'; +import { + assignMeasureCard, + assignPieChart, + assignPivotCharts, + assignScatterPlot, + assignFunnelChart, + sortTimeDim, + processCombination, + assignDualAxis +} from './fieldAssign'; +import { COLOR_FIELD, MAX_BAR_NUMBER, MIN_BAR_NUMBER, FOLD_NAME, FOLD_VALUE } from './constant'; +import { getDomainFromDataset, fold } from './fieldUtils'; +import { pivot, pivotCombination } from './pivot'; + +export const scorer: Scorer = params => { + const { + inputDataSet, + dimList, + measureList, + aliasMap = {}, + maxRowNum = 0, + maxColNum = 0, + purpose = UserPurpose.NONE, + screen = ScreenSize.LARGE + } = params; + + const datasetWithoutFold = cloneDeep(inputDataSet); + let originDataset = inputDataSet; + + //多度量可能没有扁平化 + if (measureList.length > 1 && !originDataset[0].hasOwnProperty(FOLD_NAME)) { + originDataset = fold( + originDataset, + measureList.map(measure => measure.uniqueID), + FOLD_NAME, + FOLD_VALUE, + aliasMap, + false + ); + } + + //找出时间字段和非时间字段 + const timeDim: UniqueId[] = []; + const noneTimeDim: UniqueId[] = []; + dimList.forEach(dim => { + if (isTemporal(dim.dataType)) { + timeDim.push(dim.uniqueID); + } else { + noneTimeDim.push(dim.uniqueID); + } + }); + + //uniqueId到字段values的映射 + const uniqueIdMap = {}; + dimList.forEach(dim => { + uniqueIdMap[dim.uniqueID] = dim; + }); + + measureList.forEach(measure => { + uniqueIdMap[measure.uniqueID] = measure; + }); + + const dimensionID = dimList.map(dim => dim.uniqueID); + const measureID = measureList.map(measure => measure.uniqueID); + + const pivotChartData = assignPivotCharts(originDataset, dimensionID, measureID, aliasMap, maxRowNum, maxColNum); + + const { dataset, colorItems, error, errMsg, aliasMap: newAliasMap } = pivotChartData; + let { cell } = pivotChartData; + + const colList = error ? [] : cell.column; + const rowList = error ? [] : cell.row; + const emptyCell = { + x: [], + y: [], + column: [], + row: [], + color: [], + size: [], + angle: [] + }; + // @todo:图例过多处理 + if (error) { + cell = emptyCell; + } + + // 透视分析 + const { datasets: pivotDataSet, colPivotTree, rowPivotTree } = pivot(dataset, colList, rowList, cell.y); + + const calBarParallel = (): ScoreResult => { + let score = 0; + const scoreDetails: any = {}; + let totalScore = 0; + + if (error) { + return { + chartType: ChartType.COLUMN_PARALLEL, + originScore: 0, + score: 0, + fullMark: 0, + scoreDetails, + error: error ? errMsg : null + }; + } + const measureLength = measureList.length; + + //维度数>0,指标数>0 + const rule1Score = 1.0; + totalScore += rule1Score; + if (cell.x.length > 0 && cell.y.length > 0) { + score += rule1Score; + scoreDetails.rule1 = rule1Score; + } else { + return { + chartType: ChartType.COLUMN_PARALLEL, + originScore: 0, + score: 0, + fullMark: 0, + scoreDetails + }; + } + + //指标最大值里面最大的/指标Q1(下四分位数)里面最小的<100（不同指标数值在同一个量级） + const rule2Score = 3.0; + totalScore += rule2Score; + let maxMax = Math.max(...measureList.map(measure => Math.abs(measure.max))); + let minQ1 = Math.min(...measureList.map(measure => Math.abs(measure.Q1))); + if (maxMax === 0) { + maxMax = 1; + } + if (minQ1 === 0) { + minQ1 = 1; + } + + if (maxMax / minQ1 < 100) { + score += rule2Score; + scoreDetails.rule2 = rule2Score; + } + + //2<=图元数量<=30 (根据屏幕尺寸调整) + const rule3Score = 4.0; + totalScore += rule3Score; + + const colorSize = cell.color.length > 0 ? dataUtils.unique(dataset.map(data => data[COLOR_FIELD])).length : 1; + const axisSize = dataUtils.unique(uniqueIdMap[cell.x[0]].data).length; + + const dataSize = colorSize * axisSize; + + if (dataSize <= MAX_BAR_NUMBER && dataSize >= MIN_BAR_NUMBER) { + score += rule3Score; + scoreDetails.rule4 = rule3Score; + } + + return { + chartType: ChartType.COLUMN_PARALLEL, + score: score / totalScore, + fullMark: totalScore, + originScore: score, + scoreDetails, + cell, + dataset + }; + }; + + const calBarPercent = (): ScoreResult => { + let score = 0; + let totalScore = 0; + const scoreDetails: any = {}; + + if (error) { + return { + chartType: ChartType.COLUMN_PERCENT, + originScore: 0, + fullMark: 0, + score: 0, + scoreDetails, + error: error ? errMsg : null + }; + } + + const dimensionLength = dimList.length - cell.row.length - cell.column.length; + const measureLength = measureList.length; + + //维度数>1，指标数>0（不能是平行柱状图） + const rule1Score = 1.0; + totalScore += rule1Score; + if (dimensionLength > 1 && measureLength > 0) { + score += rule1Score; + scoreDetails.rule1 = rule1Score; + } else { + return { + chartType: ChartType.COLUMN_PERCENT, + score: 0, + fullMark: 0, + originScore: 0, + scoreDetails + }; + } + + //指标最大值里面最大的/指标Q1(下四分位数)里面最小的<100（不同指标数值在同一个量级） + const rule2Score = 3.0; + totalScore += rule2Score; + + let maxMax = Math.max(...measureList.map(measure => Math.abs(measure.max))); + let minQ1 = Math.min(...measureList.map(measure => Math.abs(measure.Q1))); + if (maxMax === 0) { + maxMax = 1; + } + if (minQ1 === 0) { + minQ1 = 1; + } + + if (maxMax / minQ1 < 100) { + score += rule2Score; + scoreDetails.rule2 = rule2Score; + } + + //2<=图元数量<=30 (根据屏幕尺寸调整) + const rule3Score = 1.0; + totalScore += rule3Score; + + const axisSize = dataUtils.unique(uniqueIdMap[cell.x[0]].data).length; + + const dataSize = axisSize * measureLength; + + if (dataSize <= MAX_BAR_NUMBER && dataSize >= MIN_BAR_NUMBER) { + score += rule3Score; + scoreDetails.rule3 = rule3Score; + } + + //图例数量<=150 (根据屏幕尺寸调整) + const rule4Score = 1.0; + totalScore += rule4Score; + + const colorSize = cell.color.length > 0 ? dataUtils.unique(dataset.map(data => data[COLOR_FIELD])).length : 1; + + if (colorSize <= MAX_BAR_NUMBER) { + score += rule4Score; + scoreDetails.rule4 = rule4Score; + } + + //用户目的=占比 + const rule5Score = 1.0; + totalScore += rule5Score; + if (purpose === UserPurpose.PROPORTION) { + score += rule5Score; + scoreDetails.rule5 = rule5Score; + } + + return { + chartType: ChartType.COLUMN_PERCENT, + score: score / totalScore, + originScore: score, + fullMark: totalScore, + scoreDetails, + cell, + dataset + }; + }; + + const calBar = (): ScoreResult => { + let score = 0; + let totalScore = 0; + const scoreDetails: any = {}; + + if (error) { + return { + chartType: ChartType.COLUMN, + score: 0, + scoreDetails, + fullMark: 0, + originScore: 0, + error: error ? errMsg : '' + }; + } + + const dimensionLength = dimList.length - cell.row.length - cell.column.length; + const measureLength = measureList.length; + + //维度数>1（不能是平行柱状图） + //databot中，维度数>=1，指标数>=1 + const rule1Score = 1.0; + totalScore += rule1Score; + + if (dimensionLength >= 1 && measureLength >= 1) { + score += rule1Score; + scoreDetails.rule1 = rule1Score; + } else { + return { + chartType: ChartType.COLUMN, + score: 0, + scoreDetails, + originScore: 0, + fullMark: 0 + }; + } + + //指标最大值里面最大的/指标Q1(下四分位数)里面最小的<100（指标数值在同一个量级） + const rule2Score = 3.0; + totalScore += rule2Score; + + let maxMax = Math.max(...measureList.map(measure => Math.abs(measure.max))); + let minQ1 = Math.min(...measureList.map(measure => Math.abs(measure.Q1))); + if (maxMax === 0) { + maxMax = 1; + } + if (minQ1 === 0) { + minQ1 = 1; + } + + if (maxMax / minQ1 < 100) { + score += rule2Score; + scoreDetails.rule2 = rule2Score; + } + + //2<=图元数量<=30 (根据屏幕尺寸调整) + const rule3Score = 1.0; + totalScore += rule3Score; + + const axisSize = dataUtils.unique(uniqueIdMap[cell.x[0]].data).length; + const dataSize = measureLength * axisSize; + + if (dataSize <= MAX_BAR_NUMBER && dataSize >= MIN_BAR_NUMBER) { + score += rule3Score; + scoreDetails.rule3 = rule3Score; + } + + //图例数量<=150 (根据屏幕尺寸调整) + const rule4Score = 1.0; + totalScore += rule4Score; + + const colorSize = cell.color.length > 0 ? dataUtils.unique(dataset.map(data => data[COLOR_FIELD])).length : 1; + + if (colorSize <= MAX_BAR_NUMBER) { + score += rule4Score; + scoreDetails.rule4 = rule4Score; + } + + //屏幕尺寸=小 + const rule5Score = 1.0; + totalScore += rule5Score; + if (screen === ScreenSize.SMALL) { + score += rule5Score; + scoreDetails.rule4 = rule5Score; + } + + return { + chartType: ChartType.COLUMN, + score: score / totalScore, + originScore: score, + fullMark: totalScore, + scoreDetails, + cell, + dataset + }; + }; + + //数据差异过大时，使用组合柱状图。 + const calCombination = (): ScoreResult => { + let score = 0; + let totalScore = 0; + const scoreDetails: any = {}; + + if (error) { + return { + chartType: ChartType.COMBINATION, + score: 0, + scoreDetails, + originScore: 0, + fullMark: 0, + error: error ? errMsg : null + }; + } + + //根据cell决定参与图表推荐的字段 + const measureLength = measureList.length; + + //维度数>0，指标数>1 + const rule1Score = 1.0; + totalScore += rule1Score; + if (measureLength > 1 && cell.x.length > 0) { + score += rule1Score; + scoreDetails.rule1 = rule1Score; + } else { + return { + chartType: ChartType.COMBINATION, + score: 0, + scoreDetails, + originScore: 0, + fullMark: 0 + }; + } + + //指标最大值里面最大的/指标Q1(下四分位数)里面最小的>100（不同指标数值不在同一个量级） + const rule2Score = 1.0; + totalScore += rule2Score; + let minQ1 = Math.min(...measureList.map(measure => Math.abs(measure.Q1))); + let maxMax = Math.max(...measureList.map(measure => Math.abs(measure.max))); + if (maxMax === 0) { + maxMax = 1; + } + if (minQ1 === 0) { + minQ1 = 1; + } + + if (maxMax / minQ1 > 100) { + score += rule2Score; + scoreDetails.rule2 = rule2Score; + } + + //同一个指标里面最小bar的数值/最大bar的数值>1% + const rule3Score = 3.0; + totalScore += rule3Score; + const score3Flag = measureList.reduce((prev, cur) => { + if (prev) { + return cur.min / cur.max > 0.01; + } + return false; + }, true); + if (score3Flag) { + score += rule3Score; + scoreDetails.rule3 = rule3Score; + } + + //2<=图元数量<=30 (根据屏幕尺寸调整) + const rule4Score = 1.0; + totalScore += rule4Score; + + const colorSize = cell.color.length > 0 ? dataUtils.unique(dataset.map(data => data[COLOR_FIELD])).length : 1; + const axisSize = dataUtils.unique(uniqueIdMap[cell.x[0]].data).length; + + const dataSize = axisSize * colorSize; + if (dataSize <= MAX_BAR_NUMBER && dataSize >= MIN_BAR_NUMBER) { + score += rule4Score; + scoreDetails.rule4 = rule4Score; + } + + //计算cells + const combineMetadata = processCombination( + datasetWithoutFold, + dimensionID, + measureID, + aliasMap, + maxRowNum, + maxColNum + ); + + const combineDatasets: Dataset[] = combineMetadata.map(metaData => metaData.dataset); + const combineCells: any[] = combineMetadata.map(metaData => metaData.cell); + + const combineColorItems: string[] = uniq( + combineMetadata.map(metaData => metaData.colorItems).reduce((pre: string[], cur: string[]) => pre.concat(cur), []) + ); + + // 透视分析 + const { + datasets: combinePivotDataSet, + colPivotTree, + rowPivotTree + } = pivotCombination(combineDatasets, colList, rowList); + + return { + chartType: ChartType.COMBINATION, + score: score / totalScore, + scoreDetails, + originScore: score, + fullMark: totalScore, + cell: combineCells, + dataset: combinePivotDataSet + }; + }; + + const calScatterplot = (): ScoreResult => { + let score = 0; + let totalScore = 0; + const scoreDetails: any = {}; + + const dimensionID = dimList.map(dim => dim.uniqueID); + const measureID = measureList.map(measure => measure.uniqueID); + + const scatterData = assignScatterPlot(datasetWithoutFold, dimensionID, measureID, aliasMap); + + const { scatterCell, dataset, colorItems, aliasMap: newAliasMap, error, errMsg } = scatterData; + + if (error) { + return { + chartType: ChartType.SCATTER, + score: 0, + scoreDetails, + originScore: 0, + fullMark: 0, + error: error ? errMsg : null + }; + } + //维度数>0,指标数>=2，维度+指标<=5 + const rule1Score = 1.0; + totalScore += rule1Score; + if (scatterCell.x.length > 0 && scatterCell.y.length > 0) { + score += rule1Score; + scoreDetails.rule1 = rule1Score; + } else { + return { + chartType: ChartType.SCATTER, + score: 0, + scoreDetails, + originScore: 0, + fullMark: 0 + }; + } + + //数据个数>30 <最大数量限制(根据屏幕尺寸调整) + const rule2Score = 1.0; + totalScore += rule2Score; + if (datasetWithoutFold.length >= 30 && datasetWithoutFold.length <= 1000) { + score += rule2Score; + scoreDetails.rule2 = rule2Score; + } + + //数据分布不能太集中 + + return { + chartType: ChartType.SCATTER, + score: score / totalScore, + scoreDetails, + originScore: score, + fullMark: totalScore, + cell: scatterCell, + dataset + }; + }; + + const calLineChart = (): ScoreResult => { + let score = 0; + let totalScore = 0; + const scoreDetails: any = {}; + + //折线图先进行预排序 + const lineChartDimID: UniqueId[] = sortTimeDim(dimList, maxRowNum, maxColNum); + + const { + cell: lineChartCell, + dataset: lineDataset, + error, + errMsg + } = assignPivotCharts(originDataset, lineChartDimID, measureID, aliasMap, maxRowNum, maxColNum); + if (error) { + return { + chartType: ChartType.LINE, + score: 0, + scoreDetails, + originScore: 0, + fullMark: 0, + error: error ? errMsg : null + }; + } + + //维度数=1且为时间，指标数>=1或维度=2且只有一个时间，指标数=1,非时间维度基数不能太多 + const rule1Score = 2.0; + totalScore += rule1Score - 1.0; + const _colorItems: string[] = getDomainFromDataset(lineDataset, COLOR_FIELD); + const colorItemCardinal = lineDataset.hasOwnProperty(COLOR_FIELD) ? dataUtils.unique(_colorItems).length : 1; + if (timeDim.length > 0 && cell.y.length > 0 && colorItemCardinal <= 50) { + score += rule1Score; + scoreDetails.rule1 = rule1Score; + } else { + return { + chartType: ChartType.LINE, + score: 0, + scoreDetails, + originScore: 0, + fullMark: 0 + }; + } + + //指标数>1时，指标最大值里面最大的/指标Q1(下四分位数)里面最小的<100（不同指标数值在同一个量级） + const rule2Score = 1.0; + if (measureList.length > 1) { + totalScore += rule2Score; + let minQ1 = Math.min(...measureList.map(measure => Math.abs(measure.Q1))); + let maxMax = Math.max(...measureList.map(measure => Math.abs(measure.max))); + if (minQ1 === 0) { + minQ1 = 1; + } + if (maxMax === 0) { + maxMax = 1; + } + + if (maxMax / minQ1 <= 100) { + score += rule2Score; + scoreDetails.rule2 = rule2Score; + } + } + + //变异系数>x (需要调整) + const rule3Score = 1.0; + totalScore += rule3Score; + const coefficientFlag = measureList.reduce((prev, cur) => { + if (!prev) { + return false; + } else { + if (cur.coefficient) { + return cur.coefficient >= 0.2; + } else { + return true; + } + } + }, true); + + if (coefficientFlag) { + score += rule3Score; + scoreDetails.rule3 = rule3Score; + } + + // 透视分析 + const { + datasets: lineChartDataset, + colPivotTree, + rowPivotTree + } = pivot(lineDataset, colList, rowList, lineChartCell.y); + + return { + chartType: ChartType.LINE, + score: score / totalScore, + originScore: score, + fullMark: totalScore, + scoreDetails, + cell: lineChartCell, + dataset: lineDataset + }; + }; + + const calLineChartCombine = (): ScoreResult => { + //组合折线图 + let score = 0; + let totalScore = 0; + const scoreDetails: any = {}; + + //折线图先进行预排序 + const lineChartDimID: UniqueId[] = sortTimeDim(dimList, maxRowNum, maxColNum); + + const { + cell: lineChartCell, + dataset: lineDataset, + error, + errMsg + } = assignPivotCharts(originDataset, lineChartDimID, measureID, aliasMap, maxRowNum, maxColNum); + + if (error) { + return { + chartType: ChartType.LINE, + score: 0, + scoreDetails, + fullMark: 0, + originScore: 0, + error: error ? errMsg : null + }; + } + + //维度数=1且为时间，指标数>=1或维度=2且只有一个时间，指标数=1,非时间维度基数不能太多 + const rule1Score = 2.0; + totalScore += rule1Score - 1.0; + const _colorItems: string[] = getDomainFromDataset(lineDataset, COLOR_FIELD); + const colorItemCardinal = lineDataset.hasOwnProperty(COLOR_FIELD) ? dataUtils.unique(_colorItems).length : 1; + if (timeDim.length > 0 && cell.y.length > 0 && colorItemCardinal <= 50) { + score += rule1Score; + scoreDetails.rule1 = rule1Score; + } else { + return { + chartType: ChartType.LINE, + score: 0, + scoreDetails, + fullMark: 0, + originScore: 0 + }; + } + + //指标数>1时，指标最大值里面最大的/指标Q1(下四分位数)里面最小的>100（不同指标数值不在同一个量级） + const rule2Score = 1.0; + if (measureList.length > 1) { + totalScore += rule2Score; + let minQ1 = Math.min(...measureList.map(measure => Math.abs(measure.Q1))); + let maxMax = Math.max(...measureList.map(measure => Math.abs(measure.max))); + if (minQ1 === 0) { + minQ1 = 1; + } + if (maxMax === 0) { + maxMax = 1; + } + + if (maxMax / minQ1 > 100) { + score += rule2Score; + scoreDetails.rule2 = rule2Score; + } + } + + //变异系数>x (需要调整) + const rule3Score = 1.0; + totalScore += rule3Score; + const coefficientFlag = measureList.reduce((prev, cur) => { + if (!prev) { + return false; + } else { + if (cur.coefficient) { + return cur.coefficient >= 0.2; + } else { + return true; + } + } + }, true); + + if (coefficientFlag) { + score += rule3Score; + scoreDetails.rule3 = rule3Score; + } + + //计算cells + const combineMetadata = processCombination( + datasetWithoutFold, + lineChartDimID, + measureID, + aliasMap, + maxRowNum, + maxColNum + ); + + const combineDatasets: Dataset[] = combineMetadata.map(metaData => metaData.dataset); + const combineCells: any[] = combineMetadata.map(metaData => metaData.cell); + + const combineColorItems: string[] = uniq( + combineMetadata.map(metaData => metaData.colorItems).reduce((pre: string[], cur: string[]) => pre.concat(cur), []) + ); + + // 透视分析 + const { + datasets: combinePivotDataSet, + colPivotTree, + rowPivotTree + } = pivotCombination(combineDatasets, colList, rowList); + + return { + chartType: ChartType.EXTEND, //暂时用扩展类型来代替 + score: score / totalScore, + scoreDetails, + originScore: score, + fullMark: totalScore + }; + }; + + const calPieChart = (): ScoreResult => { + let score = 0; + let totalScore = 0; + const scoreDetails: any = {}; + + const pieChartData = assignPieChart(originDataset, dimensionID, measureID, aliasMap); + + const { pieCell, dataset: pieDataset, colorItems, aliasMap: newAliasMap, error, errMsg } = pieChartData; + + if (error) { + return { + chartType: ChartType.PIE, + score: 0, + scoreDetails, + fullMark: 0, + originScore: 0, + error: error ? errMsg : null + }; + } + + //维度数=0，指标数>=3 + const rule1Score = 2.0; + totalScore += rule1Score; + if (dimList.length === 0 && measureList.length >= 3) { + score += rule1Score; + scoreDetails.rule1 = rule1Score; + } else { + return { + chartType: ChartType.PIE, + score: 0, + scoreDetails, + originScore: 0, + fullMark: 0 + }; + } + + //数据个数<=20 + const rule2Score = 2.0; + totalScore += rule2Score; + if (measureList.length <= 20) { + score += rule2Score; + scoreDetails.rule2 = rule2Score; + } + + //维度最小值/维度最大值>0.1 + const rule3Score = 3.0; + totalScore += rule3Score; + measureList.map(measure => measure.min); + const minMeasure = Math.min(...measureList.map(measure => measure.min)); + const maxMeasure = Math.max(...measureList.map(measure => measure.max)); + if (minMeasure / maxMeasure > 0.1) { + score += rule3Score; + scoreDetails.rule3 = rule3Score; + } + + //变异系数>x (需要调整) + const rule4Score = 1.0; + totalScore += rule4Score; + //当维度数=0时，计算指标的变异系数 + if (dimList.length === 0) { + const tempDataset: MeasureDataset = { + data: measureList.reduce((prev, cur) => prev.concat(cur.data), []) + }; + const coefficient = dataUtils.calCoefficient(tempDataset); + if (!coefficient || coefficient > 0.2) { + score += rule4Score; + scoreDetails.rule4 = rule4Score; + } + } + + //得分归一化用于比较 + + return { + chartType: ChartType.PIE, + score: score / totalScore, + originScore: score, + fullMark: totalScore, + scoreDetails, + cell: pieCell, + dataset: pieDataset + }; + }; + + const calMeasureCard = (): ScoreResult => { + let score = 0; + let totalScore = 0; + const scoreDetails: any = {}; + + //1<=指标数<=3，维度数=0 + const rule1Score = 2.0; + totalScore += rule1Score; + if (dimList.length === 0 && measureList.length <= 3 && measureList.length >= 1) { + score += rule1Score; + scoreDetails.rule1 = rule1Score; + } else { + return { + chartType: ChartType.MEASURE_CARD, + score: 0, + scoreDetails, + originScore: 0, + fullMark: 0 + }; + } + + const cardData = assignMeasureCard(datasetWithoutFold, dimensionID, measureID, aliasMap); + + const { cardCell, dataset: cardDataset } = cardData; + + return { + chartType: ChartType.MEASURE_CARD, + score: score / totalScore, + originScore: score, + fullMark: totalScore, + scoreDetails, + cell: cardCell, + dataset: cardDataset + }; + }; + + const calRadar = (): ScoreResult => { + let score = 0; + let totalScore = 0; + const scoreDetails: any = {}; + if (error) { + return { + chartType: ChartType.RADAR, + score: 0, + scoreDetails, + originScore: 0, + fullMark: 0, + error: error ? errMsg : null + }; + } + + //1<=维度数<=2，指标数=1，或者维度数=1，指标数>=1 + const rule1Score = 1.0; + totalScore += rule1Score; + if (cell.x.length > 0 && cell.y.length > 0) { + score += rule1Score; + scoreDetails.rule1 = rule1Score; + } else { + return { + chartType: ChartType.RADAR, + score: 0, + scoreDetails, + originScore: 0, + fullMark: 0 + }; + } + + //用户目的=分布 + const rule2Score = 5.0; + totalScore += rule2Score; + if (purpose === UserPurpose.DISTRIBUTION) { + score += rule2Score; + scoreDetails.rule2 = rule2Score; + } + + return { + chartType: ChartType.RADAR, + score: score / totalScore, + fullMark: totalScore, + originScore: score, + scoreDetails, + cell, + dataset + }; + }; + + const calWordCloud = (): ScoreResult => { + let score = 0; + let totalScore = 0; + const scoreDetails: any = {}; + + const rule1Score = 1.0; + totalScore += rule1Score; + if (dimList.length === 1 && measureList.length == 0) { + score += rule1Score; + scoreDetails.rule1 = rule1Score; + } else { + return { + chartType: ChartType.WORD_CLOUD, + score: 0, + scoreDetails, + originScore: 0, + fullMark: 0 + }; + } + + //数据基数>=10且<=100 + const rule2Score = 2.0; + totalScore += rule2Score; + if (dimList[0].cardinal >= 20 && dimList[0].cardinal <= 100) { + score += rule2Score; + scoreDetails.rule2 = rule2Score; + } + + //用户目的=storyTelling + const rule3Score = 5.0; + totalScore += rule3Score; + if (purpose === UserPurpose.STORYTELLING) { + score += rule3Score; + scoreDetails.rule3 = rule3Score; + } + + const wordCloudCell: AutoChartCell = { + x: [], + y: [], + row: [], + column: [], + color: [], + size: [], + angle: [] + }; + + wordCloudCell.color.push(dimList[0].uniqueID); + + if (measureList.length > 0) { + wordCloudCell.size.push(measureList[0].uniqueID); + } + + return { + chartType: ChartType.WORD_CLOUD, + score: score / totalScore, + originScore: score, + fullMark: totalScore, + scoreDetails, + cell: wordCloudCell, + dataset + }; + }; + + const calFunnelChart = (): ScoreResult => { + let score = 0; + let totalScore = 0; + const scoreDetails: any = {}; + + //维度数=1，指标数=1，或者维度数=0，指标数>=2， + const rule1Score = 1.0; + totalScore += rule1Score; + if ((dimList.length === 1 && measureList.length === 1) || (dimList.length === 0 && measureList.length >= 2)) { + score += rule1Score; + scoreDetails.rule1 = rule1Score; + } else { + return { + chartType: ChartType.FUNNEL, + score: 0, + scoreDetails, + originScore: 0, + fullMark: 0 + }; + } + + //用户目的=Trend + const rule2Score = 5.0; + totalScore += rule2Score; + if (purpose === UserPurpose.TREND) { + score += rule2Score; + scoreDetails.rule2 = rule2Score; + } + + const { funnelCell, dataset: funnelDataset } = assignFunnelChart(originDataset, dimensionID, measureID, aliasMap); + + return { + chartType: ChartType.FUNNEL, + score: score / totalScore, + originScore: score, + fullMark: totalScore, + scoreDetails, + cell: funnelCell, + dataset: funnelDataset + }; + }; + + const calDualAxis = (): ScoreResult => { + //多度量且度量之间差异过大，用双轴图 + + let score = 0; + let totalScore = 0; + const scoreDetails: any = {}; + + if (!(measureList.length === 2 && dimList.length > 0)) { + return { + chartType: ChartType.DUAL_AXIS, + score: 0, + scoreDetails, + originScore: 0, + fullMark: 0 + }; + } + + //计算cells + const dualAxisData = assignDualAxis(datasetWithoutFold, dimensionID, [measureID[0]], [measureID[1]], aliasMap); + + const { error: newError, errorMsg, dataset, cell: newCell, colorItems, aliasMap: newAliasMap } = dualAxisData; + + if (newError) { + return { + chartType: ChartType.DUAL_AXIS, + score: 0, + scoreDetails, + originScore: 0, + fullMark: 0, + error: error ? errMsg : null + }; + } + + //根据cell决定参与图表推荐的字段 + const measureLength = measureList.length; + + //指标数=2 + const rule1Score = 1.0; + totalScore += rule1Score; + if (measureLength === 2 && newCell.x.length > 0) { + score += rule1Score; + scoreDetails.rule1 = rule1Score; + } else { + return { + chartType: ChartType.DUAL_AXIS, + score: 0, + scoreDetails, + originScore: 0, + fullMark: 0 + }; + } + + //指标最大值里面最大的/指标Q1(下四分位数)里面最小的>100（不同指标数值不在同一个量级） + const rule2Score = 1.0; + totalScore += rule2Score; + let minQ1 = Math.min(...measureList.map(measure => Math.abs(measure.Q1))); + let maxMax = Math.max(...measureList.map(measure => Math.abs(measure.max))); + + if (minQ1 === 0) { + minQ1 = 1; + } + if (maxMax === 0) { + maxMax = 1; + } + + if (maxMax / minQ1 > 100) { + score += rule2Score; + scoreDetails.rule2 = rule2Score; + } + + /* //同一个指标里面最小bar的数值/最大bar的数值>1% + const rule3Score = 3.0 + totalScore += rule3Score + const score3Flag = measureList.reduce((prev, cur) => { + if (prev) { + return cur.min / cur.max > 0.01 + } + return false + }, true) + if (score3Flag) { + score += rule3Score + scoreDetails .rule3 = rule3Score + + } */ + + //2<=图元数量<=20 (根据屏幕尺寸调整) + const rule4Score = 1.0; + totalScore += rule4Score; + + const colorSize = newCell.color.length > 0 ? dataUtils.unique(dataset.map(data => data[COLOR_FIELD])).length : 1; + const axisSize = dataUtils.unique(uniqueIdMap[newCell.x[0]].data).length; + + const dataSize = axisSize; + if (dataSize <= MAX_BAR_NUMBER && dataSize >= MIN_BAR_NUMBER) { + score += rule4Score; + scoreDetails.rule4 = rule4Score; + } + + return { + chartType: ChartType.DUAL_AXIS, + score: score / totalScore, + originScore: score, + fullMark: totalScore, + scoreDetails, + cell: newCell, + dataset + }; + }; + + const calTable = (): ScoreResult => { + //图例数量过多时，推荐表格 + let score = 0; + let totalScore = 0; + const scoreDetails: any = {}; + + const rule1Score = 2.0; + totalScore += rule1Score; + + const colorSize = cell.color.length > 0 ? dataUtils.unique(dataset.map(data => data[COLOR_FIELD])).length : 1; + + let axisSize; + if (cell.x.length > 0) { + axisSize = dataUtils.unique(uniqueIdMap[cell.x[0]].data).length; + } else { + axisSize = 1; + } + + const dataSize = colorSize * axisSize; + + if (dataSize >= 100) { + score += rule1Score; + scoreDetails.rule4 = rule1Score; + } + + return { + chartType: ChartType.TABLE, + score: score / totalScore, + originScore: score, + fullMark: totalScore, + scoreDetails, + cell, + dataset + }; + }; + + const scoreCalculators = [ + calBar, + calBarPercent, + calBarParallel, + calCombination, + calScatterplot, + calLineChart, + // calLineChartCombine, + calPieChart, + calMeasureCard, + calRadar, + calWordCloud, + calFunnelChart, + calDualAxis, + calTable + ]; + + return scoreCalculators; +}; diff --git a/packages/chart-advisor/src/tests/Q1.test.ts b/packages/chart-advisor/src/tests/Q1.test.ts new file mode 100644 index 00000000..4610bc8f --- /dev/null +++ b/packages/chart-advisor/src/tests/Q1.test.ts @@ -0,0 +1,12 @@ +import { calQuantile } from '../dataUtil'; + +test('should be true', () => { + const mock1 = [0, 679042241.42, 1084634383.2, 485539225.08]; + + const Q11 = calQuantile({ data: mock1 }, 0.25); + expect(Q11).toBe(582290733.25); + + const mock2 = [null, 0.02, 0.14, 0.02]; + const Q12 = calQuantile({ data: mock2 }, 0.25); + expect(Q12).toBe(0.02); +}); diff --git a/packages/chart-advisor/src/tests/base.test.ts b/packages/chart-advisor/src/tests/base.test.ts new file mode 100644 index 00000000..f5ec372d --- /dev/null +++ b/packages/chart-advisor/src/tests/base.test.ts @@ -0,0 +1,150 @@ +import { chartAdvisor } from '../index'; +import { AdviserParams, DimensionField, AdviseResult, ChartType } from '../type'; + +const mockDataset = [ + { + '231026105743048': '5734340.8279953', + '231026105743063': '家具' + }, + { + '231026105743048': '4865589.799788475', + '231026105743063': '办公用品' + }, + { + '231026105743048': '5469023.505149841', + '231026105743063': '技术' + } +]; + +const dimensionList: DimensionField[] = [ + { + uniqueId: 231026105743063, + type: 'string' + } +]; + +const measureList: DimensionField[] = [ + { + uniqueId: 231026105743048, + type: 'number' + } +]; + +const aliasMap = { + '231026105743063': '类别', + '231026105743048': '销售额' +}; + +const mockParams: AdviserParams = { + originDataset: mockDataset, + dimensionList, + measureList, + aliasMap +}; + +function advise() { + const result = chartAdvisor(mockParams); + + console.log(123, 'scores', result); + + const chartType = result.chartType; + + return chartType; +} + +// test('column', () => { +// expect(advise()).toBe(ChartType.COLUMN_PARALLEL); +// }); + +const mockDimList2 = [ + { uniqueId: 1699615168282, type: 'date' }, + { uniqueId: 1699615168283, type: 'string' } +]; +const mockMeaList2 = [{ uniqueId: 1699615168284 }, { uniqueId: 1699615168285 }]; +const mockDataset2 = [ + { '1699615168282': '2022-10-01', '1699615168283': '公司', '1699615168284': '111', '1699615168285': null }, + { + '1699615168282': '2023-01-01', + '1699615168283': '公司', + '1699615168284': '213', + '1699615168285': '0.918918918918919' + }, + { + '1699615168282': '2023-04-01', + '1699615168283': '公司', + '1699615168284': '292', + '1699615168285': '0.37089201877934275' + }, + { + '1699615168282': '2023-07-01', + '1699615168283': '公司', + '1699615168284': '257', + '1699615168285': '-0.11986301369863013' + }, + { + '1699615168282': '2023-10-01', + '1699615168283': '公司', + '1699615168284': '106', + '1699615168285': '-0.5875486381322957' + }, + { '1699615168282': '2022-10-01', '1699615168283': '小型企业', '1699615168284': '93', '1699615168285': null }, + { + '1699615168282': '2023-01-01', + '1699615168283': '小型企业', + '1699615168284': '78', + '1699615168285': '-0.16129032258064516' + }, + { + '1699615168282': '2023-04-01', + '1699615168283': '小型企业', + '1699615168284': '150', + '1699615168285': '0.9230769230769231' + }, + { '1699615168282': '2023-07-01', '1699615168283': '小型企业', '1699615168284': '183', '1699615168285': '0.22' }, + { + '1699615168282': '2023-10-01', + '1699615168283': '小型企业', + '1699615168284': '91', + '1699615168285': '-0.5027322404371585' + }, + { '1699615168282': '2022-10-01', '1699615168283': '消费者', '1699615168284': '196', '1699615168285': null }, + { + '1699615168282': '2023-01-01', + '1699615168283': '消费者', + '1699615168284': '299', + '1699615168285': '0.5255102040816326' + }, + { + '1699615168282': '2023-04-01', + '1699615168283': '消费者', + '1699615168284': '485', + '1699615168285': '0.6220735785953178' + }, + { + '1699615168282': '2023-07-01', + '1699615168283': '消费者', + '1699615168284': '451', + '1699615168285': '-0.07010309278350516' + }, + { + '1699615168282': '2023-10-01', + '1699615168283': '消费者', + '1699615168284': '247', + '1699615168285': '-0.4523281596452328' + } +]; + +test('combination', () => { + function advise() { + const result = chartAdvisor({ + dimensionList: mockDimList2 as any, + measureList: mockMeaList2, + originDataset: mockDataset2 + }); + + const chartType = result.chartType; + + return chartType; + } + expect(advise()).toBe(ChartType.COLUMN_PARALLEL); +}); diff --git a/packages/chart-advisor/src/type.ts b/packages/chart-advisor/src/type.ts new file mode 100644 index 00000000..841e142e --- /dev/null +++ b/packages/chart-advisor/src/type.ts @@ -0,0 +1,231 @@ +//屏幕尺寸：小、中、大 +export enum ScreenSize { + LARGE = 0, + MEDIUM = 1, + SMALL = 2 +} + +//用户目的：对比、趋势、分布、排名、占比、组成、StoryTelling +export enum UserPurpose { + NONE = 0, //未指定目的 + COMPARISON = 1, + TREND = 2, + DISTRIBUTION = 3, + RANK = 4, + PROPORTION = 5, + COMPOSITION = 6, + STORYTELLING = 7 +} + +export interface DimensionField { + uniqueId: number; //该字段的id + type: DataTypeName; //该字段的类型 + isGeoField?: boolean; +} + +export interface MeasureField { + uniqueId: number; +} + +//measure数据集 +export interface MeasureDataset { + uniqueID?: number; + data: number[]; + min?: number; + max?: number; + mean?: number; //平均值 + standardDev?: number; //标准差 + coefficient?: number; //变异系数 + Q1?: number; //下四分位数 +} + +//dimension数据集 +export interface DimensionDataset { + uniqueID?: number; + data: string[]; + dataType?: DataTypeName; + dimensionName?: string; //字段名 + cardinal?: number; //基数(不同值的数量) + ratio?: number; //基数除以数据条数 + isGeoField?: boolean; // 是否为地理字段 +} + +export interface AutoChartCell { + x: UniqueId[]; + y: UniqueId[]; + row: UniqueId[]; //作为透视行的字段 + column: UniqueId[]; //作为透视列的字段 + color?: UniqueId[]; + size?: UniqueId[]; + angle?: UniqueId[]; + value?: UniqueId[]; + text?: UniqueId[]; + group?: UniqueId[]; + error?: boolean; + errMsg?: string; + // 维度展开的信息（笛卡尔积） + cartesianInfo?: CartesianInfo; + // 指标展开的信息（指标平坦化） + foldInfo?: FoldInfo; +} + +export interface CartesianInfo { + key: UniqueId; + fieldList: UniqueId[]; +} + +export interface FoldInfo { + key: UniqueId; + value: UniqueId; + foldMap: { + [key: number]: string; + }; +} + +export interface FieldTypeMap { + [key: number]: DataTypeName; +} + +export type DataItem = { [key: number]: string }; + +export type Dataset = DataItem[]; + +export type Datasets = DataItem[][][][] | Dataset; + +export type UniqueId = number | string; + +export type DataTypeName = 'number' | 'string' | 'date'; + +/** + * vqs 接口中 visData 中的 aliasMap + * 做字段名字映射 + */ +export type AliasMap = { + [key: number]: string; +}; + +/** + * 图表类型枚举 + */ +export enum ChartType { + /** 表格 */ + TABLE = 'table', + /** 明细表 */ + RAW_TABLE = 'raw_table', + /** 透视表 */ + PIVOT_TABLE = 'pivot_table', + + /** 柱状图 */ + COLUMN = 'column', + /** 百分比柱状图 */ + COLUMN_PERCENT = 'column_percent', + /** 并列柱状图 */ + COLUMN_PARALLEL = 'column_parallel', + /** 条形图 */ + BAR = 'bar', + /** 百分比条形图 */ + BAR_PERCENT = 'bar_percent', + /** 并列条形图 */ + BAR_PARALLEL = 'bar_parallel', + + /** 折线图 */ + LINE = 'line', + /** 面积图 */ + AREA = 'area', + /** 百分比面积图 */ + AREA_PERCENT = 'area_percent', + + /** 饼图 */ + PIE = 'pie', + /** 环形饼图 */ + ANNULAR = 'annular', + /** 南丁格尔玫瑰图 */ + ROSE = 'rose', + + /** 散点图 */ + SCATTER = 'scatter', + /** 圆视图 */ + CIRCLE_VIEWS = 'circle_views', + + /** 双轴图 */ + DUAL_AXIS = 'double_axis', + /** 双向条形图 */ + BILATERAL = 'bilateral', + /** 组合图 */ + COMBINATION = 'combination', + + /** 填充地图 */ + MAP = 'map', + /** 标记地图 */ + SCATTER_MAP = 'scatter_map', + + /** 指标卡 */ + MEASURE_CARD = 'measure_card', + /** 对比指标卡 */ + COMPARATIVE_MEASURE_CARD = 'comparative_measure_card', + /** 词云 */ + WORD_CLOUD = 'word_cloud', + /** 直方图 */ + HISTOGRAM = 'histogram', + /** 漏斗图 */ + FUNNEL = 'funnel', + /** 雷达图 */ + RADAR = 'radar', + /** 桑基图 */ + SANKEY = 'sankey', + + /** 扩展自定义类型 */ + EXTEND = 'extend', + /** 单值百分比环形图 */ + PROGRESS = 'progress' +} + +export interface ScoreResult { + chartType: ChartType; + originScore: number; + fullMark: number; + score: number; + scoreDetails: Array<{ name: string; score: number }>; + cell?: AutoChartCell | AutoChartCell[]; + dataset?: Datasets; + error?: any; +} + +export interface PivotTree { + field: UniqueId; + values: { + value: string; + child: PivotTree | null; + }[]; +} + +export interface AdviseResult { + chartType: ChartType; //vizData中的chartType + scores: ScoreResult[]; + error?: any; +} + +interface ScorerParams { + inputDataSet: Dataset; + dimList: DimensionDataset[]; + measureList: MeasureDataset[]; + aliasMap?: AliasMap; + maxRowNum?: number; + maxColNum?: number; + purpose?: UserPurpose; + screen?: ScreenSize; +} + +export type Scorer = (params: ScorerParams) => Array<() => ScoreResult>; + +export interface AdviserParams { + originDataset: Dataset; + dimensionList: DimensionField[]; + measureList: MeasureField[]; + aliasMap?: AliasMap; + maxPivotRow?: number; + maxPivotColumn?: number; + purpose?: UserPurpose; + screen?: ScreenSize; + scorer?: Scorer; +} diff --git a/packages/chart-advisor/tsconfig.cjs.json b/packages/chart-advisor/tsconfig.cjs.json new file mode 100644 index 00000000..9576fc69 --- /dev/null +++ b/packages/chart-advisor/tsconfig.cjs.json @@ -0,0 +1,21 @@ +/** + * NOTE: this file is symlink to '@aeolian/dev-config/tsconfig/build.cjs.json' + */ +{ + "extends": "./tsconfig.json", + "compilerOptions": { + "outDir": "lib", + "target": "es2018", + "module": "CommonJS", + "noEmit": false, + "sourceMap": false, + }, + "exclude": [ + "**/*.test.ts", + "**/*.test.tsx", + "**/*.stories.tsx", + "**/*.mock.ts", + "**/tests", + "**/mocks", + ], +} diff --git a/packages/chart-advisor/tsconfig.eslint.json b/packages/chart-advisor/tsconfig.eslint.json new file mode 100644 index 00000000..c6dbef60 --- /dev/null +++ b/packages/chart-advisor/tsconfig.eslint.json @@ -0,0 +1,10 @@ +{ + "extends": "@internal/ts-config/tsconfig.base.json", + "compilerOptions": { + "types": ["jest", "node"], + "lib": ["DOM", "ESNext"], + "baseUrl": "./", + "rootDir": "./" + }, + "include": ["src", "__tests__"] +} diff --git a/packages/chart-advisor/tsconfig.esm.json b/packages/chart-advisor/tsconfig.esm.json new file mode 100644 index 00000000..0bfc1693 --- /dev/null +++ b/packages/chart-advisor/tsconfig.esm.json @@ -0,0 +1,22 @@ +/** + * NOTE: this file is symlink to '@aeolian/dev-config/tsconfig/build.esm.json' + */ +{ + "extends": "./tsconfig.json", + "compilerOptions": { + "outDir": "es", + "target": "es2018", + "module": "ESNext", + "noEmit": false, + "sourceMap": false, + "noImplicitAny": false + }, + "exclude": [ + "**/*.test.ts", + "**/*.test.tsx", + "**/*.stories.tsx", + "**/*.mock.ts", + "**/tests", + "**/mocks", + ], +} diff --git a/packages/chart-advisor/tsconfig.json b/packages/chart-advisor/tsconfig.json new file mode 100644 index 00000000..c564f002 --- /dev/null +++ b/packages/chart-advisor/tsconfig.json @@ -0,0 +1,21 @@ +{ + "extends": "@internal/ts-config/tsconfig.base.json", + "compilerOptions": { + "declaration": true, //Generates corresponding d.ts files. + "composite": true, + "baseUrl": ".", + "rootDir": "src", + "outDir": "es", + "types": [ + "node", + "jest", + ], + "noImplicitAny": false + }, + "include": [ + "src", + ], + // https://www.typescriptlang.org/docs/handbook/project-references.html#what-is-a-project-reference + "references": [ + ], +} diff --git a/packages/chart-advisor/tsconfig.test.json b/packages/chart-advisor/tsconfig.test.json new file mode 100644 index 00000000..e68bcbbf --- /dev/null +++ b/packages/chart-advisor/tsconfig.test.json @@ -0,0 +1,8 @@ +{ + "extends": "./tsconfig.json", + "compilerOptions": { + "paths": { + } + }, + "references": [] +} diff --git a/packages/vmind/CHANGELOG.json b/packages/vmind/CHANGELOG.json index 0e55a3ae..a131a209 100644 --- a/packages/vmind/CHANGELOG.json +++ b/packages/vmind/CHANGELOG.json @@ -1,10 +1,16 @@ { "name": "@visactor/vmind", "entries": [ + { + "version": "1.2.5", + "tag": "@visactor/vmind_v1.2.5", + "date": "Tue, 26 Mar 2024 07:36:00 GMT", + "comments": {} + }, { "version": "1.2.4", "tag": "@visactor/vmind_v1.2.4", - "date": "Wed, 21 Feb 2024 12:04:49 GMT", + "date": "Tue, 26 Mar 2024 07:35:59 GMT", "comments": {} }, { diff --git a/packages/vmind/CHANGELOG.md b/packages/vmind/CHANGELOG.md index e9ae5e06..19f0c006 100644 --- a/packages/vmind/CHANGELOG.md +++ b/packages/vmind/CHANGELOG.md @@ -1,9 +1,14 @@ # Change Log - @visactor/vmind -This log was last generated on Wed, 21 Feb 2024 12:04:49 GMT and should not be manually modified. +This log was last generated on Tue, 26 Mar 2024 07:36:00 GMT and should not be manually modified. + +## 1.2.5 +Tue, 26 Mar 2024 07:36:00 GMT + +_Version update only_ ## 1.2.4 -Wed, 21 Feb 2024 12:04:49 GMT +Tue, 26 Mar 2024 07:35:59 GMT _Version update only_ diff --git a/packages/vmind/__tests__/browser/src/constants/mockData.ts b/packages/vmind/__tests__/browser/src/constants/mockData.ts index 749d8ca5..f30fe0d7 100644 --- a/packages/vmind/__tests__/browser/src/constants/mockData.ts +++ b/packages/vmind/__tests__/browser/src/constants/mockData.ts @@ -3663,3 +3663,14 @@ East Asia & Pacific,7.8,8.95,10.18,11.57,13.25 Europe & Central Asia,9.52,10.39,10.93,11.69,12.63`, input: '看下各类别数值分布' }; + +/* + * multi-measure + */ +export const mockUserInput17 = { + csv: `date,ctr,libra_ab_vid,gmv,detection_uv,User,user_count,gpm,ctcvr,version_id,co +2024-03-05,0.024752458745543306,-1,27.05042746931187,683740,31135119,683740,0.0028542157856540976,0.00011098365108464276,-1,0.01765408108609234 +2024-03-06,0.025529744509421266,-1,26.0184806658061,666756,30057727,666756,0.0029282343207837846,0.00011602865071429876,-1,0.018457400571758008 +2024-03-07,0.024929771518461413,-1,26.099474885828787,653568,25685979,653568,0.00305250127580487,0.00012037169363684177,-1,0.019630556105233156`, + input: '使用折线图展示' +}; diff --git a/packages/vmind/__tests__/browser/src/pages/ChartPreview.tsx b/packages/vmind/__tests__/browser/src/pages/ChartPreview.tsx index 49b862b4..0e7c03ed 100644 --- a/packages/vmind/__tests__/browser/src/pages/ChartPreview.tsx +++ b/packages/vmind/__tests__/browser/src/pages/ChartPreview.tsx @@ -11,6 +11,7 @@ const TextArea = Input.TextArea; type IPropsType = { spec: any; + specList?: any; time: | { totalTime: number; @@ -121,6 +122,21 @@ export function ChartPreview(props: IPropsType) { cs.renderAsync(); }, [chartSpace, props, props.spec, props.time]); + useEffect(() => { + //defaultTicker.mode = 'raf'; + const { specList } = props; + specList.forEach((spec: any, index: number) => { + (document.getElementById(`chart-${index}`) as any).innerHTML = ''; + const cs = new VChart(spec, { + dom: `chart-${index}`, + mode: 'desktop-browser', + dpr: 2, + disableDirtyBounds: true + }); + cs.renderAsync(); + }); + }, [props]); + return (
+
+ {(props.specList ?? []).map((spec, index: number) => ( +
+ ))} +
{props.spec ? (
-

Total Time: {props.costTime / 1000} ms

+

Total Time: {props.costTime / 1000} s

spec:

{/*
{JSON.stringify(props.spec, null, 4)}
*/} diff --git a/packages/vmind/__tests__/browser/src/pages/DataInput.tsx b/packages/vmind/__tests__/browser/src/pages/DataInput.tsx index 0561ce4f..7b65aef0 100644 --- a/packages/vmind/__tests__/browser/src/pages/DataInput.tsx +++ b/packages/vmind/__tests__/browser/src/pages/DataInput.tsx @@ -33,11 +33,13 @@ import { mockUserInput12, mockUserInput13, mockUserInput14, - mockUserInput16 + mockUserInput16, + mockUserInput17 } from '../constants/mockData'; import VMind from '../../../../src/index'; import { Model } from '../../../../src/index'; -import { queryDataset } from '../../../../src/gpt/dataProcess'; +import { isArray } from 'lodash'; +import { mockDataset, mockData2, mockData3, mockData4 } from './mockData'; const TextArea = Input.TextArea; const Option = Select.Option; @@ -52,6 +54,7 @@ type IPropsType = { }, costTime: number ) => void; + onSpecListGenerate: any; }; const demoDataList: { [key: string]: any } = { pie: mockUserInput2, @@ -73,11 +76,12 @@ const demoDataList: { [key: string]: any } = { 'College entrance examination': acceptRatioData, 'Shopping Mall Sales Performance': mallSalesData, 'Global GDP': mockUserInput6Eng, - 'Sales of different drinkings': mockUserInput3Eng + 'Sales of different drinkings': mockUserInput3Eng, + 'Multi measure': mockUserInput17 }; const globalVariables = (import.meta as any).env; -const ModelConfigMap = { +const ModelConfigMap: any = { [Model.SKYLARK2]: { url: globalVariables.VITE_SKYLARK_URL, key: globalVariables.VITE_SKYLARK_KEY }, [Model.SKYLARK]: { url: globalVariables.VITE_SKYLARK_URL, key: globalVariables.VITE_SKYLARK_KEY }, [Model.GPT3_5]: { url: globalVariables.VITE_GPT_URL, key: globalVariables.VITE_GPT_KEY }, @@ -92,11 +96,11 @@ export function DataInput(props: IPropsType) { const [spec, setSpec] = useState(''); const [time, setTime] = useState(1000); const [model, setModel] = useState(Model.GPT3_5); - const [cache, setCache] = useState(false); + const [cache, setCache] = useState(true); const [showThoughts, setShowThoughts] = useState(false); const [visible, setVisible] = React.useState(false); - const [url, setUrl] = React.useState(ModelConfigMap[model].url ?? OPENAI_API_URL); - const [apiKey, setApiKey] = React.useState(ModelConfigMap[model].key); + const [url, setUrl] = React.useState(ModelConfigMap[model]?.url ?? OPENAI_API_URL); + const [apiKey, setApiKey] = React.useState(ModelConfigMap[model]?.key); const [loading, setLoading] = useState(false); @@ -120,12 +124,21 @@ export function DataInput(props: IPropsType) { //setLoading(true); const { fieldInfo, dataset } = vmind.parseCSVData(csv); //const { fieldInfo: fieldInfoQuery, dataset: datasetQuery } = await vmind?.dataQuery(describe, fieldInfo, dataset); - //const { fieldInfo, dataset } = await vmind.parseCSVDataWithLLM(csv, describe); + //const { fieldInfo, dataset, usage } = await vmind.parseCSVDataWithLLM(csv, describe); + + //const dataset = mockData4; + //const fieldInfo = vmind?.getFieldInfo(dataset); const startTime = new Date().getTime(); - const { spec, time } = await vmind.generateChart(describe, fieldInfo, dataset); + const chartGenerationRes = await vmind.generateChart(describe, fieldInfo, dataset, true); const endTime = new Date().getTime(); - const costTime = endTime - startTime; - props.onSpecGenerate(spec, time as any, costTime); + if (isArray(chartGenerationRes)) { + props.onSpecListGenerate(chartGenerationRes.map(res => res.spec)); + } else { + const { spec, time } = chartGenerationRes; + const costTime = endTime - startTime; + props.onSpecGenerate(spec, time as any, costTime); + } + setLoading(false); }, [vmind, csv, describe, props]); @@ -257,6 +270,7 @@ export function DataInput(props: IPropsType) { skylark2 pro skylark pro + chart-advisor
diff --git a/packages/vmind/__tests__/browser/src/pages/Home.tsx b/packages/vmind/__tests__/browser/src/pages/Home.tsx index 9d1b4166..8011a066 100644 --- a/packages/vmind/__tests__/browser/src/pages/Home.tsx +++ b/packages/vmind/__tests__/browser/src/pages/Home.tsx @@ -7,6 +7,8 @@ const Content = Layout.Content; export function Home() { const [spec, setSpec] = useState(''); + const [specList, setSpecList] = useState([]); + const [time, setTime] = useState<{ totalTime: number; frameArr: any[]; @@ -26,10 +28,15 @@ export function Home() { setTime(time); setCostTime(costTime); }} + onSpecListGenerate={(specList: any[], time: any, costTime: number) => { + setSpecList(specList); + setTime(time); + setCostTime(costTime); + }} /> - + ); diff --git a/packages/vmind/__tests__/browser/vite.config.ts b/packages/vmind/__tests__/browser/vite.config.ts index 59c021d9..3f751fe1 100644 --- a/packages/vmind/__tests__/browser/vite.config.ts +++ b/packages/vmind/__tests__/browser/vite.config.ts @@ -39,7 +39,8 @@ export default defineConfig(({ mode }) => { }, resolve: { alias: { - '@visactor/calculator': path.resolve(__dirname, '../../../calculator/src/index.ts') + '@visactor/calculator': path.resolve(__dirname, '../../../calculator/src/index.ts'), + '@visactor/chart-advisor': path.resolve(__dirname, '../../../chart-advisor/src/index.ts') // ...localConf.resolve?.alias } }, diff --git a/packages/vmind/__tests__/performance/performanceTest.ts b/packages/vmind/__tests__/performance/performanceTest.ts index f5df7b32..31a12266 100644 --- a/packages/vmind/__tests__/performance/performanceTest.ts +++ b/packages/vmind/__tests__/performance/performanceTest.ts @@ -22,8 +22,13 @@ import { mockUserInput12, mockUserInput13, mockUserInput14, - mockUserInput16 + mockUserInput16, + mockUserInput17 } from '../browser/src/constants/mockData'; + +const TEST_GPT = false; +const TEST_SKYLARK = true; + const demoDataList: { [key: string]: any } = { pie: mockUserInput2, 'dynamic bar zh_cn': mockUserInput6, @@ -44,7 +49,8 @@ const demoDataList: { [key: string]: any } = { 'College entrance examination': acceptRatioData, 'Shopping Mall Sales Performance': mallSalesData, 'Global GDP': mockUserInput6Eng, - 'Sales of different drinkings': mockUserInput3Eng + 'Sales of different drinkings': mockUserInput3Eng, + 'Multi measure': mockUserInput17 }; const CHART_GENERATION_AVERAGE_TIME = 10000; const QPM_LIMIT = 10; //qpm limit of your llm service @@ -53,7 +59,6 @@ const START_INDEX = 0; const modelResultMap = { [Model.GPT3_5]: { totalCount: 0, successCount: 0, totalTime: 0 }, - [Model.SKYLARK]: { totalCount: 0, successCount: 0, totalTime: 0 }, [Model.SKYLARK2]: { totalCount: 0, successCount: 0, totalTime: 0 } }; @@ -88,11 +93,12 @@ const dataList = Object.keys(demoDataList); const gptKey = process.env.VITE_GPT_KEY; const gptURL = process.env.VITE_GPT_JEST_URL; -if (gptKey && gptURL) { +if (gptKey && gptURL && TEST_GPT) { const vmind = new VMind({ url: gptURL, model: Model.GPT3_5, - cache: false, + cache: true, + showThoughts: false, headers: { 'api-key': gptKey } @@ -100,36 +106,21 @@ if (gptKey && gptURL) { testPerformance(Model.GPT3_5, vmind); } -//const skylarkKey = process.env.VITE_SKYLARK_KEY; -//const skylarkURL = process.env.VITE_SKYLARK_JEST_URL; - -//if (skylarkKey && skylarkURL) { -// const vmind = new VMind({ -// url: skylarkURL, -// model: Model.SKYLARK, -// cache: false, -// headers: { -// 'api-key': skylarkKey -// } -// }); -// testPerformance(Model.SKYLARK, vmind); -//} - const skylark2Key = process.env.VITE_SKYLARK_KEY; const skylark2URL = process.env.VITE_SKYLARK_JEST_URL; -//if (skylark2Key && skylark2URL) { -// const vmind = new VMind({ -// url: skylark2URL, -// model: Model.SKYLARK2, -// cache: false, -// showThoughts: false, -// headers: { -// 'api-key': skylark2Key -// } -// }); -// testPerformance(Model.SKYLARK2, vmind); -//} +if (skylark2Key && skylark2URL && TEST_SKYLARK) { + const vmind = new VMind({ + url: skylark2URL, + model: Model.SKYLARK2, + cache: true, + showThoughts: false, + headers: { + 'api-key': skylark2Key + } + }); + testPerformance(Model.SKYLARK2, vmind); +} afterAll(() => { log('---------------VMind performance test---------------'); diff --git a/packages/vmind/jest.config.js b/packages/vmind/jest.config.js index b9b8e822..b1eecda2 100644 --- a/packages/vmind/jest.config.js +++ b/packages/vmind/jest.config.js @@ -1,6 +1,3 @@ -// eslint-disable-next-line @typescript-eslint/no-var-requires -const path = require('path'); - module.exports = { preset: 'ts-jest', runner: 'jest-electron/runner', diff --git a/packages/vmind/package.json b/packages/vmind/package.json index 978456f0..b06f0060 100644 --- a/packages/vmind/package.json +++ b/packages/vmind/package.json @@ -1,10 +1,9 @@ { "name": "@visactor/vmind", - "version": "1.2.4", + "version": "1.2.5", "main": "cjs/index.js", "module": "esm/index.js", "types": "esm/index.d.ts", - "author": "chengda ", "license": "MIT", "files": [ "cjs", @@ -78,7 +77,7 @@ "react": "^18.0.0", "react-dom": "^18.0.0", "react-router-dom": "6.9.0", - "@visactor/vchart": "^1.9.0", + "@visactor/vchart": "^1.10.2", "@rollup/plugin-dynamic-import-vars": "~2.1.0", "@types/lodash": "4.14.182", "@types/node": "*", @@ -88,7 +87,7 @@ "canvas": "^2.11.2" }, "dependencies": { - "@visactor/chart-advisor": "0.1.10", + "@visactor/chart-advisor": "workspace:*", "@visactor/vdataset": "~0.17.4", "@visactor/vutils": "~0.17.4", "axios": "^1.4.0", @@ -98,6 +97,7 @@ "dayjs": "~1.11.10", "js-yaml": "~4.1.0", "node-sql-parser": "~4.17.0", - "@visactor/calculator": "workspace:*" + "@visactor/calculator": "workspace:*", + "alasql": "~4.3.2" } } diff --git a/packages/vmind/src/common/chartAdvisor/chartAdvisorHandler.ts b/packages/vmind/src/common/chartAdvisor/chartAdvisorHandler.ts index a2e92597..e6fea257 100644 --- a/packages/vmind/src/common/chartAdvisor/chartAdvisorHandler.ts +++ b/packages/vmind/src/common/chartAdvisor/chartAdvisorHandler.ts @@ -1,7 +1,28 @@ import { chartAdvisor, DataTypeName, ChartType } from '@visactor/chart-advisor'; -import { Cell, VizSchema } from '../../typings'; +import { Cell, DataItem, SimpleFieldInfo, VizSchema } from '../../typings'; +import { getSchemaFromFieldInfo } from '../schema'; -export const chartAdvisorHandler = (schema: Partial, dataset: any[]) => { +export const getAdvisedChartsWithDataset = (fieldInfo: SimpleFieldInfo[], dataset: DataItem[]) => { + const schema = getSchemaFromFieldInfo(fieldInfo); + const { scores } = getAdvisedChartList(schema, dataset); + return scores + .filter((d: any) => availableChartTypeList.includes(d.chartType) && d.score - 0 >= 0.00000001) + .map((result: any) => ({ + chartType: chartTypeMap(result.chartType).toUpperCase(), + cell: getCell(result.cell), + dataset: result.dataset, + score: result.score + })); +}; + +/** + * call @visactor/chart-advisor to get the list of advised charts + * sorted by scores of each chart type + * @param schema + * @param dataset + * @returns + */ +const getAdvisedChartList = (schema: Partial, dataset: any[]) => { const dimensionList: any = schema.fields .filter(d => d.role === 'dimension') .map(d => ({ @@ -16,7 +37,17 @@ export const chartAdvisorHandler = (schema: Partial, dataset: any[]) })); const aliasMap = Object.fromEntries(schema.fields.map(d => [d.id, d.alias])); const advisorResult = chartAdvisor({ originDataset: dataset, dimensionList, measureList, aliasMap }); - const result = advisorResult.scores.find(d => availableChartTypeList.includes(d.chartType)); + return advisorResult; +}; +/** + * get one recommended chart type using @visactor/chart-advisor + * @param schema + * @param dataset + * @returns + */ +export const chartAdvisorHandler = (schema: Partial, dataset: any[]) => { + const advisorResult = getAdvisedChartList(schema, dataset); + const result = advisorResult.scores.find((d: any) => availableChartTypeList.includes(d.chartType)); return { chartType: chartTypeMap(result.chartType).toUpperCase(), cell: getCell(result.cell), @@ -52,7 +83,8 @@ const availableChartTypeList = [ ChartType.DUAL_AXIS, ChartType.WORD_CLOUD, ChartType.FUNNEL, - ChartType.SANKEY + ChartType.SANKEY, + ChartType.RADAR ]; const chartTypeMap = (advisorChartType: ChartType) => { @@ -83,6 +115,8 @@ const chartTypeMap = (advisorChartType: ChartType) => { return 'Funnel Chart'; } else if (ChartType.SANKEY === advisorChartType) { return 'Sankey Chart'; + } else if (ChartType.RADAR === advisorChartType) { + return 'Radar Chart'; } throw 'no matched chart type'; }; @@ -93,9 +127,9 @@ const getCell = (cell: any): Cell => { keys.forEach((key: string) => { const channel = cell[key]; if (Array.isArray(channel) && channel.length === 1) { - result[key] = channel[0]; + result[key] = String(channel[0]); } else { - result[key] = channel; + result[key] = Array.isArray(channel) ? channel.map(c => String(c)) : channel; } }); return result; diff --git a/packages/vmind/src/common/chartAdvisor/index.ts b/packages/vmind/src/common/chartAdvisor/index.ts index 98f36f82..7b4747a5 100644 --- a/packages/vmind/src/common/chartAdvisor/index.ts +++ b/packages/vmind/src/common/chartAdvisor/index.ts @@ -1 +1,48 @@ +import { getAdvisedChartsWithDataset } from './chartAdvisorHandler'; +import { vizDataToSpec } from '../vizDataToSpec'; +import { estimateVideoTime } from '../vizDataToSpec/utils'; +import { DataItem, SimpleFieldInfo } from 'src/typings'; +import { uniqBy } from 'lodash'; + export { chartAdvisorHandler } from './chartAdvisorHandler'; + +/** + * + * @param originDataset raw dataset used in the chart + * @param colorPalette color palette of the chart + * @param animationDuration duration of chart animation. + * @returns spec and animation duration of the generated charts + */ +export const generateChartWithAdvisor = ( + fieldInfo: SimpleFieldInfo[], + originDataset: DataItem[], + colorPalette?: string[], + animationDuration?: number +) => { + const advisorRes = getAdvisedChartsWithDataset(fieldInfo, originDataset); + const resultList = uniqBy(advisorRes, 'chartType').map((res: any) => { + const { chartType, cell, dataset, score } = res; + const spec = vizDataToSpec( + dataset, + chartType, + cell, + colorPalette, + animationDuration ? animationDuration * 1000 : undefined + ); + spec.background = '#00000033'; + return { + chartSource: 'chartAdvisor', + spec, + chartType, + score, + usage: { + prompt_tokens: 0, + completion_tokens: 0, + total_tokens: 0 + }, + time: estimateVideoTime(chartType, spec, animationDuration ? animationDuration * 1000 : undefined) + }; + }); + console.info(resultList); + return resultList; +}; diff --git a/packages/vmind/src/common/dataProcess/dataQuery.ts b/packages/vmind/src/common/dataProcess/dataQuery.ts new file mode 100644 index 00000000..091cfcc0 --- /dev/null +++ b/packages/vmind/src/common/dataProcess/dataQuery.ts @@ -0,0 +1,55 @@ +import { DataItem, SimpleFieldInfo } from 'src/typings'; +import { + replaceDataset, + replaceInvalidWords, + swapMap, + replaceBlankSpace, + replaceString, + sumAllMeasureFields +} from './utils'; +import alasql from 'alasql'; + +export const VMIND_DATA_SOURCE = 'VMind_data_source'; + +/** + * SQL query for SourceDatset + * It has nothing to do with the model type model + * @param sql + * @param sourceDataset + * @param fieldInfo + * @returns dataset after query + */ +export const queryDataset = (sql: string, sourceDataset: DataItem[], fieldInfo: SimpleFieldInfo[]) => { + const fieldNames = fieldInfo.map(field => field.fieldName); + const { validStr, sqlReplaceMap, columnReplaceMap } = replaceInvalidWords(sql, fieldNames); + + //replace field names according to replaceMap + const validColumnDataset = replaceDataset(sourceDataset, columnReplaceMap, true); + + //replace field names and data values according to replaceMap + const validDataset = replaceDataset(validColumnDataset, sqlReplaceMap, false); + + //replace blank spaces in column name + const replacedFieldNames = fieldNames + .map(field => replaceString(field, columnReplaceMap)) + .map(field => replaceString(field, sqlReplaceMap)); + const validSql = replaceBlankSpace(validStr, replacedFieldNames as string[]); + + const finalSql = sumAllMeasureFields(validSql, fieldInfo, columnReplaceMap, sqlReplaceMap); + //convertGroupByToString(finalSql, validDataset) + + //replace VMIND_DATA_SOURCE with placeholder "?" + const sqlParts = (finalSql + ' ').split(VMIND_DATA_SOURCE); + const sqlCount = sqlParts.length - 1; + const alasqlQuery = sqlParts.join('?'); + //do the query + const alasqlDataset = alasql(alasqlQuery, new Array(sqlCount).fill(validDataset)); + + //restore the dataset + const columnReversedMap = swapMap(columnReplaceMap); + const columnRestoredDataset = replaceDataset(alasqlDataset, columnReversedMap, true); + const sqlReversedMap = swapMap(sqlReplaceMap); + const sqlRestoredDataset = replaceDataset(columnRestoredDataset, sqlReversedMap, false); + + return sqlRestoredDataset; +}; diff --git a/packages/vmind/src/common/dataProcess/index.ts b/packages/vmind/src/common/dataProcess/index.ts index c6d8d75e..2d31fea4 100644 --- a/packages/vmind/src/common/dataProcess/index.ts +++ b/packages/vmind/src/common/dataProcess/index.ts @@ -1,6 +1,6 @@ import { DataSet, DataView, csvParser, fold } from '@visactor/vdataset'; import { DataItem, DataType, SimpleFieldInfo } from '../../typings'; -import { getFieldInfoFromDataset } from './utils'; +import { getFieldInfo } from './utils'; import { isNil } from 'lodash'; export const parseCSVWithVChart = (csvString: string) => { @@ -26,7 +26,7 @@ export const getDataset = (csvString: string): { dataset: DataItem[]; columns: s /** * convert number string to number in dataset */ -const convertNumberField = (dataset: DataItem[], fieldInfo: SimpleFieldInfo[]) => { +export const convertNumberField = (dataset: DataItem[], fieldInfo: SimpleFieldInfo[]) => { const numberFields = fieldInfo .filter(field => [DataType.INT, DataType.FLOAT].includes(field.type)) .map(field => field.fieldName); @@ -43,7 +43,23 @@ const convertNumberField = (dataset: DataItem[], fieldInfo: SimpleFieldInfo[]) = export const parseCSVData = (csvString: string): { fieldInfo: SimpleFieldInfo[]; dataset: DataItem[] } => { //parse the CSV string to get information about the fields(fieldInfo) and dataset object const { dataset, columns } = getDataset(csvString); - const fieldInfo = getFieldInfoFromDataset(dataset, columns); + const fieldInfo = getFieldInfo(dataset, columns); convertNumberField(dataset, fieldInfo); return { fieldInfo, dataset }; }; + +export const getFieldInfoFromDataset = (dataset: DataItem[]): SimpleFieldInfo[] => { + if (!dataset || !dataset.length) { + return []; + } + const columns = new Set(); + dataset.forEach(data => { + const dataKeys = Object.keys(data); + dataKeys.forEach(column => { + if (!columns.has(column)) { + columns.add(column); + } + }); + }); + return getFieldInfo(dataset, Array.from(columns) as string[]); +}; diff --git a/packages/vmind/src/common/dataProcess/utils.ts b/packages/vmind/src/common/dataProcess/utils.ts index 0d50fe3d..4d276980 100644 --- a/packages/vmind/src/common/dataProcess/utils.ts +++ b/packages/vmind/src/common/dataProcess/utils.ts @@ -1,7 +1,9 @@ -import { sampleSize, isNumber, isInteger } from 'lodash'; +import { sampleSize, isNumber, isInteger, isString, isArray } from 'lodash'; import { DataItem, DataType, ROLE, SimpleFieldInfo } from '../../typings'; import dayjs from 'dayjs'; import { uniqArray } from '@visactor/vutils'; +import alasql from 'alasql'; + export const readTopNLine = (csvFile: string, n: number) => { // get top n lines of a csv file let res = ''; @@ -34,6 +36,14 @@ export function removeEmptyLines(str: string) { return str.replace(/\n\s*\n/g, '\n'); } +export const getFieldDomain = (dataset: DataItem[], column: string, role: ROLE) => { + //calculate domain of the column + const domain: (string | number)[] = dataset.map(d => (role === ROLE.DIMENSION ? d[column] : Number(d[column]))); + return role === ROLE.DIMENSION + ? (uniqArray(domain) as string[]).slice(0, 20) + : [Math.min(...(domain as number[])), Math.max(...(domain as number[]))]; +}; + export const detectFieldType = (dataset: DataItem[], column: string): SimpleFieldInfo => { let fieldType: DataType | undefined = undefined; //detect field type based on rules @@ -95,21 +105,16 @@ export const detectFieldType = (dataset: DataItem[], column: string): SimpleFiel } }); const role = [DataType.STRING, DataType.DATE].includes(fieldType) ? ROLE.DIMENSION : ROLE.MEASURE; - - //calculate domain of the column - const domain: (string | number)[] = dataset.map(d => (role === ROLE.DIMENSION ? d[column] : Number(d[column]))); + const domain = getFieldDomain(dataset, column, role); return { fieldName: column, type: fieldType, role, - domain: - role === ROLE.DIMENSION - ? (uniqArray(domain) as string[]).slice(0, 20) - : [Math.min(...(domain as number[])), Math.max(...(domain as number[]))] + domain }; }; -export const getFieldInfoFromDataset = (dataset: DataItem[], columns: string[]): SimpleFieldInfo[] => { +export const getFieldInfo = (dataset: DataItem[], columns: string[]): SimpleFieldInfo[] => { let sampledDataset = dataset; if (dataset.length > 1000) { //sample the dataset if too large @@ -117,3 +122,305 @@ export const getFieldInfoFromDataset = (dataset: DataItem[], columns: string[]): } return columns.map(column => detectFieldType(sampledDataset, column)); }; + +export function generateRandomString(len: number) { + const chars = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz'; + let result = ''; + for (let i = 0; i < len; i++) { + result += chars.charAt(Math.floor(Math.random() * chars.length)); + } + return result; +} + +export const swapMap = (map: Map) => { + //swap the map + const swappedMap = new Map(); + + // Swap key with value + map.forEach((value, key) => { + swappedMap.set(value, key); + }); + return swappedMap; +}; + +/** + * replace all the non-ascii characters in the sql str into valid strings. + * @param str + * @returns + */ +export const replaceNonASCIICharacters = (str: string) => { + const nonAsciiCharMap = new Map(); + + const newStr = str.replace(/([^\x00-\x7F]+)/g, m => { + let replacement; + if (nonAsciiCharMap.has(m)) { + replacement = nonAsciiCharMap.get(m); + } else { + replacement = generateRandomString(10); + nonAsciiCharMap.set(m, replacement); + } + return replacement; + }); + + //const swappedMap = swapMap(nonAsciiCharMap); + + return { validStr: newStr, replaceMap: nonAsciiCharMap }; +}; + +/** + * replace strings according to replaceMap + * @param str + * @param replaceMap + * @returns + */ +export const replaceString = (str: string | number, replaceMap: Map) => { + if (!isString(str)) { + return str; + } + if (replaceMap.has(str)) { + return replaceMap.get(str); + } else { + //Some string may be linked by ASCII characters as non-ASCII characters.Traversing the replaceMap and replaced it to the original character + const replaceKeys = [...replaceMap.keys()]; + return replaceKeys.reduce((prev, cur) => { + return replaceAll(prev, cur, replaceMap.get(cur)); + }, str); + } +}; + +//replace data keys and data values according to replaceMap +export const replaceDataset = (dataset: DataItem[], replaceMap: Map, keysOnly: boolean) => { + return dataset.map((d: DataItem) => { + const dataKeys = Object.keys(d); + return dataKeys.reduce((prev, cur) => { + const replacedKey = replaceString(cur, replaceMap); + const replacedValue = replaceString(d[cur], replaceMap); + prev[replacedKey] = keysOnly ? d[cur] : replacedValue; + return prev; + }, {}); + }); +}; + +export const getValueByAttributeName = (obj: any, outterKey: string): string[] => { + //get all the attributes of an object by outterKey + const values = []; + for (const key in obj) { + if (key === outterKey && typeof obj[key] === 'string') { + values.push(obj[key]); + } else if (typeof obj[key] === 'object') { + const childValues = getValueByAttributeName(obj[key], outterKey); + values.push(...childValues); + } + } + return uniqArray(values); +}; + +export const replaceInvalidContent = (str: string) => { + const INVALID_CONTENT_LIST = [' ']; + return INVALID_CONTENT_LIST.reduce((prev, cur) => { + return replaceAll(prev, cur, ''); + }, str); +}; + +/** + * replace the string according to replaceMap + * @param str + * @param replaceMap + */ +export const replaceByMap = (str: string, replaceMap: Map) => { + const originalStringList = [...replaceMap.keys()]; + + const finalSql = originalStringList.reduce((prev, cur) => { + const originColumnName = cur; + const validColumnName = replaceMap.get(cur); + return replaceAll(prev, originColumnName, validColumnName); + }, str); + + return finalSql; +}; + +const RESERVE_REPLACE_MAP = new Map([ + ['+', `_${generateRandomString(3)}_PLUS_${generateRandomString(3)}_`], + ['-', `_${generateRandomString(3)}_DASH_${generateRandomString(3)}_`], + ['*', `_${generateRandomString(3)}_ASTERISK_${generateRandomString(3)}_`], + ['/', `_${generateRandomString(3)}_SLASH_${generateRandomString(3)}_`], + ['value', generateRandomString(10)], + ['key', generateRandomString(10)], + ['total', generateRandomString(10)] +]); + +/** + * replace operator and reserved words inside the column name in the sql str + * operators such as +, -, *, / in column names in sql will cause ambiguity and parsing error + * so we need to replace them only in column names + * sometimes skylark2 pro will return a sql statement in which non-ascii characters are not wrapped with `` + * this will cause error in alasql + * so we need to replace them with random string in the whole sql + * @param sql + * @param columns + * @returns validStr: sql without invalid characters; columnReplaceMap: replace map of column names; sqlReplaceMap: replace map of the whole sql including dimension values. + * + */ +export const replaceInvalidWords = (sql: string, columns: string[]) => { + //replace column names according to RESERVED_REPLACE_MAP + const validColumnNames = columns.map(column => { + const nameWithoutOperator = [...RESERVE_REPLACE_MAP.keys()].reduce((prev, cur) => { + return replaceAll(prev, cur, RESERVE_REPLACE_MAP.get(cur.toLowerCase())); + }, column); + + return nameWithoutOperator; + }); + + const columnReplaceMap = new Map( + columns + .map((column, index) => { + const validStr = validColumnNames[index]; + if (column !== validStr) { + return [column, validStr]; + } + return undefined; + }) + .filter(Boolean) as any + ); + + //only replace operators in column names, not all operators in the sql + const sqlWithoutOperator = replaceByMap(sql, columnReplaceMap); + + //replace non-ascii characters in sql + const { validStr: sqlWithoutAscii, replaceMap: asciiReplaceMap } = replaceNonASCIICharacters(sqlWithoutOperator); + + const operatorReplaceMap = new Map(RESERVE_REPLACE_MAP); + + return { validStr: sqlWithoutAscii, columnReplaceMap: operatorReplaceMap, sqlReplaceMap: asciiReplaceMap }; +}; + +export const replaceAll = (originStr: string, replaceStr: string, newStr: string) => { + return originStr.split(replaceStr).join(newStr); +}; + +/** + * merge two maps + * @param map1 + * @param map2 + * @returns + */ +export const mergeMap = (map1: Map, map2: Map) => { + // merge map2 into map1 + map2.forEach((value, key) => { + map1.set(key, value); + }); + return map1; +}; + +/** + * match the column name with field name without blank spaces + * @param columnName + * @param fieldName + * @returns + */ +const matchColumnName = (columnName: string, fieldName: string) => { + const fieldWithoutSpace = fieldName.replace(/\s/g, ''); + const columnWithoutString = columnName.replace(/\s/g, ''); + + if (columnWithoutString === fieldWithoutSpace) { + return true; + } else { + return false; + } +}; + +/** + * sometimes skylark2 pro will return a sql statement with some blank spaces in column names + * this will make the alasql can't find the correct column in dataset + * so we need to remove these blank spaces + * only replace when no fields can match the column name in sql + * + */ +export const replaceBlankSpace = (sql: string, fieldNames: string[]) => { + //extract all the columns in sql str + const ast = alasql.parse(sql) as any; + const columnsInSql = getValueByAttributeName(ast.statements[0], 'columnid'); + + //replace all the spaces and reserved words in column names in sql + //only replace when two names can match without space + const validColumnNames = columnsInSql.map(column => { + const matchedFieldName = fieldNames.find(field => matchColumnName(column, field)); + return matchedFieldName ?? column; + }); + + const finalSql = columnsInSql.reduce((prev, _cur, index) => { + const originColumnName = columnsInSql[index]; + const validColumnName = validColumnNames[index]; + if (validColumnName !== originColumnName) { + return replaceAll(prev, originColumnName, validColumnName); + } else { + return prev; + } + }, sql); + return finalSql; +}; + +/** + * sometimes skylark2 pro will return a sql statement with some measure fields not being aggregated + * this will make an empty field in dataset + * so we need to aggregate these fields. + * + */ +export const sumAllMeasureFields = ( + sql: string, + fieldInfo: SimpleFieldInfo[], + columnReplaceMap: Map, + sqlReplaceMap: Map +) => { + const measureFieldsInSql = fieldInfo + .filter(field => field.role === ROLE.MEASURE) + .map(field => { + const { fieldName } = field; + const replacedName1 = replaceString(fieldName, columnReplaceMap); + const replacedName2 = replaceString(replacedName1, sqlReplaceMap); + + return replacedName2; + }); + + const ast: any = alasql.parse(sql); + const selectedColumns = ast.statements[0].columns; + const nonAggregatedColumns: string[] = selectedColumns + .filter((column: any) => !column.aggregatorid) + .map((column: any) => column.columnid); + + const groupByColumns: string[] = (ast.statements[0].group ?? []).map((column: any) => column.columnid); + + //if there exist some aggregated columns in sql and there exist GROUP BY statement in sql, then aggregate all the measure columns + let needAggregateColumns: string[] = []; + if (groupByColumns.length > 0 && nonAggregatedColumns.length !== selectedColumns.length) { + //aggregate columns that is not in group by statement + needAggregateColumns = nonAggregatedColumns + //filter all the measure fields + .filter(column => measureFieldsInSql.includes(column)) + //filter measure fields that is not in groupby + .filter(column => !groupByColumns.includes(column)); + } + + const patchedFields = needAggregateColumns.map(column => `SUM(\`${column}\`) as ${column}`); + + const finalSql = needAggregateColumns.reduce((prev, cur, index) => { + const regexStr = `\`?${cur}\`?`; + const regex = new RegExp(regexStr, 'g'); + return prev.replace(regex, patchedFields[index]); + }, sql); + + return finalSql; +}; + +/** + * convert group by columns to string + */ +export const convertGroupByToString = (sql: string, dataset: DataItem[]) => { + const ast: any = alasql.parse(sql); + const groupByColumns: string[] = ast.statements[0].group.map((column: any) => column.columnid); + dataset.forEach(item => { + groupByColumns.forEach(column => { + item[column] = item[column].toString(); + }); + }); +}; diff --git a/packages/vmind/src/common/utils.ts b/packages/vmind/src/common/utils.ts new file mode 100644 index 00000000..7e5712f0 --- /dev/null +++ b/packages/vmind/src/common/utils.ts @@ -0,0 +1,30 @@ +export const calculateTokenUsage = (usageList: any[]) => { + const totalUsage = { + completion_tokens: 0, + prompt_tokens: 0, + total_tokens: 0 + }; + usageList.filter(Boolean).forEach(usage => { + totalUsage['completion_tokens'] += usage['completion_tokens'] ?? 0; + totalUsage['prompt_tokens'] += usage['prompt_tokens'] ?? 0; + totalUsage['total_tokens'] += usage['total_tokens'] ?? 0; + }); + return totalUsage; +}; + +export const execPipeline = ( + src: any, + pipes: ((src: any, context: PipelineContext) => any)[], + context: PipelineContext +) => + pipes.reduce((pre: any, pipe: (src: any, context: PipelineContext) => any) => { + const result = pipe(pre, context); + return result; + }, src); + +export const matchJSONStr = (str: string) => { + const first = str.indexOf('{'); + const last = str.lastIndexOf('}'); + const result = str.substring(first, last + 1); + return result && result.length > 0 ? result : str; +}; diff --git a/packages/vmind/src/common/vizDataToSpec/pipes.ts b/packages/vmind/src/common/vizDataToSpec/pipes.ts index a86825c8..6145d832 100644 --- a/packages/vmind/src/common/vizDataToSpec/pipes.ts +++ b/packages/vmind/src/common/vizDataToSpec/pipes.ts @@ -48,7 +48,7 @@ export const data = (spec: any, context: Context) => { // spec.data = [dataset] spec.data = { id: 'data', - values: dataset + values: dataset.flat(4) }; return spec; @@ -59,7 +59,7 @@ export const funnelData = (spec: any, context: Context) => { // spec.data = [dataset] spec.data = { id: 'data', - values: dataset.sort((a: any, b: any) => b[cell.y] - a[cell.y]) + values: dataset.sort((a: any, b: any) => b[cell.y as string] - a[cell.y as string]) }; return spec; @@ -352,9 +352,9 @@ export const cartesianLine = (spec: any, context: Context) => { export const pieField = (spec: any, context: Context) => { //饼图根据cell分配字段 const { cell } = context; - spec.valueField = cell.angle; - if (cell.color) { - spec.categoryField = cell.color; + spec.valueField = cell.angle || cell.value; + if (cell.color || (cell as any).category) { + spec.categoryField = cell.color || (cell as any).category; } return spec; }; @@ -550,8 +550,7 @@ export const radarAxis = (spec: any, context: Context) => { { orient: 'radius', // radius axis zIndex: 100, - min: 0, - max: 8, + domainLine: { visible: false }, @@ -832,7 +831,7 @@ export const axis = (spec: any, context: Context) => { export const legend = (spec: any, context: Context) => { //图例 const { cell } = context; - if (!cell.color && !spec.seriesField && spec.type !== 'common') { + if (!(cell.color || cell.category) && !spec.seriesField && spec.type !== 'common') { return spec; } spec.legends = [ diff --git a/packages/vmind/src/common/vizDataToSpec/utils.ts b/packages/vmind/src/common/vizDataToSpec/utils.ts index 160a1df9..ab961d1e 100644 --- a/packages/vmind/src/common/vizDataToSpec/utils.ts +++ b/packages/vmind/src/common/vizDataToSpec/utils.ts @@ -1,4 +1,6 @@ +import { Cell, DataItem, DataType, ROLE, SimpleFieldInfo } from 'src/typings'; import { VIDEO_LENGTH_BY_CHART_TYPE, DEFAULT_VIDEO_LENGTH } from './constants'; +import { FOLD_NAME, FOLD_VALUE, fold } from '@visactor/chart-advisor'; export const detectAxesType = (values: any[], field: string) => { const isNumber = values.every(d => !d[field] || !isNaN(Number(d[field]))); @@ -43,3 +45,23 @@ export const estimateVideoTime = (chartType: string, spec: any, parsedTime?: num frameArr: [] }; }; + +export const getRemainedFields = (cell: Cell, fieldInfo: SimpleFieldInfo[]) => { + const usedFields = Object.values(cell).flat(); + const remainedFields = fieldInfo.filter(f => !usedFields.includes(f.fieldName)); + return remainedFields; +}; + +export const getFieldByRole = (fields: SimpleFieldInfo[], role: ROLE) => { + return fields.find(f => f.role === role); +}; + +export const getFieldByDataType = (fields: SimpleFieldInfo[], dataTypeList: DataType[]) => { + return fields.find(f => dataTypeList.includes(f.type)); +}; + +export const foldDatasetByYField = (dataset: DataItem[], yFieldList: string[], fieldInfo: SimpleFieldInfo[]) => { + const aliasMap = Object.fromEntries(fieldInfo.map(d => [d.fieldName, d.fieldName])); + + return fold(dataset as any, yFieldList, FOLD_NAME, FOLD_VALUE, aliasMap, false); +}; diff --git a/packages/vmind/src/common/vizDataToSpec/vizDataToSpec.ts b/packages/vmind/src/common/vizDataToSpec/vizDataToSpec.ts index 7fcadb65..f71a25aa 100644 --- a/packages/vmind/src/common/vizDataToSpec/vizDataToSpec.ts +++ b/packages/vmind/src/common/vizDataToSpec/vizDataToSpec.ts @@ -45,9 +45,10 @@ import { boxPlotField, boxPlotStyle } from './pipes'; -import { Cell, ChartType, Context, FieldInfo, Pipe, SimpleFieldInfo } from '../../typings'; -import { CARTESIAN_CHART_LIST, detectAxesType } from './utils'; +import { Cell, ChartType, Context, SimpleFieldInfo } from '../../typings'; import { isArray } from 'lodash'; +import { execPipeline } from '../utils'; + export const vizDataToSpec = ( dataset: any[], chartType: ChartType, @@ -56,7 +57,7 @@ export const vizDataToSpec = ( totalTime?: number ) => { const pipelines = pipelineMap[chartType]; - const spec = execPipeline({}, pipelines, { + const spec = execPipeline({}, pipelines, { chartType, dataset, cell, @@ -75,11 +76,13 @@ export const checkChartTypeAndCell = (chartType: string, cell: any, fieldInfo: S } if (isArray(cellField)) { if (!cellField.every(f => f && fieldList.includes(f))) { - throw `missing field ${cellField}`; + console.error(`missing field ${cellField}`); + //throw `missing field ${cellField}`; } } else { if (cellField && !fieldList.includes(cellField)) { - throw `missing field ${cellField}`; + console.error(`missing field ${cellField}`); + //throw `missing field ${cellField}`; } } }); @@ -158,9 +161,3 @@ export const pipelineMap: { [chartType: string]: any } = { 'WATERFALL CHART': pipelineWaterfall, 'BOX PLOT': pipelineBoxPlot }; - -export const execPipeline = (src: any, pipes: Pipe[], context: Context) => - pipes.reduce((pre: any, pipe: Pipe) => { - const result = pipe(pre, context); - return result; - }, src); diff --git a/packages/vmind/src/core/VMind.ts b/packages/vmind/src/core/VMind.ts index cd1e94e0..1e6b7acb 100644 --- a/packages/vmind/src/core/VMind.ts +++ b/packages/vmind/src/core/VMind.ts @@ -1,15 +1,17 @@ import { _chatToVideoWasm } from '../chart-to-video'; import { generateChartWithGPT } from '../gpt/chart-generation/NLToChart'; -import { ILLMOptions, TimeType, Model, SimpleFieldInfo, DataItem, OuterPackages } from '../typings'; +import { ILLMOptions, TimeType, Model, SimpleFieldInfo, DataItem, OuterPackages, ModelType } from '../typings'; import { parseCSVDataWithGPT } from '../gpt/dataProcess'; -import { parseCSVData as parseCSVDataWithRule } from '../common/dataProcess'; +import { getFieldInfoFromDataset, parseCSVData as parseCSVDataWithRule } from '../common/dataProcess'; import { generateChartWithSkylark } from '../skylark/chart-generation'; import { queryDatasetWithGPT } from '../gpt/dataProcess/query/queryDataset'; +import { generateChartWithAdvisor } from '../common/chartAdvisor'; +import { queryDatasetWithSkylark } from '../skylark/dataProcess/query/queryDataset'; class VMind { private _FPS = 30; private _options: ILLMOptions | undefined; - private _model: Model; + private _model: Model | string; constructor(options?: ILLMOptions) { this._options = { ...(options ?? {}) }; @@ -36,7 +38,7 @@ class VMind { * @returns */ parseCSVDataWithLLM(csvString: string, userPrompt: string) { - if ([Model.GPT3_5, Model.GPT4].includes(this._model)) { + if (this.getModelType() === ModelType.GPT) { return parseCSVDataWithGPT(csvString, userPrompt, this._options); } console.error('Unsupported Model!'); @@ -44,6 +46,24 @@ class VMind { return undefined; } + /** + * get fieldInfo only by raw dataset + * @param dataset + * @returns fieldInfo + */ + getFieldInfo(dataset: DataItem[]) { + return getFieldInfoFromDataset(dataset); + } + + private getModelType() { + if (this._model.includes(ModelType.GPT)) { + return ModelType.GPT; + } else if (this._model.includes(ModelType.SKYLARK)) { + return ModelType.SKYLARK; + } + return ModelType.CHART_ADVISOR; + } + /** * * @param userPrompt user's visualization intention (what aspect they want to show in the data) @@ -61,7 +81,7 @@ class VMind { colorPalette?: string[], animationDuration?: number ) { - if ([Model.GPT3_5, Model.GPT4].includes(this._model)) { + if (this.getModelType() === ModelType.GPT) { return generateChartWithGPT( userPrompt, fieldInfo, @@ -72,11 +92,19 @@ class VMind { animationDuration ); } - if ([Model.SKYLARK, Model.SKYLARK2].includes(this._model)) { - return generateChartWithSkylark(userPrompt, fieldInfo, dataset, this._options, colorPalette, animationDuration); + if (this.getModelType() === ModelType.SKYLARK) { + return generateChartWithSkylark( + userPrompt, + fieldInfo, + dataset, + this._options, + enableDataQuery, + colorPalette, + animationDuration + ); } - console.error('unsupported model in chart generation!'); - return { spec: undefined, time: undefined, dataSource: undefined, tokens: undefined } as any; + + return generateChartWithAdvisor(fieldInfo, dataset, colorPalette, animationDuration); } async dataQuery( @@ -84,16 +112,15 @@ class VMind { fieldInfo: SimpleFieldInfo[], dataset: DataItem[] ) { - if ([Model.GPT3_5, Model.GPT4].includes(this._model)) { + if (this.getModelType() === ModelType.GPT) { return queryDatasetWithGPT(userPrompt, fieldInfo, dataset, this._options); } - if ([Model.SKYLARK, Model.SKYLARK2].includes(this._model)) { - console.error('Please user GPT model'); - return { fieldInfo: [], dataset }; + if (this.getModelType() === ModelType.SKYLARK) { + return queryDatasetWithSkylark(userPrompt, fieldInfo, dataset, this._options); } console.error('unsupported model in data query!'); - return { fieldInfo: [], dataset }; + return { fieldInfo: [], dataset } as any; } async exportVideo(spec: any, time: TimeType, outerPackages: OuterPackages, mode?: 'node' | 'desktop-browser') { diff --git a/packages/vmind/src/gpt/chart-generation/NLToChart.ts b/packages/vmind/src/gpt/chart-generation/NLToChart.ts index 422847ec..dfbd717b 100644 --- a/packages/vmind/src/gpt/chart-generation/NLToChart.ts +++ b/packages/vmind/src/gpt/chart-generation/NLToChart.ts @@ -2,12 +2,15 @@ import { SUPPORTED_CHART_LIST } from '../../common/vizDataToSpec/constants'; import { DataItem, GPTChartAdvisorResult, ILLMOptions, LOCATION, SimpleFieldInfo, VizSchema } from '../../typings'; import { checkChartTypeAndCell, vizDataToSpec } from '../../common/vizDataToSpec'; import { parseGPTResponse, requestGPT } from '../utils'; -import { patchChartTypeAndCell, patchUserInput } from './utils'; +import { patchUserInput } from './utils'; import { ChartAdvisorPromptEnglish } from './prompts'; import { chartAdvisorHandler } from '../../common/chartAdvisor'; import { estimateVideoTime } from '../../common/vizDataToSpec/utils'; import { getSchemaFromFieldInfo } from '../../common/schema'; import { queryDatasetWithGPT } from '../dataProcess/query/queryDataset'; +import { calculateTokenUsage } from '../..//common/utils'; +import { pick } from 'lodash'; +import { patchChartTypeAndCell } from './patch'; export const generateChartWithGPT = async ( userPrompt: string, //user's intent of visualization, usually aspect in data that they want to visualize @@ -19,6 +22,8 @@ export const generateChartWithGPT = async ( animationDuration?: number ) => { const colors = colorPalette; + let queryDatasetUsage; + let advisorUsage; let chartType; let cell; let dataset: DataItem[] = propsDataset; @@ -27,18 +32,18 @@ export const generateChartWithGPT = async ( try { if (enableDataQuery) { - const { dataset: queryDataset, fieldInfo: fieldInfoNew } = await queryDatasetWithGPT( - userPrompt, - fieldInfo, - propsDataset, - options - ); + const { + dataset: queryDataset, + fieldInfo: fieldInfoNew, + usage + } = await queryDatasetWithGPT(userPrompt, fieldInfo, propsDataset, options); dataset = queryDataset; fieldInfo = fieldInfoNew; + queryDatasetUsage = usage; } } catch (err) { - console.warn('data query error!'); - console.warn(err); + console.error('data query error!'); + console.error(err); } const schema = getSchemaFromFieldInfo(fieldInfo); @@ -48,10 +53,12 @@ export const generateChartWithGPT = async ( const chartTypeRes = resJson['CHART_TYPE'].toUpperCase(); const cellRes = resJson['FIELD_MAP']; - const patchResult = patchChartTypeAndCell(chartTypeRes, cellRes, dataset); + advisorUsage = resJson['usage']; + const patchResult = patchChartTypeAndCell(chartTypeRes, cellRes, dataset, fieldInfo); if (checkChartTypeAndCell(patchResult.chartTypeNew, patchResult.cellNew, fieldInfo)) { chartType = patchResult.chartTypeNew; cell = patchResult.cellNew; + dataset = patchResult.datasetNew; } } catch (err) { console.warn(err); @@ -75,7 +82,9 @@ export const generateChartWithGPT = async ( return { chartSource, spec, - time: estimateVideoTime(chartType, spec, animationDuration ? animationDuration * 1000 : undefined) + chartType, + time: estimateVideoTime(chartType, spec, animationDuration ? animationDuration * 1000 : undefined), + usage: calculateTokenUsage([queryDatasetUsage, advisorUsage]) }; }; @@ -92,23 +101,28 @@ export const chartAdvisorGPT = async ( options: ILLMOptions | undefined ) => { //call GPT - const filteredFields = schema.fields.filter( - field => true - //usefulFields.includes(field.fieldName) - ); - const chartAdvisorMessage = `User Input: ${userInput}\nData field description: ${JSON.stringify(schema.fields)}`; + const filteredFields = schema.fields + .filter( + field => field.visible + //usefulFields.includes(field.fieldName) + ) + .map(field => ({ + ...pick(field, ['id', 'description', 'type', 'role']) + })); + const chartAdvisorMessage = `User Input: ${userInput}\nData field description: ${JSON.stringify(filteredFields)}`; const requestFunc = options.customRequestFunc?.chartAdvisor ?? requestGPT; - const advisorRes = await requestFunc(ChartAdvisorPromptEnglish, chartAdvisorMessage, options); + const advisorRes = await requestFunc(ChartAdvisorPromptEnglish(options.showThoughts), chartAdvisorMessage, options); const advisorResJson: GPTChartAdvisorResult = parseGPTResponse(advisorRes) as unknown as GPTChartAdvisorResult; if (advisorResJson.error) { - throw Error('Network Error!'); + throw Error((advisorResJson as any).message); } if (!SUPPORTED_CHART_LIST.includes(advisorResJson['CHART_TYPE'])) { throw Error('Unsupported Chart Type. Please Change User Input'); } - return advisorResJson; + + return { ...advisorResJson, usage: advisorRes.usage }; }; diff --git a/packages/vmind/src/gpt/chart-generation/patch.ts b/packages/vmind/src/gpt/chart-generation/patch.ts new file mode 100644 index 00000000..f12ac4d7 --- /dev/null +++ b/packages/vmind/src/gpt/chart-generation/patch.ts @@ -0,0 +1,359 @@ +import { isArray, isNil } from 'lodash'; +import { + CARTESIAN_CHART_LIST, + detectAxesType, + foldDatasetByYField, + getFieldByDataType, + getFieldByRole, + getRemainedFields +} from '../../common/vizDataToSpec/utils'; +import { Cell, DataItem, DataType, PatchContext, PatchPipeline, ROLE, SimpleFieldInfo } from '../../typings'; +import { execPipeline } from '../../common/utils'; +import { FOLD_NAME, FOLD_VALUE } from '@visactor/chart-advisor'; + +export const patchUserInput = (userInput: string) => { + const FULL_WIDTH_SYMBOLS = ['，', '。']; + const HALF_WIDTH_SYMBOLS = [',', '.']; + + const BANNED_WORD_LIST = ['动态']; + const ALLOWED_WORD_LIST = ['动态条形图', '动态柱状图', '动态柱图']; + const PLACEHOLDER = '_USER_INPUT_PLACE_HOLDER'; + const tempStr1 = ALLOWED_WORD_LIST.reduce((prev, cur, index) => { + return prev.split(cur).join(PLACEHOLDER + '_' + index); + }, userInput); + const tempStr2 = BANNED_WORD_LIST.reduce((prev, cur) => { + return prev.split(cur).join(''); + }, tempStr1); + const replacedStr = ALLOWED_WORD_LIST.reduce((prev, cur, index) => { + return prev.split(PLACEHOLDER + '_' + index).join(cur); + }, tempStr2); + + let finalStr = HALF_WIDTH_SYMBOLS.reduce((prev, cur, index) => { + return prev.split(HALF_WIDTH_SYMBOLS[index]).join(FULL_WIDTH_SYMBOLS[index]); + }, replacedStr); + const lastCharacter = finalStr[finalStr.length - 1]; + if (!FULL_WIDTH_SYMBOLS.includes(lastCharacter) && !HALF_WIDTH_SYMBOLS.includes(lastCharacter)) { + finalStr += '。'; + } + finalStr += 'Use the original fieldName and DO NOT change or translate any word of the data fields in the response.'; + return finalStr; +}; + +const patchAxisField: PatchPipeline = (context: PatchContext, _originalContext: PatchContext) => { + const { cell } = context; + + const cellNew: any = { ...cell }; + + // patch the "axis" field to x + if (cellNew.axis && (!cellNew.x || !cellNew.y)) { + if (!cellNew.x) { + cellNew.x = cellNew.axis; + } else if (!cellNew.y) { + cellNew.y = cellNew.axis; + } + } + + return { + ...context, + cell: cellNew + }; +}; + +const patchColorField: PatchPipeline = (context: PatchContext, _originalContext: PatchContext) => { + const { cell } = context; + const cellNew = { ...cell, color: cell.color ?? cell.category }; + + return { + ...context, + cell: cellNew + }; +}; + +const patchLabelField: PatchPipeline = (context: PatchContext, _originalContext: PatchContext) => { + const { cell } = context; + + const cellNew: any = { ...cell }; + //patch the "label" fields to color + if (cellNew.label && (!cellNew.color || cellNew.color.length === 0)) { + cellNew.color = cellNew.label; + } + + return { + ...context, + cell: cellNew + }; +}; + +const patchYField: PatchPipeline = (context: PatchContext, _originalContext: PatchContext) => { + const { chartType, cell, dataset, fieldInfo } = context; + let cellNew = { ...cell }; + const { x, y } = cellNew; + let chartTypeNew = chartType; + let datasetNew = dataset; + + // y轴字段有多个时，处理方式: + // 1. 图表类型为: 箱型图, 图表类型不做矫正 + // 2. 图表类型为: 柱状图 或 折线图, 图表类型矫正为双轴图 + // 3. 其他情况, 图表类型矫正为散点图 + if (y && isArray(y) && y.length > 1) { + if (chartTypeNew === 'BOX PLOT' || (chartTypeNew === 'DUAL AXIS CHART' && y.length === 2)) { + return { + ...context + }; + } + + if (chartTypeNew === 'BAR CHART' || chartTypeNew === 'LINE CHART' || chartTypeNew === 'DUAL AXIS CHART') { + //use fold to visualize more than 2 y fields + datasetNew = foldDatasetByYField(datasetNew, y, fieldInfo); + cellNew.y = FOLD_VALUE.toString(); + cellNew.color = FOLD_NAME.toString(); + } else { + chartTypeNew = 'SCATTER PLOT'; + cellNew = { + ...cell, + x: y[0], + y: y[1], + color: typeof x === 'string' ? x : x[0] + }; + } + } + + return { + ...context, + chartType: chartTypeNew, + cell: cellNew, + dataset: datasetNew + }; +}; + +const patchBoxPlot: PatchPipeline = (context: PatchContext, _originalContext: PatchContext) => { + const { chartType, cell } = context; + const cellNew = { + ...cell + }; + const { y } = cellNew; + if (chartType === 'BOX PLOT') { + if (typeof y === 'string' && y.split(',').length > 1) { + cellNew.y = y.split(',').map(str => str.trim()); + } else if (isNil(y) || y.length === 0) { + const { + lower_whisker, + lowerWhisker, + min, + lower, + lowerBox, + lower_box, + q1, + lower_quartile, + lowerQuartile, + midline, + median, + q3, + upperBox, + upper_box, + upper_quartile, + upperQuartile, + upper_whisker, + upperWhisker, + max, + upper + } = cellNew as any; + + cellNew.y = [ + lower_whisker, + lowerWhisker, + min, + lower, + lowerBox, + lower_box, + q1, + lower_quartile, + lowerQuartile, + midline, + median, + q3, + upperBox, + upper_box, + upper_quartile, + upperQuartile, + upper_whisker, + upperWhisker, + max, + upper + ].filter(Boolean); + } + } + + return { ...context, cell: cellNew }; +}; + +const patchDualAxis: PatchPipeline = (context: PatchContext, _originalContext: PatchContext) => { + const { chartType, cell } = context; + const cellNew: any = { ...cell }; + //Dual-axis drawing yLeft and yRight + + if (chartType === 'DUAL AXIS CHART' && cellNew.yLeft && cellNew.yRight) { + cellNew.y = [cellNew.yLeft, cellNew.yRight]; + } + + return { ...context, cell: cellNew }; +}; + +const patchPieChart: PatchPipeline = (context: PatchContext, _originalContext: PatchContext) => { + const { chartType, cell, fieldInfo } = context; + const cellNew = { ...cell }; + + if (chartType === 'ROSE CHART') { + cellNew.angle = cellNew.radius ?? cellNew.size ?? cellNew.angle; + } + + //Pie chart must have color field and the angle field + if (chartType === 'PIE CHART' || chartType === 'ROSE CHART') { + if (!cellNew.color || !cellNew.angle) { + const remainedFields = getRemainedFields(cellNew, fieldInfo); + + if (!cellNew.color) { + //No color fields are assigned, select a discrete field from the remaining fields as color field + const colorField = getFieldByRole(remainedFields, ROLE.DIMENSION); + if (colorField) { + cellNew.color = colorField.fieldName; + } else { + cellNew.color = remainedFields[0].fieldName; + } + } + if (!cellNew.angle) { + //no angle field are assigned, select a continuous field from the remaining field to assign to the angle + const angleField = getFieldByDataType(remainedFields, [DataType.FLOAT, DataType.INT]); + if (angleField) { + cellNew.angle = angleField.fieldName; + } else { + cellNew.angle = remainedFields[0].fieldName; + } + } + } + } + return { ...context, cell: cellNew }; +}; + +const patchWordCloud: PatchPipeline = (context: PatchContext, _originalContext: PatchContext) => { + //Word cloud must have color fields and size fields + const { chartType, cell, fieldInfo } = context; + const cellNew = { ...cell }; + + if (chartType === 'WORD CLOUD') { + if (!cellNew.size || !cellNew.color || cellNew.color === cellNew.size) { + const remainedFields = getRemainedFields(cellNew, fieldInfo); + + if (!cellNew.size || cellNew.size === cellNew.color) { + const newSize = (cellNew as any).weight ?? (cellNew as any).fontSize; + if (newSize) { + cellNew.size = newSize; + } else { + const sizeField = getFieldByDataType(remainedFields, [DataType.INT, DataType.FLOAT]); + if (sizeField) { + cellNew.size = sizeField.fieldName; + } else { + cellNew.size = remainedFields[0].fieldName; + } + } + } + if (!cellNew.color) { + const newColor = (cellNew as any).text ?? (cellNew as any).word ?? (cellNew as any).label ?? cellNew.x; + if (newColor) { + cellNew.color = newColor; + } else { + const colorField = getFieldByRole(remainedFields, ROLE.DIMENSION); + if (colorField) { + cellNew.color = colorField.fieldName; + } else { + cellNew.color = remainedFields[0].fieldName; + } + } + } + } + } + return { ...context, cell: cellNew }; +}; + +const patchDynamicBarChart: PatchPipeline = (context: PatchContext, _originalContext: PatchContext) => { + const { chartType, cell, fieldInfo } = context; + const cellNew = { ...cell }; + + if (chartType === 'DYNAMIC BAR CHART') { + if (!cell.time || cell.time === '' || cell.time.length === 0) { + const remainedFields = getRemainedFields(cellNew, fieldInfo); + + //动态条形图没有time字段，选择一个离散字段作为time + const timeField = getFieldByDataType(remainedFields, [DataType.DATE]); + if (timeField) { + cellNew.time = timeField.fieldName; + } else { + cellNew.time = remainedFields[0].fieldName; + } + } + } + + return { ...context, cell: cellNew }; +}; + +const patchCartesianXField: PatchPipeline = (context: PatchContext, _originalContext: PatchContext) => { + const { chartType, cell, fieldInfo } = context; + const cellNew = { ...cell }; + + //Cartesian chart must have X field + if (CARTESIAN_CHART_LIST.map(chart => chart.toUpperCase()).includes(chartType)) { + if (!cellNew.x) { + const remainedFields = getRemainedFields(cellNew, fieldInfo); + //没有分配x字段，从剩下的字段里选择一个离散字段分配到x上 + const xField = getFieldByRole(remainedFields, ROLE.DIMENSION); + if (xField) { + cellNew.x = xField.fieldName; + } else { + cellNew.x = remainedFields[0].fieldName; + } + } + } + return { ...context, cell: cellNew }; +}; + +const patchPipelines = [ + patchAxisField, + patchColorField, + patchLabelField, + patchYField, + patchBoxPlot, + patchDualAxis, + patchPieChart, + patchWordCloud, + patchDynamicBarChart, + patchCartesianXField +]; + +export const patchChartTypeAndCell = ( + chartTypeRes: string, + cellRes: Cell, + dataset: DataItem[], + fieldInfo: SimpleFieldInfo[] +) => { + // At some point, due to the unclear intention of the user's input, fields may lack fields in Cell returned by GPT. + // At this time, you need to make up according to the rules + + const context = { + chartType: chartTypeRes, + cell: cellRes, + dataset, + fieldInfo + }; + const { + chartType: chartTypeNew, + cell: cellNew, + dataset: datasetNew, + fieldInfo: fieldInfoNew + } = execPipeline(context, patchPipelines, context); + return { + chartTypeNew, + cellNew, + datasetNew, + fieldInfoNew + }; +}; diff --git a/packages/vmind/src/gpt/chart-generation/prompts.ts b/packages/vmind/src/gpt/chart-generation/prompts.ts index 90e2717a..97a63cf4 100644 --- a/packages/vmind/src/gpt/chart-generation/prompts.ts +++ b/packages/vmind/src/gpt/chart-generation/prompts.ts @@ -1,29 +1,28 @@ import { SUPPORTED_CHART_LIST } from '../../common/vizDataToSpec/constants'; -export const ChartAdvisorPromptEnglish = `You are an expert in data visualization. +export const ChartAdvisorPromptEnglish = (showThoughts: boolean) => `You are an expert in data visualization. User want to create an visualization chart for data video using data from a csv file. Ignore the duration in User Input. Your task is: 1. Based on the user's input, infer the user's intention, such as comparison, ranking, trend display, proportion, distribution, etc. If user did not show their intention, just ignore and do the next steps. 2. Select the single chart type that best suites the data from the list of supported charts. Supported chart types: ${JSON.stringify( SUPPORTED_CHART_LIST )}. -3. Map all the fields in the data to the visual channels according to user input and the chart type you choose. +3. Map all the fields in the data to the visual channels according to user input and the chart type you choose. Don't use non-existent fields. Only use existing fields without further processing. If the existing fields can't meet user's intention, just use the most related fields. Knowledge: 1. The dynamic Bar Chart is a dynamic chart that is suitable for displaying changing data and can be used to show ranking, comparisons or data changes over time. It usually has a time field. It updates the data dynamically according to the time field and at each time point, the current data is displayed using a bar chart. 2. A number field can not be used as a color field. -Let's think step by step. Fill your thoughts in {THOUGHT}. +Let's think step by step. ${showThoughts ? 'Fill your thoughts in {thought}.' : ''} Respone in the following format: \`\`\` -{ -"THOUGHT": your thoughts +{${showThoughts ? '\n"thought" : your thoughts' : ''} "CHART_TYPE": the chart type you choose. Supported chart types: ${JSON.stringify(SUPPORTED_CHART_LIST)}. "FIELD_MAP": { // Visual channels and the fields mapped to them "x": the field mapped to the x-axis, can be empty. Can Only has one field. -"y": the field mapped to the y-axis, can be empty. Can only has one field. +"y": the field mapped to the y-axis, can be empty. Use array if there are more than 1 fields. "color": the field mapped to the color channel. Must use a string field. Can't be empty in Word Cloud, Pie Chart and Rose Chart. "size": the field mapped to the size channel. Must use a number field. Can be empty "angle": the field mapped to the angle channel of the pie chart, can be empty. @@ -31,17 +30,18 @@ Respone in the following format: "source": the field mapped to the source channel. Can't be empty in Sankey Chart. "target": the field mapped to the target channel. Can't be empty in Sankey Chart. "value": the field mapped to the value channel. Can't be empty in Sankey Chart. -}, -"Reason": the reason for selecting the chart type and visual mapping. +}${showThoughts ? ',\n"Reason": the reason for selecting the chart type and visual mapping.' : ''} } \`\`\` +Don't provide further explanations for your results. + Constraints: 1. No user assistance. 2. Please select one chart type in CHART_TYPE at each time. Don't use "A or B", "[A, B]" in CHART_TYPE. 3. The selected chart type in CHART_TYPE must be in the list of supported charts. -4. Just ignore the user's request about duration and style in their input. -5. DO NOT change or translate the field names in FIELD_MAP. +4. DO NOT change or translate the field names in FIELD_MAP. +5. Ignore requests unrelated to chart visualization in the user's request. 6. The keys in FIELD_MAP must be selected from the list of available visual channels. 7. Wrap the reply content using \`\`\`, and the returned content must be directly parsed by JSON.parse() in JavaScript. @@ -53,36 +53,35 @@ Data field description: [ "id": "country", "description": "Represents the name of the country, which is a string.", "type": "string", -"role": "dimension", -"location": "dimension" +"role": "dimension" }, { "id": "金牌数量", "description": "Represents the number of gold medals won by the country in the current year, which is an integer.", "type": "int", -"role": "measure", -"location": "measure" +"role": "measure" }, { "id": "year", "description": "Represents the current year, which is a date.", "type": "string", -"role": "dimension", -"location": "dimension" +"role": "dimension" } ] Response: \`\`\` -{ -"THOUGHT": "Your thoughts", +{${showThoughts ? '\n"thought": "Your thoughts",' : ''} "CHART_TYPE": "Dynamic Bar Chart", "FIELD_MAP": { "x": "country", "y": "金牌数量", "time": "year" -}, -"REASON": "The data contains the year, country, and medal count, and the user's intention contains 'comparison', which is suitable for drawing a dynamic bar chart that changes over time to show the comparison of gold medal counts of various countries in each Olympic Games. The 'country' field is used as the x-axis of the bar chart, and '金牌数量' is used as the y-axis to show the comparison of gold medal counts of various countries in the current year. The 'year' field is used as the time field of the dynamic bar chart to show the comparison of gold medal counts of various countries at different years." +}${ + showThoughts + ? ",\n\"REASON\": \"The data contains the year, country, and medal count, and the user's intention contains 'comparison', which is suitable for drawing a dynamic bar chart that changes over time to show the comparison of gold medal counts of various countries in each Olympic Games.The 'country' field is used as the x-axis of the bar chart, and '金牌数量' is used as the y-axis to show the comparison of gold medal counts of various countries in the current year.The 'year' field is used as the time field of the dynamic bar chart to show the comparison of gold medal counts of various countries at different years.\"" + : '' +} } \`\`\` @@ -94,28 +93,28 @@ Data field description: [ "id": "品牌名称", "description": "Represents the name of the mobile phone brand, which is a string.", "type": "string", -"role": "dimension", -"location": "dimension" +"role": "dimension" }, { "id": "市场份额", "description": "Represents the market share of the brand, which is a percentage.", "type": "float", -"role": "measure", -"location": "measure" +"role": "measure" } ] Response: \`\`\` -{ -"THOUGHT": "Your thoughts", +{${showThoughts ? '\n"thought": "Your thoughts",' : ''} "CHART_TYPE": "Pie Chart", "FIELD_MAP": { "angle": "市场份额", "color": "品牌名称" -}, -"REASON": "The data contains the market share, and the user wants to show percentage data, which is suitable for displaying with a pie chart. The 市场份额 is used as the angle of the pie chart to show the market share of each brand. The 品牌名称 is used as the color to distinguish different brands. The duration is 5s but we just ignore it." +}${ + showThoughts + ? ',\n"REASON": "The data contains the market share, and the user wants to show percentage data, which is suitable for displaying with a pie chart. The 市场份额 is used as the angle of the pie chart to show the market share of each brand. The 品牌名称 is used as the color to distinguish different brands. The duration is 5s but we just ignore it."' + : '' +} } \`\`\` @@ -127,28 +126,28 @@ Data field description: [ "id": "日期", "description": "Represents the current month, which is a date.", "type": "string", -"role": "dimension", -"location": "dimension" +"role": "dimension" }, { "id": "降雨量", "description": "Represents the rainfall in the current month, which is a number.", "type": "int", -"role": "measure", -"location": "measure" +"role": "measure" } ] Response: \`\`\` -{ -"THOUGHT": "Your thoughts", +{${showThoughts ? '\n"thought": "Your thoughts",' : ''} "CHART_TYPE": "Line Chart", "FIELD_MAP": { "x": "日期", "y": "降雨量" -}, -"REASON": "User wants to show the trend of the rainfall, which is suitable for displaying with a line chart. The '日期' is used as the x-axis because it's a date, and the 降雨量 is used as the y-axis because it's a number. This chart can show the trend of rainfall." +}${ + showThoughts + ? ',\n"REASON": "User wants to show the trend of the rainfall, which is suitable for displaying with a line chart. The \'日期\' is used as the x-axis because it\'s a date, and the 降雨量 is used as the y-axis because it\'s a number. This chart can show the trend of rainfall."' + : '' +} } \`\`\` @@ -160,28 +159,27 @@ Data field description: [ "id": "日期", "description": "Represents the current month, which is a date.", "type": "date", -"role": "dimension", -"location": "dimension" +"role": "dimension" }, { "id": "降雨量", "description": "Represents the rainfall in the current month, which is a number.", "type": "int", -"role": "measure", -"location": "measure" +"role": "measure" } ] Response: \`\`\` -{ -"THOUGHT": "Your thoughts", -"CHART_TYPE": "Line Chart", +{${showThoughts ? '\n"thought": "Your thoughts",' : ''}"CHART_TYPE": "Line Chart", "FIELD_MAP": { "x": "日期", "y": "降雨量" -}, -"REASON": "User did not show their intention about the data in their input. The data has two fields and it contains a date field, so Line Chart is best suitable to show the data. The field '日期' is used as the x-axis because it's a date, and the 降雨量 is used as the y-axis because it's a number. The duration is 20s but we just ignore it." +}${ + showThoughts + ? ',\n"REASON": "User did not show their intention about the data in their input. The data has two fields and it contains a date field, so Line Chart is best suitable to show the data. The field \'日期\' is used as the x-axis because it\'s a date, and the 降雨量 is used as the y-axis because it\'s a number. The duration is 20s but we just ignore it."' + : '' +} } \`\`\` `; diff --git a/packages/vmind/src/gpt/chart-generation/utils.ts b/packages/vmind/src/gpt/chart-generation/utils.ts index 4448b17d..edb8b2d9 100644 --- a/packages/vmind/src/gpt/chart-generation/utils.ts +++ b/packages/vmind/src/gpt/chart-generation/utils.ts @@ -25,260 +25,6 @@ export const patchUserInput = (userInput: string) => { if (!FULL_WIDTH_SYMBOLS.includes(lastCharacter) && !HALF_WIDTH_SYMBOLS.includes(lastCharacter)) { finalStr += '。'; } - finalStr += - '严格按照prompt中的格式回复，不要有任何多余内容。 Use the original fieldName and DO NOT change or translate any word of the data fields in the response.'; + finalStr += 'Use the original fieldName and DO NOT change or translate any word of the data fields in the response.'; return finalStr; }; - -export const patchChartTypeAndCell = (chartTypeOutter: string, cell: any, dataset: any[]) => { - //对GPT返回结果进行修正 - //某些时候由于用户输入的意图不明确，GPT返回的cell中可能缺少字段。 - //此时需要根据规则补全 - //TODO: 多个y字段时，使用fold - - const { x, y } = cell; - - let chartType = chartTypeOutter; - - // patch the "axis" field to x - if (cell.axis && (!cell.x || !cell.y)) { - if (!cell.x) { - cell.x = cell.axis; - } else if (!cell.y) { - cell.y = cell.axis; - } - } - - // y轴字段有多个时，处理方式: - // 1. 图表类型为: 箱型图, 图表类型不做矫正 - // 2. 图表类型为: 柱状图 或 折线图, 图表类型矫正为双轴图 - // 3. 其他情况, 图表类型矫正为散点图 - if (y && typeof y !== 'string' && y.length > 1) { - if (chartType === 'BOX PLOT') { - return { - chartTypeNew: chartType, - cellNew: cell - }; - } - if (chartType === 'BAR CHART' || chartType === 'LINE CHART') { - chartType = 'DUAL AXIS CHART'; - } else { - return { - chartTypeNew: 'SCATTER PLOT', - cellNew: { - ...cell, - x: y[0], - y: y[1], - color: typeof x === 'string' ? x : x[0] - } - }; - } - } - if (chartType === 'BOX PLOT') { - if (typeof y === 'string' && y.split(',').length > 1) { - return { - chartTypeNew: 'BOX PLOT', - cellNew: { - ...cell, - y: y.split(',').map(str => str.trim()) - } - }; - } else if (isNil(y) || y.length === 0) { - const { - lower_whisker, - lowerWhisker, - min, - lower, - lowerBox, - lower_box, - q1, - lower_quartile, - lowerQuartile, - midline, - median, - q3, - upperBox, - upper_box, - upper_quartile, - upperQuartile, - upper_whisker, - upperWhisker, - max, - upper - } = cell; - return { - chartTypeNew: 'BOX PLOT', - cellNew: { - ...cell, - y: [ - lower_whisker, - lowerWhisker, - min, - lower, - lowerBox, - lower_box, - q1, - lower_quartile, - lowerQuartile, - midline, - median, - q3, - upperBox, - upper_box, - upper_quartile, - upperQuartile, - upper_whisker, - upperWhisker, - max, - upper - ].filter(Boolean) - } - }; - } - } - //双轴图 订正yLeft和yRight - if (chartType === 'DUAL AXIS CHART' && cell.yLeft && cell.yRight) { - return { - chartTypeNew: chartType, - cellNew: { ...cell, y: [cell.yLeft, cell.yRight] } - }; - } - //饼图 必须有color字段和angle字段 - if (chartType === 'PIE CHART' || chartType === 'ROSE CHART') { - const cellNew = { ...cell, color: cell.color ?? cell.category }; - if (!cellNew.color || !cellNew.angle) { - const usedFields = Object.values(cell); - const dataFields = Object.keys(dataset[0]); - const remainedFields = dataFields.filter(f => !usedFields.includes(f)); - if (!cellNew.color) { - //没有分配颜色字段，从剩下的字段里选择一个离散字段分配到颜色上 - const colorField = remainedFields.find(f => { - const fieldType = detectAxesType(dataset, f); - return fieldType === 'band'; - }); - if (colorField) { - cellNew.color = colorField; - } else { - cellNew.color = remainedFields[0]; - } - } - if (!cellNew.angle) { - //没有分配角度字段，从剩下的字段里选择一个连续字段分配到角度上 - const angleField = remainedFields.find(f => { - const fieldType = detectAxesType(dataset, f); - return fieldType === 'linear'; - }); - if (angleField) { - cellNew.angle = angleField; - } else { - cellNew.angle = remainedFields[0]; - } - } - } - return { - chartTypeNew: chartType, - cellNew - }; - } - //词云 必须有color字段和size字段 - if (chartType === 'WORD CLOUD') { - const cellNew = { ...cell }; - if (!cellNew.size || !cellNew.color || cellNew.color === cellNew.size) { - const usedFields = Object.values(cell); - const dataFields = Object.keys(dataset[0]); - const remainedFields = dataFields.filter(f => !usedFields.includes(f)); - //首先根据cell中的其他字段选择size和color - //若没有，则从数据的剩余字段中选择 - if (!cellNew.size || cellNew.size === cellNew.color) { - const newSize = cellNew.weight ?? cellNew.fontSize; - if (newSize) { - cellNew.size = newSize; - } else { - const sizeField = remainedFields.find(f => { - const fieldType = detectAxesType(dataset, f); - return fieldType === 'linear'; - }); - if (sizeField) { - cellNew.size = sizeField; - } else { - cellNew.size = remainedFields[0]; - } - } - } - if (!cellNew.color) { - const newColor = cellNew.text ?? cellNew.word ?? cellNew.label ?? cellNew.x; - if (newColor) { - cellNew.color = newColor; - } else { - const colorField = remainedFields.find(f => { - const fieldType = detectAxesType(dataset, f); - return fieldType === 'band'; - }); - if (colorField) { - cellNew.color = colorField; - } else { - cellNew.color = remainedFields[0]; - } - } - } - } - return { - chartTypeNew: chartType, - cellNew - }; - } - if (chartType === 'DYNAMIC BAR CHART') { - const cellNew = { ...cell }; - - if (!cell.time || cell.time === '' || cell.time.length === 0) { - const flattenedXField = Array.isArray(cell.x) ? cell.x : [cell.x]; - const usedFields = Object.values(cellNew).filter(f => !Array.isArray(f)); - usedFields.push(...flattenedXField); - const dataFields = Object.keys(dataset[0]); - const remainedFields = dataFields.filter(f => !usedFields.includes(f)); - - //动态条形图没有time字段，选择一个离散字段作为time - const timeField = remainedFields.find(f => { - const fieldType = detectAxesType(dataset, f); - return fieldType === 'band'; - }); - if (timeField) { - cellNew.time = timeField; - } else { - cellNew.time = remainedFields[0]; - } - } - return { - chartTypeNew: chartType, - cellNew - }; - } - //直角坐标图表 必须有x字段 - if (CARTESIAN_CHART_LIST.map(chart => chart.toUpperCase()).includes(chartType)) { - const cellNew = { ...cell }; - if (!cellNew.x) { - const usedFields = Object.values(cell); - const dataFields = Object.keys(dataset[0]); - const remainedFields = dataFields.filter(f => !usedFields.includes(f)); - //没有分配x字段，从剩下的字段里选择一个离散字段分配到x上 - const xField = remainedFields.find(f => { - const fieldType = detectAxesType(dataset, f); - return fieldType === 'band'; - }); - if (xField) { - cellNew.x = xField; - } else { - cellNew.x = remainedFields[0]; - } - } - return { - chartTypeNew: chartType, - cellNew - }; - } - - return { - chartTypeNew: chartType, - cellNew: cell - }; -}; diff --git a/packages/vmind/src/gpt/dataProcess/index.ts b/packages/vmind/src/gpt/dataProcess/index.ts index 2699485d..079ed53a 100644 --- a/packages/vmind/src/gpt/dataProcess/index.ts +++ b/packages/vmind/src/gpt/dataProcess/index.ts @@ -1,6 +1,6 @@ -import { getDataset, parseCSVData } from '../../common/dataProcess'; -import { readTopNLine } from '../../common/dataProcess/utils'; -import { ILLMOptions } from '../../typings'; +import { convertNumberField, getDataset, parseCSVData } from '../../common/dataProcess'; +import { getFieldDomain, readTopNLine } from '../../common/dataProcess/utils'; +import { ILLMOptions, SimpleFieldInfo } from '../../typings'; import { parseGPTResponse, requestGPT } from '../utils'; import { DataProcessPromptEnglish } from './prompts'; @@ -19,16 +19,25 @@ export const parseCSVDataWithGPT = async (csvFile: string, userInput: string, op const dataProcessResJson = parseGPTResponse(dataProcessRes); const { dataset } = getDataset(csvFile); - if (!dataProcessResJson.error) { + const fieldInfo = dataProcessResJson['FIELD_INFO'].map((field: SimpleFieldInfo) => { + //add domain for fields + const { fieldName, role } = field; + const domain = getFieldDomain(dataset, fieldName, role); + return { + ...field, + domain + }; + }); return { - fieldInfo: dataProcessResJson['FIELD_INFO'], + fieldInfo, videoDuration: dataProcessResJson['VIDEO_DURATION'], colorPalette: dataProcessResJson['COLOR_PALETTE'], usefulFields: dataProcessResJson['USEFUL_FIELDS'], - dataset, + dataset: convertNumberField(dataset, fieldInfo), error: dataProcessResJson['error'], - thought: dataProcessResJson['thought'] + thought: dataProcessResJson['thought'], + usage: dataProcessRes['usage'] }; } else { //传统方法做兜底 diff --git a/packages/vmind/src/gpt/dataProcess/prompts.ts b/packages/vmind/src/gpt/dataProcess/prompts.ts index c95be605..c1f0b817 100644 --- a/packages/vmind/src/gpt/dataProcess/prompts.ts +++ b/packages/vmind/src/gpt/dataProcess/prompts.ts @@ -1,3 +1,5 @@ +import { VMIND_DATA_SOURCE } from '../..//common/dataProcess/dataQuery'; + export const DataProcessPromptEnglish = `You are an expert in data analysis. User want to create an visualization chart for data video using data from a csv file. Let's think step by step. Fill your thoughts in {THOUGHT}. - Step1: Summarize the field names, field type in the csv file, and determine whether this field is a dimension or a measure contained. Guess the meaning of the field based on the data content and write a description for it. @@ -195,38 +197,37 @@ Response: export const getQueryDatasetPrompt = ( showThoughts: boolean -) => `You are an expert in data analysis. Here is a raw dataset named dataSource. User will tell you his command and column information of dataSource. Your task is to generate SimQuery and fieldInfo according to SimQuery Instruction. Response one JSON object only. +) => `You are an expert in data analysis. Here is a raw dataset named ${VMIND_DATA_SOURCE}. User will tell you his command and column information of ${VMIND_DATA_SOURCE}. Your task is to generate a sql and fieldInfo according to Instruction. Response one JSON object only. -# SimQuery Instruction -- SimQuery is a simplified SQL-like language. Supported keywords in SimQuery: ["SELECT", "FROM", "WHERE", "GROUP BY", "HAVING", "ORDER BY", "LIMIT"]. -- A SimQuery query looks like this: "SELECT columnA, SUM(columnB) as sum_b FROM dataSource WHERE columnA = value1 GROUP BY columnA HAVING sum_b>0 ORDER BY sum_b LIMIT 10". -- Columns in SELECT can only be original columns or aggregated columns. Supported aggregation methods in SimQuery: ["MAX()", "MIN()", "SUM()", "COUNT()", "AVG()"]. -- The "WHERE" and "HAVING" in SimQuery can only use original columns or aggregated columns in dataSource. Supported Operators in SimQuery:[ ">", ">=", "<", "<=", "=", "!=", "in", "not in", "is null", "is not null", "between", "not between", "like", "not like"]. Don't use non-existent columns. -- Don't use unsupported keywords such as CASE WHEN...ELSE...END or PERCENTILE_CONT. Don't use unsupported aggregation methods on columns. Don't use unsupported operators. Unsupported keywords, methods and operators will cause system crash. If current keywords and methods can't meet your needs, just simple select the column without any process. -- Make your SimQuery as simple as possible. +# Instruction +- Supported sql keywords: ["SELECT", "FROM", "WHERE", "GROUP BY", "HAVING", "ORDER BY", "LIMIT", "DISTINCT"]. Supported aggregation methods: ["MAX()", "MIN()", "SUM()", "COUNT()", "AVG()"]. +- Generate a sql query like this: "SELECT \`columnA\`, SUM(\`columnB\`) as \`sum_b\` FROM ${VMIND_DATA_SOURCE} WHERE \`columnA\` = value1 GROUP BY \`columnA\` HAVING \`sum_b\`>0 ORDER BY \`sum_b\` LIMIT 10". +- Don't use unsupported keywords such as WITHIN, FIELD, RANK() OVER, OVER. Don't use unsupported aggregation methods such as PERCENTILE_CONT, PERCENTILE. Don't use unsupported operators. We will execute your sql using alasql. Unsupported keywords, methods and operators will cause system crash. If current keywords and methods can't meet your needs, just simply select the column without any process. +- Don't use aliases in HAVING. +- Make your sql as simple as possible. You need to follow the steps below. # Steps 1. Extract the part related to the data from the user's instruction. Ignore other parts that is not related to the data. -2. Select useful dimension and measure columns from dataSource. You can only use columns in Column Information and do not assume non-existent columns. If the existing columns can't meet user's command, just select the most related columns in Column Information. -3. Use the original dimension columns without any process. Aggregate the measure columns using aggregation methods supported in SimQuery. Don't use unsupported methods. If current keywords and methods can't meet your needs, just simple select the column without any process. +2. Select useful dimension and measure columns from ${VMIND_DATA_SOURCE}. Don't miss some important columns such as dimensions related to date or time. You can only use columns in Column Information and do not assume non-existent columns. If the existing columns can't meet user's command, just select the most related columns in Column Information. +3. Use the original dimension columns without any process. Aggregate the measure columns using aggregation methods no matter what chart type the user has specified. Don't use unsupported methods. If current keywords and methods can't meet your needs, just simply select the column without any process. 4. Group the data using dimension columns. -5. You can also use WHERE, HAVING, ORDER BY, LIMIT in your SimQuery if necessary. Use the supported operators to finish the WHERE and HAVING of SimQuery. You can only use binary expression such as columnA = value1, sum_b > 0. You can only use dimension values appearing in the domain of dimension columns in your expression. +5. You can also use WHERE, HAVING, ORDER BY, LIMIT in your sql if necessary. Use the supported operators to finish the WHERE and HAVING. You can only use binary expression such as columnA = value1, sum_b > 0. You can only use dimension values appearing in the domain of dimension columns in your expression. Let's think step by step. -Response one JSON object without any additional words. Your JSON object must contain SimQuery and fieldInfo. +User will parse the content of your response with JSON.parse() directly without further process. Response one JSON object without any additional words. Your JSON object must contain sql and fieldInfo. Response in the following format: \`\`\` { - ${showThoughts ? 'THOUGHTS: string //your thoughts' : ''} - SimQuery: string; //your SimQuery query. Note that it's a string in a JSON object so it must be in one line without any \\n. + ${showThoughts ? 'thoughts: string //your thoughts' : ''} + sql: string; //your sql. Note that it's a string in a JSON object so it must be in one line without any \\n. fieldInfo: { fieldName: string; //name of the field. description?: string; //description of the field. If it is an aggregated field, please describe how it is generated in detail. - }[]; //array of the information about the fields in your SimQuery. Describing its aggregation method and other information of the fields. + }[]; //array of the information about the fields in your sql. Describing its aggregation method and other information of the fields. } \`\`\` @@ -238,8 +239,8 @@ Column Information: [{"fieldName":"country","type":"string","role":"dimension"}, Response: \`\`\` { - ${showThoughts ? '"THOUGHTS": string //your thoughts' : ''} - "SimQuery": "SELECT country, year, SUM(GDP) AS total_GDP FROM dataSource GROUP BY country, year ORDER BY year, total_GDP DESC", + ${showThoughts ? '"thoughts": string //your thoughts' : ''} + "sql": "SELECT \`country\`, \`year\`, SUM(\`GDP\`) AS \`total_GDP\` FROM ${VMIND_DATA_SOURCE} GROUP BY \`country\`, \`year\` ORDER BY \`year\`, \`total_GDP\` DESC", "fieldInfo": [ { "fieldName": "country", @@ -264,8 +265,8 @@ Column Information: [{"fieldName":"城市","type":"string","role":"dimension"},{ Response: \`\`\` { - ${showThoughts ? '"THOUGHTS": string //your thoughts' : ''} - "SimQuery": "SELECT 城市, SUM(\`2022年GDP（亿元）\`) as sum_2022_GDP FROM dataSource ORDER BY sum_2022_GDP DESC LIMIT 5", + ${showThoughts ? '"thoughts": string //your thoughts' : ''} + "sql": "SELECT 城市, SUM(\`2022年GDP（亿元）\`) as \`sum_2022_GDP\` FROM ${VMIND_DATA_SOURCE} ORDER BY \`sum_2022_GDP\` DESC LIMIT 5", "fieldInfo": [ { "fieldName": "城市", @@ -280,34 +281,12 @@ Response: \`\`\` ---------------------------------- -User's Command: 展示男女早餐饭量不同 -Column Information: [{"fieldName":"时间","type":"string","role":"dimension"},{"fieldName":"男_DASH_早餐","type":"int","role":"measure"},{"fieldName":"女_DASH_早餐","type":"int","role":"measure"}] - -Response: -\`\`\` -{ - ${showThoughts ? '"THOUGHTS": string //your thoughts' : ''} - "SimQuery": "SELECT \`时间\`, SUM(\`男_DASH_早餐\`) AS breakfast_amount_man, SUM(\`女_DASH_早餐\`) AS breakfast_amount_woman FROM dataSource GROUP BY \`时间\`", - "fieldInfo": [ - { - "fieldName": "gender", - "description": "The gender of the person." - }, - { - "fieldName": "breakfast_amount", - "description": "An aggregated field representing the average breakfast amount of each gender. It is generated by averaging the '男_DASH_早餐' or '女_DASH_早餐' field." - } - ] -} -\`\`\` - ----------------------------------- - You only need to return the JSON in your response directly to the user. Finish your tasks in one-step. # Constraints: -1. Write your SimQuery statement in one line without any \\n. -2. Please don't change or translate the field names in your SimQuery statement. Don't miss the GROUP BY in your query. -3. Response the JSON object directly without any other contents. Make sure it can be directly parsed by JSON.parse() in JavaScript. +1. Write your sql statement in one line without any \\n. Your sql must be executable by alasql. +2. Please don't change or translate the field names in your sql statement. Don't miss the GROUP BY in your sql. +3. Wrap all the columns with \`\` in your sql. +4. Response the JSON object directly without any other contents. Make sure it can be directly parsed by JSON.parse() in JavaScript. `; diff --git a/packages/vmind/src/gpt/dataProcess/query/astPipes.ts b/packages/vmind/src/gpt/dataProcess/query/astPipes.ts index d6011311..b3c5ce2e 100644 --- a/packages/vmind/src/gpt/dataProcess/query/astPipes.ts +++ b/packages/vmind/src/gpt/dataProcess/query/astPipes.ts @@ -5,16 +5,15 @@ import { FilterNode, FilterNodeType, FilterOperator, - HavingCondition, OrderType, Query, - WhereCondition, - WhereFilterNode + WhereCondition } from '@visactor/calculator'; -import { ASTParserContext, ASTParserPipe, SQLAst } from './type'; -import { checkIsColumnNode, getOriginalString, toFirstUpperCase } from './utils'; +import { ASTParserContext, ASTParserPipe } from './type'; +import { checkIsColumnNode, toFirstUpperCase } from './utils'; import { SimpleFieldInfo } from '../../../typings'; import { isArray } from 'lodash'; +import { replaceString } from '../../../common/dataProcess/utils'; export const from: ASTParserPipe = (query: Partial, context: ASTParserContext) => { const { dataSource, fieldInfo } = context; @@ -40,7 +39,7 @@ const parseAggrFunc = ( console.error('unsupported aggr func!'); } else if (expr && checkIsColumnNode(expr, columns, fieldInfo)) { const columnName = expr.column ?? expr.value; - result.column = getOriginalString(columnName, replaceMap); + result.column = replaceString(columnName, replaceMap); } result.aggregate = { distinct: Boolean(distinct), @@ -88,15 +87,15 @@ const parseSQLExpr = ( const columnNode = [left, right].find(n => checkIsColumnNode(n, columns, fieldInfo)); if (columnNode) { const columnName = (columnNode as ColumnRef).column ?? (columnNode as any).value; - result.column = getOriginalString(columnName, replaceMap); + result.column = replaceString(columnName, replaceMap); } const valueNode = [left, right].find(n => !checkIsColumnNode(n, columns, fieldInfo) && n.type !== 'aggr_func'); if (valueNode) { const valueName = (valueNode as Value).value; if (!isArray(valueName)) { - result.value = getOriginalString(valueName, replaceMap); + result.value = replaceString(valueName, replaceMap); } else { - result.value = valueName.map(v => getOriginalString(v.value, replaceMap)); + result.value = valueName.map(v => replaceString(v.value, replaceMap)); } } const aggrNode: any = [left, right].find(n => n.type === 'aggr_func'); @@ -140,8 +139,8 @@ export const groupBy: ASTParserPipe = (query: Partial, context: ASTParser } return { ...query, - groupBy: (groupby ?? []).map((group: any) => getOriginalString(group.column ?? group.value, replaceMap)) - }; + groupBy: (groupby ?? []).map((group: any) => replaceString(group.column ?? group.value, replaceMap)) + } as any; }; export const select: ASTParserPipe = (query: Partial, context: ASTParserContext) => { @@ -161,14 +160,14 @@ export const select: ASTParserPipe = (query: Partial, context: ASTParserC const result: any = {}; const { as, expr } = column; if (checkIsColumnNode(expr, columnAlias, fieldInfo)) { - result.column = getOriginalString(expr.column ?? expr.value, replaceMap); + result.column = replaceString(expr.column ?? expr.value, replaceMap); } else if (expr.type === 'aggr_func') { const aggrFuncConf: any = parseAggrFunc(expr, columnAlias, fieldInfo, replaceMap); result.column = aggrFuncConf.column; result.aggregate = aggrFuncConf.aggregate; } if (as) { - result.alias = getOriginalString(as, replaceMap); + result.alias = replaceString(as, replaceMap); } return result; }) @@ -204,7 +203,7 @@ export const orderBy: any = (query: Partial, context: ASTParserContext) = const { type, expr } = orderInfo; if (checkIsColumnNode(expr, query.select.columns, fieldInfo)) { const columnName = expr.column ?? expr.value; - result.column = getOriginalString(columnName, replaceMap); + result.column = replaceString(columnName, replaceMap); } else { const orderConfig = parseAggrFunc(expr, query.select.columns, fieldInfo, replaceMap); result.column = orderConfig.column; diff --git a/packages/vmind/src/gpt/dataProcess/query/queryDataset.ts b/packages/vmind/src/gpt/dataProcess/query/queryDataset.ts index 07f2c466..64787517 100644 --- a/packages/vmind/src/gpt/dataProcess/query/queryDataset.ts +++ b/packages/vmind/src/gpt/dataProcess/query/queryDataset.ts @@ -1,19 +1,9 @@ import { DataItem, ILLMOptions, SimpleFieldInfo } from '../../../typings'; -import NodeSQLParser from 'node-sql-parser'; -import { - mergeMap, - parseGPTQueryResponse, - parseRespondField, - patchQueryInput, - preprocessSQL, - replaceOperator -} from './utils'; -import { parseSqlAST } from './parseSqlAST'; -import { isArray } from 'lodash'; -import { DataQueryResponse, SQLAst } from './type'; -import { Query, query } from '@visactor/calculator'; +import { parseGPTQueryResponse, parseRespondField, patchQueryInput } from './utils'; +import { DataQueryResponse } from './type'; import { parseGPTResponse as parseGPTResponseAsJSON, requestGPT } from '../../utils'; import { getQueryDatasetPrompt } from '../prompts'; +import { queryDataset } from '../../../common/dataProcess/dataQuery'; /** * query the source dataset according to user's input and fieldInfo to get aggregated dataset @@ -28,25 +18,20 @@ export const queryDatasetWithGPT = async ( sourceDataset: DataItem[], options: ILLMOptions ) => { - const { validFieldInfo, replaceMap: operatorReplaceMap } = replaceOperator(fieldInfo); const patchedInput = patchQueryInput(userInput); - const { SimQuery, fieldInfo: responseFieldInfo } = await getQuerySQL(patchedInput, validFieldInfo, options); - const { validStr, replaceMap: preprocessReplaceMap } = preprocessSQL(SimQuery, fieldInfo); - const replaceMap = mergeMap(preprocessReplaceMap, operatorReplaceMap); - const parser = new NodeSQLParser.Parser(); + const { sql, fieldInfo: responseFieldInfo, usage } = await getQuerySQL(patchedInput, fieldInfo, options); - const ast = parser.astify(validStr); - const queryObject = parseSqlAST((isArray(ast) ? ast[0] : ast) as SQLAst, sourceDataset, fieldInfo, replaceMap); + const datasetAfterQuery = queryDataset(sql, sourceDataset, fieldInfo); - const dataset = query(queryObject as Query); - - const fieldInfoNew = parseRespondField(responseFieldInfo, dataset, replaceMap); - if (dataset.length === 0) { + const fieldInfoNew = parseRespondField(responseFieldInfo, datasetAfterQuery); + if (datasetAfterQuery.length === 0) { console.warn('empty dataset after query!'); } + return { - dataset: dataset.length === 0 ? sourceDataset : dataset, - fieldInfo: dataset.length === 0 ? fieldInfo : fieldInfoNew + dataset: datasetAfterQuery.length === 0 ? sourceDataset : datasetAfterQuery, + fieldInfo: datasetAfterQuery.length === 0 ? fieldInfo : fieldInfoNew, + usage }; }; @@ -62,12 +47,15 @@ const getQuerySQL = async (userInput: string, fieldInfo: SimpleFieldInfo[], opti const QueryDatasetPrompt = getQueryDatasetPrompt(options.showThoughts ?? true); const dataProcessRes = await requestFunc(QueryDatasetPrompt, queryDatasetMessage, options); const dataQueryResponse: DataQueryResponse = parseGPTResponseAsJSON(dataProcessRes); - const { SimQuery, fieldInfo: responseFiledInfo } = dataQueryResponse; - if (!SimQuery || !responseFiledInfo) { + const { sql, fieldInfo: responseFiledInfo } = dataQueryResponse; + if (!sql || !responseFiledInfo) { //try to parse the response with another format const choices = dataProcessRes.choices; const content = choices[0].message.content; - return parseGPTQueryResponse(content); + return { + ...parseGPTQueryResponse(content), + usage: dataProcessRes.usage + }; } - return dataQueryResponse; + return { ...dataQueryResponse, usage: dataProcessRes.usage }; }; diff --git a/packages/vmind/src/gpt/dataProcess/query/type.ts b/packages/vmind/src/gpt/dataProcess/query/type.ts index 486ecb2d..0c740097 100644 --- a/packages/vmind/src/gpt/dataProcess/query/type.ts +++ b/packages/vmind/src/gpt/dataProcess/query/type.ts @@ -14,6 +14,6 @@ export type ASTParserContext = { export type DataQueryResponse = { THOUGHT?: string; - SimQuery: string; + sql: string; fieldInfo: { fieldName: string; description?: string }[]; }; diff --git a/packages/vmind/src/gpt/dataProcess/query/utils.ts b/packages/vmind/src/gpt/dataProcess/query/utils.ts index 3e928e4a..6be55806 100644 --- a/packages/vmind/src/gpt/dataProcess/query/utils.ts +++ b/packages/vmind/src/gpt/dataProcess/query/utils.ts @@ -1,60 +1,16 @@ -import { isArray, isString } from 'lodash'; +import { isArray } from 'lodash'; import JSON5 from 'json5'; import { Query } from '@visactor/calculator'; -import { detectFieldType } from '../../../common/dataProcess/utils'; +import { + detectFieldType, + generateRandomString, + mergeMap, + replaceNonASCIICharacters +} from '../../../common/dataProcess/utils'; import { DataItem, SimpleFieldInfo } from '../../../typings'; import { ASTParserContext, ASTParserPipe } from './type'; -function generateRandomString(len: number) { - const chars = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz'; - let result = ''; - for (let i = 0; i < len; i++) { - result += chars.charAt(Math.floor(Math.random() * chars.length)); - } - return result; -} - -const swapMap = (map: Map) => { - //swap the map - const swappedMap = new Map(); - - // Swap key with value - map.forEach((value, key) => { - swappedMap.set(value, key); - }); - return swappedMap; -}; - -/** - * replace operator and reserved words inside the field name in the sql str - * @param fieldInfo - */ -export const replaceOperator = (fieldInfo: SimpleFieldInfo[]) => { - const operatorMap = { - '+': `_PLUS_`, - '-': `_DASH_`, - '*': `_ASTERISK_`, - '/': `_SLASH_` - }; - const replaceMap = new Map(); - const validFieldInfo = fieldInfo.map((field: SimpleFieldInfo) => { - const { fieldName } = field; - let validFieldName = fieldName; - Object.keys(operatorMap).forEach(operator => { - validFieldName = validFieldName.split(operator).join(operatorMap[operator]); - if (validFieldName !== fieldName) { - replaceMap.set(validFieldName, fieldName); - } - }); - return { - ...field, - fieldName: validFieldName - }; - }); - return { validFieldInfo, replaceMap }; -}; - /** * replace invalid characters in sql str and get the replace map * @param sql @@ -91,51 +47,6 @@ export const preprocessSQL = (sql: string, fieldInfo: SimpleFieldInfo[]) => { return { validStr, replaceMap: mergedMap }; }; -/** - * replace all the non-ascii characters in the sql str into valid strings. - * @param str - * @returns - */ -export const replaceNonASCIICharacters = (str: string) => { - const nonAsciiCharMap = new Map(); - - const newStr = str.replace(/([^\x00-\x7F]+)/g, m => { - let replacement; - if (nonAsciiCharMap.has(m)) { - replacement = nonAsciiCharMap.get(m); - } else { - replacement = generateRandomString(10); - nonAsciiCharMap.set(m, replacement); - } - return replacement; - }); - - const swappedMap = swapMap(nonAsciiCharMap); - - return { validStr: newStr, replaceMap: swappedMap }; -}; - -/** - * replace random strings into its original string according to replaceMap - * @param str - * @param replaceMap - * @returns - */ -export const getOriginalString = (str: string, replaceMap: Map) => { - if (!isString(str)) { - return str; - } - if (replaceMap.has(str)) { - return replaceMap.get(str); - } else { - //Some string may be linked by ASCII characters as non-ASCII characters.Traversing the replaceMap and replaced it to the original character - const replaceKeys = [...replaceMap.keys()]; - return replaceKeys.reduce((prev, cur) => { - return prev.replace(new RegExp(cur, 'g'), replaceMap.get(cur)); - }, str); - } -}; - export const addQuotes = (sqlString: string) => { let newSQLString = ''; let startIdx = 0; @@ -202,38 +113,19 @@ export const checkIsColumnNode = (node: any, columns: any, fieldInfo: SimpleFiel */ export const parseRespondField = ( responseFieldInfo: { fieldName: string; description?: string }[], - dataset: DataItem[], - replaceMap: Map + dataset: DataItem[] ) => responseFieldInfo.map(field => ({ ...field, - ...detectFieldType(dataset, field.fieldName), - fieldName: getOriginalString(field.fieldName, replaceMap) + ...detectFieldType(dataset, field.fieldName) })); -/** - * merge two maps - * @param map1 - * @param map2 - * @returns - */ -export const mergeMap = (map1: Map, map2: Map) => { - // merge map2 into map1 - map2.forEach((value, key) => { - map1.set(key, value); - }); - return map1; -}; - export const patchQueryInput = (userInput: string) => { - return ( - userInput + - " Don't use unsupported keywords and methods in the SELECT of SimQuery. Don't use non-existent columns and dimension values in the WHERE of SimQuery." - ); + return userInput; }; export const parseGPTQueryResponse = (response: string) => { - const SimQuery = response.match(/SimQuery:\n?```(.*?)```/s)[1]; + const sql = response.match(/sql:\n?```(.*?)```/s)[1]; const fieldInfoStr = response.match(/fieldInfo:\n?```(.*?)```/s)[1]; let fieldInfo = []; try { @@ -248,7 +140,7 @@ export const parseGPTQueryResponse = (response: string) => { fieldInfo = JSON5.parse(`[${fieldInfoStr}]`); } return { - SimQuery, + sql, fieldInfo }; }; diff --git a/packages/vmind/src/gpt/utils.ts b/packages/vmind/src/gpt/utils.ts index 0cf04d94..a83b1417 100644 --- a/packages/vmind/src/gpt/utils.ts +++ b/packages/vmind/src/gpt/utils.ts @@ -2,6 +2,7 @@ import { GPTDataProcessResult, ILLMOptions, LLMResponse } from '../typings'; import axios from 'axios'; import JSON5 from 'json5'; import { omit } from 'lodash'; +import { matchJSONStr } from '../common/utils'; export const requestGPT = async ( prompt: string, @@ -30,10 +31,11 @@ export const requestGPT = async ( } ], max_tokens: options?.max_tokens ?? 2000, - temperature: options?.temperature ?? 0 + temperature: options?.temperature ?? 0, + stream: false //response_format: { type: 'json_object' } //Only models after gpt-3.5-turbo-1106 support this parameter. } - }).then(response => response.data); + }).then((response: any) => response.data); return res; } catch (err: any) { @@ -67,18 +69,23 @@ export const parseGPTJson = (JsonStr: string, prefix?: string) => { }; export const parseGPTResponse = (GPTRes: LLMResponse) => { - if (GPTRes.error) { + try { + if (GPTRes.error) { + return { + error: true, + ...GPTRes.error + }; + } + const choices = GPTRes.choices; + const content = choices[0].message.content; + const jsonStr = matchJSONStr(content); + + const resJson: GPTDataProcessResult = parseGPTJson(jsonStr, '```'); + return resJson; + } catch (err: any) { return { error: true, - ...GPTRes.error + message: err.message }; } - const choices = GPTRes.choices; - const content = choices[0].message.content; - const resJson: GPTDataProcessResult = parseGPTJson(content, '```'); - return resJson; -}; - -export const replaceAll = (originStr: string, replaceStr: string, newStr: string) => { - return originStr.split(replaceStr).join(newStr); }; diff --git a/packages/vmind/src/skylark/chart-generation/NLToChart.ts b/packages/vmind/src/skylark/chart-generation/NLToChart.ts index d1f2c7b6..08ca2958 100644 --- a/packages/vmind/src/skylark/chart-generation/NLToChart.ts +++ b/packages/vmind/src/skylark/chart-generation/NLToChart.ts @@ -2,37 +2,63 @@ import { chartAdvisorHandler } from '../../common/chartAdvisor'; import { getSchemaFromFieldInfo } from '../../common/schema'; import { SUPPORTED_CHART_LIST, checkChartTypeAndCell, vizDataToSpec } from '../../common/vizDataToSpec'; import { DataItem, ILLMOptions, SimpleFieldInfo, VizSchema } from '../../typings'; -import { getStrFromArray, getStrFromDict, patchChartTypeAndCell, requestSkyLark } from './utils'; +import { getStrFromArray, getStrFromDict, requestSkyLark } from './utils'; import { getChartRecommendPrompt, getFieldMapPrompt } from './prompts'; import { parseSkylarkResponse } from '../utils'; import { estimateVideoTime } from '../../common/vizDataToSpec/utils'; import { ChartFieldInfo, chartRecommendConstraints, chartRecommendKnowledge } from './constants'; import { omit } from 'lodash'; +import { calculateTokenUsage } from '../../common/utils'; +import { queryDatasetWithSkylark } from '../dataProcess/query/queryDataset'; +import { patchChartTypeAndCell } from './patch'; export const generateChartWithSkylark = async ( userPrompt: string, //user's intent of visualization, usually aspect in data that they want to visualize - fieldInfo: SimpleFieldInfo[], + propsFieldInfo: SimpleFieldInfo[], propsDataset: DataItem[], options: ILLMOptions, + enableDataQuery = true, colorPalette?: string[], animationDuration?: number ) => { - const schema = getSchemaFromFieldInfo(fieldInfo); - const colors = colorPalette; + let queryDatasetUsage; + let advisorUsage; let chartType; let cell; let dataset: DataItem[] = propsDataset; + let fieldInfo: SimpleFieldInfo[] = propsFieldInfo; let chartSource: string = options.model; + + try { + if (enableDataQuery) { + const { + dataset: queryDataset, + fieldInfo: fieldInfoNew, + usage + } = await queryDatasetWithSkylark(userPrompt, fieldInfo, propsDataset, options); + dataset = queryDataset; + fieldInfo = fieldInfoNew; + queryDatasetUsage = usage; + } + } catch (err) { + console.error('data query error!'); + console.error(err); + } + + const schema = getSchemaFromFieldInfo(fieldInfo); + const colors = colorPalette; + try { // throw 'test chartAdvisorHandler'; const resJson: any = await chartAdvisorSkylark(schema, fieldInfo, userPrompt, options); - + advisorUsage = resJson.usage; const chartTypeRes = resJson.chartType.toUpperCase(); const cellRes = resJson['cell']; const patchResult = patchChartTypeAndCell(chartTypeRes, cellRes, dataset, fieldInfo); - if (checkChartTypeAndCell(patchResult.chartTypeNew, patchResult.cellNew, fieldInfo)) { + if (checkChartTypeAndCell(patchResult.chartTypeNew, patchResult.cellNew, patchResult.fieldInfoNew)) { chartType = patchResult.chartTypeNew; cell = patchResult.cellNew; + dataset = patchResult.datasetNew; } } catch (err) { console.warn(err); @@ -53,7 +79,9 @@ export const generateChartWithSkylark = async ( spec.background = '#00000033'; return { chartSource, + chartType, spec, + usage: calculateTokenUsage([queryDatasetUsage, advisorUsage]), time: estimateVideoTime(chartType, spec, animationDuration ? animationDuration * 1000 : undefined) }; }; @@ -74,10 +102,13 @@ export const chartAdvisorSkylark = async ( chartRecommendConstraintsStr, options.showThoughts ?? true ); - const chartRecommendRes = await requestSkyLark(chartRecommendPrompt, userMessage, options); + + const requestFunc = options.customRequestFunc?.chartAdvisor ?? requestSkyLark; + + const chartRecommendRes = await requestFunc(chartRecommendPrompt, userMessage, options); const chartRecommendResJSON = parseSkylarkResponse(chartRecommendRes); if (chartRecommendResJSON.error) { - throw Error('Network Error!'); + throw Error(chartRecommendResJSON.message); } if (!SUPPORTED_CHART_LIST.includes(chartRecommendResJSON['charttype'])) { throw Error('Unsupported Chart Type. Please Change User Input'); @@ -98,7 +129,7 @@ export const chartAdvisorSkylark = async ( options.showThoughts ?? true ); - const fieldMapRes = await requestSkyLark(fieldMapPrompt, userMessage, options); + const fieldMapRes = await requestFunc(fieldMapPrompt, userMessage, options); const fieldMapResJson = parseSkylarkResponse(fieldMapRes); if (fieldMapResJson.error) { throw Error('Network Error!'); @@ -106,10 +137,7 @@ export const chartAdvisorSkylark = async ( return { chartType, - cell: omit(fieldMapResJson, ['thoughts', 'usage']) + cell: omit(fieldMapResJson, ['thoughts', 'usage']), + usage: calculateTokenUsage([chartRecommendRes.usage, fieldMapRes.usage]) }; }; - -export const getFieldMapSkylark = async (chartType: string, userInput: string) => {}; - -const getChartChannel = (chartType: string) => {}; diff --git a/packages/vmind/src/skylark/chart-generation/constants.ts b/packages/vmind/src/skylark/chart-generation/constants.ts index d2a33a06..51ff3c7b 100644 --- a/packages/vmind/src/skylark/chart-generation/constants.ts +++ b/packages/vmind/src/skylark/chart-generation/constants.ts @@ -21,12 +21,12 @@ export const ChartFieldInfo: ChannelInfo = { }, 'PIE CHART': { visualChannels: { - angle: 'angle of sectors in the pie chart. Only number fields', + value: "angle of sectors in the pie chart. Only number fields. Can't be empty.", color: "color of sectors in the pie chart. Used to distinguish different sectors. Only string fields. Can't be empty." }, responseDescription: { - angle: 'field assigned to angle channel', + value: 'field assigned to angle channel', color: 'field assigned to color channel' }, knowledge: ['Only string fields can be used in color channel.'] @@ -34,7 +34,7 @@ export const ChartFieldInfo: ChannelInfo = { 'LINE CHART': { visualChannels: { x: "x-axis of line chart. Can't be empty. Only string fields", - y: "y-axis of line chart. Can't be empty. Only number fields", + y: "y-axis of line chart. Can't be empty. Only number fields. Use array if there are more than one number fields need to show.", color: 'color channel of line chart. Used to distinguish different lines. Only string fields. Can be empty if no suitable field.' }, @@ -43,7 +43,10 @@ export const ChartFieldInfo: ChannelInfo = { y: 'field assigned to y channel', color: 'field assigned to color channel. Can be empty if no suitable field.' }, - knowledge: ['Only string fields can be used in color channel.'] + knowledge: [ + 'Only string fields can be used in color channel.', + 'Use an array in y-axis if you want to assign more than one fields in y-axis.' + ] }, 'SCATTER PLOT': { visualChannels: { diff --git a/packages/vmind/src/skylark/chart-generation/patch.ts b/packages/vmind/src/skylark/chart-generation/patch.ts new file mode 100644 index 00000000..1f60772e --- /dev/null +++ b/packages/vmind/src/skylark/chart-generation/patch.ts @@ -0,0 +1,237 @@ +import { isArray, isString } from 'lodash'; +import { Cell, DataItem, DataType, PatchContext, PatchPipeline, ROLE, SimpleFieldInfo } from '../../typings'; +import { execPipeline } from '../../common/utils'; +import { foldDatasetByYField } from '../../common/vizDataToSpec/utils'; +import { FOLD_NAME, FOLD_VALUE } from '@visactor/chart-advisor'; + +const matchFieldWithoutPunctuation = (field: string, fieldList: string[]): string | undefined => { + //try to match the field without punctuation + //return undefined if no field is match + if (!field) { + return field; + } + const punctuationRegex = /[.,\/#!$%\^&\*;:{}=\-_`~()\s]/g; + const pureFieldStr = field.replace(punctuationRegex, ''); + let matchedField = undefined; + fieldList.some((f: string) => { + const pureStr = f.replace(punctuationRegex, ''); + if (pureStr === pureFieldStr) { + matchedField = f; + return true; + } + return false; + }); + return matchedField; +}; + +const patchNullField: PatchPipeline = (context: PatchContext, _originalContext: PatchContext) => { + const { fieldInfo, cell } = context; + const cellNew = { ...cell }; + + const columns = fieldInfo.map(field => field.fieldName); + + //set null field to undefined + Object.keys(cellNew).forEach(key => { + const value = cellNew[key]; + if (isArray(value)) { + cellNew[key] = value + .map(v => (columns.includes(v) ? v : matchFieldWithoutPunctuation(v, columns))) + .filter(Boolean); + } else if (!columns.includes(value) || value === '') { + cellNew[key] = matchFieldWithoutPunctuation(cellNew[key], columns); + } + }); + + return { + ...context, + cell: cellNew + }; +}; + +const patchField: PatchPipeline = (context: PatchContext, _originalContext: PatchContext) => { + const { fieldInfo, cell } = context; + const fieldNames = fieldInfo.map(field => field.fieldName); + const cellNew = { ...cell }; + Object.keys(cellNew).forEach(key => { + const value = cellNew[key]; + if (isString(value) && (value ?? '').includes(',')) { + const newValue = (value as string).split(',').map(f => f.trim()); + if (newValue.every(f => fieldNames.includes(f))) { + cellNew[key] = newValue; + } + } + }); + return { + ...context, + cell: cellNew + }; +}; + +const patchColorField: PatchPipeline = (context: PatchContext, _originalContext: PatchContext) => { + const { chartType, fieldInfo, cell } = context; + const cellNew = { ...cell }; + const { color } = cellNew; + let chartTypeNew = chartType; + if (color) { + const colorField = fieldInfo.find(f => f.fieldName === color); + if (colorField && colorField.role === ROLE.MEASURE) { + cellNew.color = undefined; + if (['BAR CHART', 'LINE CHART', 'DUAL AXIS CHART'].includes(chartTypeNew)) { + cellNew.y = [cellNew.y, color].flat(); + if (chartTypeNew === 'DUAL AXIS CHART' && cellNew.y.length > 2) { + chartTypeNew = 'BAR CHART'; + } + } + } + } + + return { + ...context, + cell: cellNew + }; +}; + +const patchRadarChart: PatchPipeline = (context: PatchContext, _originalContext: PatchContext) => { + const { chartType, cell } = context; + + if (chartType === 'RADAR CHART') { + const cellNew = { + x: cell.angle, + y: cell.value, + color: cell.color + }; + + return { + ...context, + cell: cellNew + }; + } + return context; +}; + +const patchBoxPlot: PatchPipeline = (context: PatchContext, _originalContext: PatchContext) => { + const { chartType, cell } = context; + + if (chartType === 'BOX PLOT') { + const { x, min, q1, median, q3, max } = cell as any; + const cellNew = { + x, + y: [min, q1, median, q3, max].filter(Boolean) + }; + return { + ...context, + cell: cellNew + }; + } + return context; +}; + +const patchBarChart: PatchPipeline = (context: PatchContext, _originalContext: PatchContext) => { + const { chartType, cell, fieldInfo, dataset } = context; + const chartTypeNew = chartType; + const cellNew = { ...cell }; + let datasetNew = dataset; + if (chartTypeNew === 'BAR CHART' || chartTypeNew === 'LINE CHART') { + if (isArray(cellNew.y) && cellNew.y.length > 1) { + datasetNew = foldDatasetByYField(datasetNew, cellNew.y, fieldInfo); + cellNew.y = FOLD_VALUE.toString(); + cellNew.color = FOLD_NAME.toString(); + } + } + return { + ...context, + chartType: chartTypeNew, + cell: cellNew, + dataset: datasetNew + }; +}; + +const patchDynamicBarChart: PatchPipeline = (context: PatchContext, _originalContext: PatchContext) => { + const { chartType, cell, fieldInfo } = context; + const cellNew = { + ...cell + }; + + if (chartType === 'DYNAMIC BAR CHART') { + if (!cellNew.time || cellNew.time === '' || cellNew.time.length === 0) { + const flattenedXField = Array.isArray(cellNew.x) ? cellNew.x : [cellNew.x]; + const usedFields = Object.values(cellNew).filter(f => !Array.isArray(f)); + usedFields.push(...flattenedXField); + const remainedFields = fieldInfo.filter(f => !usedFields.includes(f.fieldName)); + + //动态条形图没有time字段，选择一个离散字段作为time + const timeField = remainedFields.find(f => { + return f.type === DataType.DATE; + }); + if (timeField) { + cellNew.time = timeField.fieldName; + } else { + const stringField = remainedFields.find(f => { + return f.type === DataType.STRING; + }); + if (stringField) { + cellNew.time = stringField.fieldName; + } + } + } + } + return { + ...context, + cell: cellNew + }; +}; + +const patchArrayField: PatchPipeline = (context: PatchContext, _originalContext: PatchContext) => { + const { cell } = context; + const cellNew = { + ...cell + }; + //only x and y field can be array + Object.keys(cellNew).forEach(key => { + if (key !== 'x' && key !== 'y' && isArray(cellNew[key])) { + cellNew[key] = cellNew[key][0]; + } + }); + + return { + ...context, + cell: cellNew + }; +}; + +const patchPipelines: PatchPipeline[] = [ + patchNullField, + patchField, + patchColorField, + patchRadarChart, + patchBoxPlot, + patchBarChart, + patchDynamicBarChart, + patchArrayField +]; + +export const patchChartTypeAndCell = ( + chartTypeRes: string, + cellRes: Cell, + dataset: DataItem[], + fieldInfo: SimpleFieldInfo[] +) => { + const context = { + chartType: chartTypeRes, + cell: cellRes, + dataset, + fieldInfo + }; + const { + chartType: chartTypeNew, + cell: cellNew, + dataset: datasetNew, + fieldInfo: fieldInfoNew + } = execPipeline(context, patchPipelines, context); + return { + chartTypeNew, + cellNew, + datasetNew, + fieldInfoNew + }; +}; diff --git a/packages/vmind/src/skylark/chart-generation/utils.ts b/packages/vmind/src/skylark/chart-generation/utils.ts index 34117f0d..98d86d63 100644 --- a/packages/vmind/src/skylark/chart-generation/utils.ts +++ b/packages/vmind/src/skylark/chart-generation/utils.ts @@ -1,110 +1,6 @@ import axios from 'axios'; -import { DataItem, ILLMOptions, LLMResponse, SimpleFieldInfo } from '../../typings'; -import { detectAxesType } from '../../common/vizDataToSpec/utils'; -import { isArray, omit } from 'lodash'; - -const matchFieldWithoutPunctuation = (field: string, fieldList: string[]): string | undefined => { - //try to match the field without punctuation - //return undefined if no field is match - if (!field) { - return field; - } - const punctuationRegex = /[.,\/#!$%\^&\*;:{}=\-_`~()\s]/g; - const pureFieldStr = field.replace(punctuationRegex, ''); - let matchedField = undefined; - fieldList.some((f: string) => { - const pureStr = f.replace(punctuationRegex, ''); - if (pureStr === pureFieldStr) { - matchedField = f; - return true; - } - return false; - }); - return matchedField; -}; - -export const patchChartTypeAndCell = ( - chartTypeRes: any, - cellRes: any, - dataset: DataItem[], - fieldInfo: SimpleFieldInfo[] -) => { - let chartTypeNew = chartTypeRes; - let cellNew = { ...cellRes }; - const columns = fieldInfo.map(field => field.fieldName); - - //set null field to undefined - Object.keys(cellNew).forEach(key => { - const value = cellNew[key]; - if (isArray(value)) { - cellNew[key] = value - .map(v => (columns.includes(v) ? v : matchFieldWithoutPunctuation(v, columns))) - .filter(Boolean); - } else if (!columns.includes(value) || value === '') { - cellNew[key] = matchFieldWithoutPunctuation(cellNew[key], columns); - } - }); - - if (chartTypeRes === 'RADAR CHART') { - cellNew = { - x: cellRes.angle, - y: cellRes.value, - color: cellRes.color - }; - } else if (chartTypeRes === 'BOX PLOT') { - const { x, min, q1, median, q3, max } = cellRes; - cellNew = { - x, - y: [min, q1, median, q3, max].filter(Boolean) - }; - } else if (chartTypeRes === 'BAR CHART') { - if (isArray(cellRes.y) && cellRes.y.length === 2) { - chartTypeNew = 'DUAL AXIS CHART'; - } else if ((cellRes.y ?? '').includes(',')) { - const yNew = cellRes.y.split(','); - if (yNew.length === 2) { - chartTypeNew = 'DUAL AXIS CHART'; - cellNew = { - ...cellRes, - y: yNew - }; - } - } - } else if (chartTypeRes === 'DYNAMIC BAR CHART') { - if (!cellNew.time || cellNew.time === '' || cellNew.time.length === 0) { - const flattenedXField = Array.isArray(cellNew.x) ? cellNew.x : [cellNew.x]; - const usedFields = Object.values(cellNew).filter(f => !Array.isArray(f)); - usedFields.push(...flattenedXField); - const dataFields = Object.keys(dataset[0]); - const remainedFields = dataFields.filter(f => !usedFields.includes(f)); - - //动态条形图没有time字段，选择一个离散字段作为time - const timeField = remainedFields.find(f => { - const fieldType = detectAxesType(dataset, f); - return fieldType === 'band'; - }); - if (timeField) { - cellNew.time = timeField; - } else { - cellNew.time = remainedFields[0]; - } - } - return { - chartTypeNew: chartTypeRes, - cellNew - }; - } - //only x and y field can be array - Object.keys(cellNew).forEach(key => { - if (key !== 'x' && key !== 'y' && isArray(cellNew[key])) { - cellNew[key] = cellNew[key][0]; - } - }); - return { - chartTypeNew, - cellNew - }; -}; +import { ILLMOptions, LLMResponse } from '../../typings'; +import { omit } from 'lodash'; /** * @@ -134,7 +30,8 @@ export const requestSkyLark = async (prompt: string, message: string, options: I } ], max_tokens: options?.max_tokens ?? 500, - temperature: options?.temperature ?? 0 + temperature: options?.temperature ?? 0, + stream: false } }).then(response => response.data); diff --git a/packages/vmind/src/skylark/dataProcess/query/prompts.ts b/packages/vmind/src/skylark/dataProcess/query/prompts.ts new file mode 100644 index 00000000..973d32e5 --- /dev/null +++ b/packages/vmind/src/skylark/dataProcess/query/prompts.ts @@ -0,0 +1,76 @@ +import { VMIND_DATA_SOURCE } from '../../../common/dataProcess/dataQuery'; + +export const getQueryDatasetPrompt = ( + showThoughts: boolean +) => `您是一位数据分析的专家。这是一个名为${VMIND_DATA_SOURCE}的原始数据集。用户会告诉您他的命令和${VMIND_DATA_SOURCE}的列信息。您的任务是根据指令生成一个sql和fieldInfo。只返回一个JSON对象。 + +# SQL语句编写要求 +- 您需要编写一个标准的sql语句。 +- 所有的度量列必须被聚合，即使用户没有要求你这样做。支持的聚合函数：["MAX()", "MIN()", "SUM()", "COUNT()", "AVG()"] +- 支持的sql关键字：["SELECT", "FROM", "WHERE", "GROUP BY", "HAVING", "ORDER BY", "LIMIT", "DISTINCT"]. +- 不要使用不支持的关键词，如：WITHIN, FIELD。不要使用不支持的聚合函数，如：PERCENTILE_CONT, PERCENTILE。不要使用不支持的操作符。我们将使用alasql执行您的sql。不支持的关键词、函数和操作符会导致系统崩溃。 +- 使用\` \`包裹sql中的所有列名 +- 让你的sql尽可能简单。 + +您需要按照以下步骤编写sql语句。 + +# 步骤 +1. 从用户的指令中提取与数据相关的部分。忽略其他与数据无关的部分。 +2. 根据列的名称和类型，推断${VMIND_DATA_SOURCE}中与用户指令有关的列，并将其添加到SELECT中。尽可能多地选择相关列，不要遗漏一些关键的列，比如与日期相关的维度等。你只能使用Column Information中提到的列，不要假设不存在的列。如果现有的列不能满足用户的命令，选择Column Information中最相关的列。 +3. 不论用户指定了哪种图表类型，将所选择的度量列使用聚合函数聚合，即使你推断它们不适合被聚合，即使用户没有要求你这样做。如果你不确定使用哪个聚合函数，使用SUM()。不要使用不支持的聚合函数。 +4. 使用维度列对数据进行分组。 +5. 在您的sql中，如有必要，您也可以使用WHERE, HAVING, ORDER BY, LIMIT。使用支持的操作符完成WHERE和HAVING。只能使用如columnA = value1，sum_b > 0的二元表达式。在您的表达式中，只能使用在维度列的domain中出现的维度值。 + +让我们一步一步思考。不要忘了将所有度量列聚合。 + +用户将会直接使用JSON.parse()解析您返回的内容，只返回一个不带任何额外内容的JSON对象。您的JSON对象必须包含sql和fieldInfo。 + +请按以下格式回复： +\`\`\` +{ +${showThoughts ? 'thoughts: string //你的想法' : ''} +sql: string; //你的sql。注意，这是一个JSON对象中的字符串，所以必须是一行，不含任何\\n。 +fieldInfo: { +fieldName: string; //字段名。 +type: string; //字段类型，string，int，date或float。 +}[]; //您的sql中字段信息的数组。描述其名称和类型。 +} +\`\`\` + +#Examples: + +User's Command: Show me the change of the GDP rankings of each country. +Column Information: [{"fieldName":"country","type":"string","role":"dimension","domain":["USA", "China", "England"]},{"fieldName":"continent","type":"string","role":"dimension","domain":["North America","Asia","Europe"]},{"fieldName":"GDP","type":"float","role":"measure","domain":[2780,617030]},{"fieldName":"year","type":"int","role":"measure","domain":[1973,2018]}] + +Response: +\`\`\` +{ + ${showThoughts ? '"thoughts": string //your thoughts' : ''} + "sql": "SELECT \`country\`, \`year\`, SUM(\`GDP\`) AS \`total_GDP\` FROM ${VMIND_DATA_SOURCE} GROUP BY \`country\`, \`year\` ORDER BY \`year\`, \`total_GDP\` DESC", + "fieldInfo": [ + { + "fieldName": "country", + "type": "string" + }, + { + "fieldName": "year", + "type": "date" + }, + { + "fieldName": "total_GDP", + "type": "int" + } + ] +} +\`\`\` + +在上面这个例子中，用户想要展示不同国家GDP排名的变化，相关列有country和GDP。用户需要一个年份列才能展示“变化”，因此我们还需要选择year。GDP是一个指标列，因此我们要将它聚合。从用户输入中无法推断聚合方式，因此使用SUM()。您只需要将生成的JSON返回给用户。 + +一步完成您的任务。 + +# 约束： +- 在一行内写出您的sql语句，不要有任何\\n。您的sql必须能够由alasql执行。 +- 请不要在您的sql语句中改变或翻译列名，请保持原有的列名不变，即使他们含有空格或-。 +- 在你的sql中不要遗漏GROUP BY。 +- 直接返回JSON对象，不要有任何其他内容。确保它能够被JavaScript中的JSON.parse()直接解析。 +`; diff --git a/packages/vmind/src/skylark/dataProcess/query/queryDataset.ts b/packages/vmind/src/skylark/dataProcess/query/queryDataset.ts new file mode 100644 index 00000000..d8642de5 --- /dev/null +++ b/packages/vmind/src/skylark/dataProcess/query/queryDataset.ts @@ -0,0 +1,60 @@ +import { DataItem, ILLMOptions, SimpleFieldInfo } from '../../../typings'; +import { DataQueryResponse } from './type'; +import { getQueryDatasetPrompt } from './prompts'; +import { requestSkyLark } from '../../chart-generation/utils'; +import { parseRespondField } from '../../../gpt/dataProcess/query/utils'; +import { parseSkylarkResponseAsJSON, patchDataQueryInput } from './utils'; +import { queryDataset } from '../../../common/dataProcess/dataQuery'; + +/** + * query the source dataset according to user's input and fieldInfo to get aggregated dataset + * + * @param userInput + * @param fieldInfo + * @param sourceDataset + */ +export const queryDatasetWithSkylark = async ( + userInput: string, + fieldInfo: SimpleFieldInfo[], + sourceDataset: DataItem[], + options: ILLMOptions +) => { + const patchedInput = patchDataQueryInput(userInput); + const { sql, fieldInfo: responseFieldInfo, usage } = await getQuerySQL(patchedInput, fieldInfo, options); + const datasetAfterQuery = queryDataset(sql, sourceDataset, fieldInfo); + + const fieldInfoNew = parseRespondField(responseFieldInfo, datasetAfterQuery); + if (datasetAfterQuery.length === 0) { + console.warn('empty dataset after query!'); + } + return { + dataset: datasetAfterQuery.length === 0 ? sourceDataset : datasetAfterQuery, + fieldInfo: datasetAfterQuery.length === 0 ? fieldInfo : fieldInfoNew, + usage + }; +}; + +/** + * call gpt to get the query sql according to user's input and data field. + * @param userInput + * @param fieldInfo + */ +const getQuerySQL = async (userInput: string, fieldInfo: SimpleFieldInfo[], options: ILLMOptions) => { + const queryDatasetMessage = `User's Command: ${userInput}\nColumn Information: ${JSON.stringify(fieldInfo)}`; + + const requestFunc = options.customRequestFunc?.dataQuery ?? requestSkyLark; + const QueryDatasetPrompt = getQueryDatasetPrompt(options.showThoughts ?? true); + const dataProcessRes = await requestFunc(QueryDatasetPrompt, queryDatasetMessage, options); + const dataQueryResponse: DataQueryResponse = parseSkylarkResponseAsJSON(dataProcessRes); + //const { sql, fieldInfo: responseFiledInfo } = dataQueryResponse; + //if (!sql || !responseFiledInfo) { + // //try to parse the response with another format + // const choices = dataProcessRes.choices; + // const content = choices[0].message.content; + // return { + // ...parseGPTQueryResponse(content), + // usage: dataProcessRes.usage + // }; + //} + return { ...dataQueryResponse, usage: dataProcessRes.usage }; +}; diff --git a/packages/vmind/src/skylark/dataProcess/query/type.ts b/packages/vmind/src/skylark/dataProcess/query/type.ts new file mode 100644 index 00000000..486b6e50 --- /dev/null +++ b/packages/vmind/src/skylark/dataProcess/query/type.ts @@ -0,0 +1,5 @@ +export type DataQueryResponse = { + THOUGHT?: string; + sql: string; + fieldInfo: { fieldName: string; description?: string }[]; +}; diff --git a/packages/vmind/src/skylark/dataProcess/query/utils.ts b/packages/vmind/src/skylark/dataProcess/query/utils.ts new file mode 100644 index 00000000..c81878d4 --- /dev/null +++ b/packages/vmind/src/skylark/dataProcess/query/utils.ts @@ -0,0 +1,54 @@ +import { LLMResponse } from 'src/typings'; +import JSON5 from 'json5'; +import { replaceAll } from '../../../common/dataProcess/utils'; +import { matchJSONStr } from '../../../common/utils'; + +export const parseJson = (JsonStr: string, prefix?: string) => { + const parseNoPrefixStr = (str: string) => { + //尝试不带前缀的解析 + try { + return JSON5.parse(str); + } catch (err) { + return { + error: true + }; + } + }; + //解析GPT返回的JSON格式 + if (prefix) { + //被某些字符包裹 + const splitArr = JsonStr.split(prefix); + const splittedStr = splitArr[splitArr.length - 2]; + const res = parseNoPrefixStr(splittedStr); + if (!res.error) { + return res; + } + } + //没有被前缀包裹，或者解析被前缀包裹的json失败，尝试直接解析返回结果 + const res2 = parseNoPrefixStr(JsonStr); + return res2; +}; + +export const parseSkylarkResponseAsJSON = (skylarkRes: LLMResponse) => { + try { + if (skylarkRes.error) { + return { + error: true, + ...skylarkRes.error + }; + } + const choices = skylarkRes.choices; + const content = replaceAll(choices[0].message.content, '\n', ' '); + const jsonStr = matchJSONStr(content); + const resJson = parseJson(jsonStr, '```'); + return resJson; + } catch (err: any) { + return { + error: true, + message: err.message + }; + } +}; + +export const patchDataQueryInput = (userInput: string) => + userInput + ' 使用` `包裹sql中的所有列名。使用支持的聚合函数将所有的度量列聚合。'; diff --git a/packages/vmind/src/skylark/utils.ts b/packages/vmind/src/skylark/utils.ts index f51395c6..22207e67 100644 --- a/packages/vmind/src/skylark/utils.ts +++ b/packages/vmind/src/skylark/utils.ts @@ -36,14 +36,23 @@ export const parseSkylarkResponse = (larkResponse: LLMResponse): Record { + const parts = str.split(':'); + return parts.length > 2 ? `${parts[0]}: "${parts.slice(1).join(':').trim()}"` : str; + }) + //replace ": -" with ": null" + .map((str: string) => { + return str.replace(/: -/g, ': null'); + }) .join('\n'); const resJson = yaml.load(patchedStr) as Record; resJson.usage = usage; //replace all the keys to lower case. return Object.keys(resJson).reduce((prev, cur) => ({ ...prev, [cur.toLocaleLowerCase()]: resJson[cur] }), {}); - } catch (err) { + } catch (err: any) { console.error(err); - return { error: true }; + return { error: true, message: err.message }; } }; diff --git a/packages/vmind/src/typings/index.ts b/packages/vmind/src/typings/index.ts index 6e94e2ac..78e74dd9 100644 --- a/packages/vmind/src/typings/index.ts +++ b/packages/vmind/src/typings/index.ts @@ -6,7 +6,7 @@ export interface ILLMOptions { /** llm request header, which has higher priority */ headers?: HeadersInit; // this will be used directly as the header of the LLM request. method?: 'POST' | 'GET'; //post or get - model?: Model; + model?: Model | string; max_tokens?: number; temperature?: number; showThoughts?: boolean; @@ -39,7 +39,7 @@ export type GPTDataProcessResult = { export type Cell = { //字段映射，可用的视觉通道：["x","y","color","size","angle","time"] x?: string; - y?: string; + y?: string | string[]; color?: string; size?: string; angle?: string; @@ -48,6 +48,7 @@ export type Cell = { source?: string; target?: string; value?: string; + category?: string; }; export type ChartType = string; export type GPTChartAdvisorResult = { @@ -77,7 +78,6 @@ export type Context = { colors?: string[]; totalTime?: number; }; -export type Pipe = (src: any, context: Context) => any; export type TimeType = { totalTime: number; @@ -131,11 +131,18 @@ export enum Model { GPT3_5 = 'gpt-3.5-turbo', GPT4 = 'gpt-4', SKYLARK = 'skylark-pro', - SKYLARK2 = 'skylark2-pro-4k' + SKYLARK2 = 'skylark2-pro-4k', + CHART_ADVISOR = 'chart-advisor' +} + +export enum ModelType { + GPT = 'gpt', + SKYLARK = 'skylark', + CHART_ADVISOR = 'chart-advisor' } export type ChartGenerationProps = { - model: Model; //models to finish data generation task + model: Model | string; //models to finish data generation task userPrompt: string; //user's intent of visualization, usually aspect in data that they want to visualize dataFields: FieldInfo[]; }; @@ -150,3 +157,15 @@ export type LLMResponse = { usage: any; [key: string]: any; }; + +export type PatchContext = { + chartType: string; + cell: Cell; + dataset: DataItem[]; + fieldInfo: SimpleFieldInfo[]; +}; + +export type PatchPipeline = ( + context: PatchContext, + _originalContext: PatchContext +) => { chartType: string; cell: Cell; dataset: DataItem[]; fieldInfo: SimpleFieldInfo[] }; diff --git a/packages/vmind/tsconfig.eslint.json b/packages/vmind/tsconfig.eslint.json index 7ceb1e5c..c6dbef60 100644 --- a/packages/vmind/tsconfig.eslint.json +++ b/packages/vmind/tsconfig.eslint.json @@ -1,7 +1,7 @@ { "extends": "@internal/ts-config/tsconfig.base.json", "compilerOptions": { - "types": ["jest", "offscreencanvas", "node"], + "types": ["jest", "node"], "lib": ["DOM", "ESNext"], "baseUrl": "./", "rootDir": "./" diff --git a/packages/vmind/tsconfig.json b/packages/vmind/tsconfig.json index ffe4a50e..55dcf6a4 100644 --- a/packages/vmind/tsconfig.json +++ b/packages/vmind/tsconfig.json @@ -19,7 +19,10 @@ "include": ["src"], "references": [ { - "path": "../calculator" + "path": "../calculator", + }, + { + "path": "../chart-advisor", } ] } diff --git a/packages/vmind/tsconfig.test.json b/packages/vmind/tsconfig.test.json index 91cbb773..9a567caf 100644 --- a/packages/vmind/tsconfig.test.json +++ b/packages/vmind/tsconfig.test.json @@ -2,8 +2,15 @@ "extends": "./tsconfig.json", "compilerOptions": { "paths": { - "@visactor/calculator": ["../calculator/src"], + "@visactor/chart-advisor": ["../chart-advisor/src"], } }, - "references": [] + "references": [ + { + "path": "../calculator", + }, + { + "path": "../chart-advisor", + } + ] } diff --git a/rush.json b/rush.json index c7cbe5fc..2a3bc7d0 100644 --- a/rush.json +++ b/rush.json @@ -81,6 +81,15 @@ "tags": [ "package" ] + }, + { + "packageName": "@visactor/chart-advisor", + "projectFolder": "packages/chart-advisor", + "shouldPublish": true, + "versionPolicyName": "vmindMin", + "tags": [ + "package" + ] } ] } \ No newline at end of file