Merge pull request #46 from VisActor/feat-dataAggregation-skylark

Feat data aggregation skylark

Author: da730

common/config/rush/pnpm-lock.yaml

@@ -3663,3 +3663,14 @@ East Asia & Pacific,7.8,8.95,10.18,11.57,13.25
 Europe & Central Asia,9.52,10.39,10.93,11.69,12.63`,
   input: '看下各类别数值分布'
 };
+
+/*
+ * multi-measure
+ */
+export const mockUserInput17 = {
+  csv: `date,ctr,libra_ab_vid,gmv,detection_uv,User,user_count,gpm,ctcvr,version_id,co
+2024-03-05,0.024752458745543306,-1,27.05042746931187,683740,31135119,683740,0.0028542157856540976,0.00011098365108464276,-1,0.01765408108609234
+2024-03-06,0.025529744509421266,-1,26.0184806658061,666756,30057727,666756,0.0029282343207837846,0.00011602865071429876,-1,0.018457400571758008
+2024-03-07,0.024929771518461413,-1,26.099474885828787,653568,25685979,653568,0.00305250127580487,0.00012037169363684177,-1,0.019630556105233156`,
+  input: '使用折线图展示'
+};

packages/vmind/__tests__/browser/src/constants/mockData.ts

@@ -33,13 +33,13 @@ import {
   mockUserInput12,
   mockUserInput13,
   mockUserInput14,
-  mockUserInput16
+  mockUserInput16,
+  mockUserInput17
 } from '../constants/mockData';
 import VMind from '../../../../src/index';
 import { Model } from '../../../../src/index';
-import { queryDataset } from '../../../../src/gpt/dataProcess';
 import { isArray } from 'lodash';
-import { mockDataset, mockData2 } from './mockData';
+import { mockDataset, mockData2, mockData3, mockData4 } from './mockData';
 
 const TextArea = Input.TextArea;
 const Option = Select.Option;
@@ -76,11 +76,12 @@ const demoDataList: { [key: string]: any } = {
   'College entrance examination': acceptRatioData,
   'Shopping Mall Sales Performance': mallSalesData,
   'Global GDP': mockUserInput6Eng,
-  'Sales of different drinkings': mockUserInput3Eng
+  'Sales of different drinkings': mockUserInput3Eng,
+  'Multi measure': mockUserInput17
 };
 
 const globalVariables = (import.meta as any).env;
-const ModelConfigMap = {
+const ModelConfigMap: any = {
   [Model.SKYLARK2]: { url: globalVariables.VITE_SKYLARK_URL, key: globalVariables.VITE_SKYLARK_KEY },
   [Model.SKYLARK]: { url: globalVariables.VITE_SKYLARK_URL, key: globalVariables.VITE_SKYLARK_KEY },
   [Model.GPT3_5]: { url: globalVariables.VITE_GPT_URL, key: globalVariables.VITE_GPT_KEY },
@@ -95,7 +96,7 @@ export function DataInput(props: IPropsType) {
   const [spec, setSpec] = useState<string>('');
   const [time, setTime] = useState<number>(1000);
   const [model, setModel] = useState<Model>(Model.GPT3_5);
-  const [cache, setCache] = useState<boolean>(false);
+  const [cache, setCache] = useState<boolean>(true);
   const [showThoughts, setShowThoughts] = useState<boolean>(false);
   const [visible, setVisible] = React.useState(false);
   const [url, setUrl] = React.useState(ModelConfigMap[model]?.url ?? OPENAI_API_URL);
@@ -125,10 +126,10 @@ export function DataInput(props: IPropsType) {
     //const { fieldInfo: fieldInfoQuery, dataset: datasetQuery } = await vmind?.dataQuery(describe, fieldInfo, dataset);
     //const { fieldInfo, dataset, usage } = await vmind.parseCSVDataWithLLM(csv, describe);
 
-    //const dataset = mockData2
-    //const fieldInfo = vmind?.getFieldInfo(dataset)
+    //const dataset = mockData4;
+    //const fieldInfo = vmind?.getFieldInfo(dataset);
     const startTime = new Date().getTime();
-    const chartGenerationRes = await vmind.generateChart(describe, fieldInfo, dataset);
+    const chartGenerationRes = await vmind.generateChart(describe, fieldInfo, dataset, true);
     const endTime = new Date().getTime();
     if (isArray(chartGenerationRes)) {
       props.onSpecListGenerate(chartGenerationRes.map(res => res.spec));

packages/vmind/__tests__/browser/src/pages/DataInput.tsx

@@ -97,6 +97,7 @@
     "dayjs": "~1.11.10",
     "js-yaml": "~4.1.0",
     "node-sql-parser": "~4.17.0",
-    "@visactor/calculator": "workspace:*"
+    "@visactor/calculator": "workspace:*",
+    "alasql": "~4.3.2"
   }
-}
+}

packages/vmind/package.json

@@ -0,0 +1,61 @@
+import { DataItem, SimpleFieldInfo } from 'src/typings';
+import {
+  getValueByAttributeName,
+  mergeMap,
+  replaceAll,
+  replaceDataset,
+  replaceInvalidContent,
+  replaceNonASCIICharacters,
+  replaceInvalidWords,
+  swapMap,
+  replaceBlankSpace,
+  replaceString,
+  sumAllMeasureFields,
+  convertGroupByToString
+} from './utils';
+import alasql from 'alasql';
+
+export const VMIND_DATA_SOURCE = 'VMind_data_source';
+
+/**
+ * SQL query for SourceDatset
+ * It has nothing to do with the model type model
+ * @param sql
+ * @param sourceDataset
+ * @param fieldInfo
+ * @returns dataset after query
+ */
+export const queryDataset = (sql: string, sourceDataset: DataItem[], fieldInfo: SimpleFieldInfo[]) => {
+  const fieldNames = fieldInfo.map(field => field.fieldName);
+  const { validStr, sqlReplaceMap, columnReplaceMap } = replaceInvalidWords(sql, fieldNames);
+
+  //replace field names according to replaceMap
+  const validColumnDataset = replaceDataset(sourceDataset, columnReplaceMap, true);
+
+  //replace field names and data values according to replaceMap
+  const validDataset = replaceDataset(validColumnDataset, sqlReplaceMap, false);
+
+  //replace blank spaces in column name
+  const replacedFieldNames = fieldNames
+    .map(field => replaceString(field, columnReplaceMap))
+    .map(field => replaceString(field, sqlReplaceMap));
+  const validSql = replaceBlankSpace(validStr, replacedFieldNames as string[]);
+
+  const finalSql = sumAllMeasureFields(validSql, fieldInfo, columnReplaceMap, sqlReplaceMap);
+  //convertGroupByToString(finalSql, validDataset)
+
+  //replace VMIND_DATA_SOURCE with placeholder "?"
+  const sqlParts = (finalSql + ' ').split(VMIND_DATA_SOURCE);
+  const sqlCount = sqlParts.length - 1;
+  const alasqlQuery = sqlParts.join('?');
+  //do the query
+  const alasqlDataset = alasql(alasqlQuery, new Array(sqlCount).fill(validDataset));
+
+  //restore the dataset
+  const columnReversedMap = swapMap(columnReplaceMap);
+  const columnRestoredDataset = replaceDataset(alasqlDataset, columnReversedMap, true);
+  const sqlReversedMap = swapMap(sqlReplaceMap);
+  const sqlRestoredDataset = replaceDataset(columnRestoredDataset, sqlReversedMap, false);
+
+  return sqlRestoredDataset;
+};