Merge pull request #48 from VisActor/feat-dataAggregation-skylark

Feat data aggregation skylark

Author: da730

packages/chart-advisor/src/index.ts

@@ -12,6 +12,9 @@ import { scorer as defaultScorer } from './score';
 import * as dataUtils from './dataUtil';
 import { isNil, isNaN } from 'lodash';
 
+export { fold } from './fieldUtils';
+export { FOLD_NAME, FOLD_VALUE } from './constant';
+
 export function chartAdvisor(params: AdviserParams): AdviseResult {
   const {
     originDataset,
@@ -66,7 +69,6 @@ export function chartAdvisor(params: AdviserParams): AdviseResult {
     };
     dimensionSet.uniqueID = uniqueID;
     originDataset.forEach(row => {
-      //后端版本可直接获取到未平坦化的dataSource，不用做此判断
       dimensionSet.data.push(row[uniqueID]);
     });
     dimensionSet.dataType = dimension.type;

packages/vmind/src/common/dataProcess/utils.ts

@@ -383,17 +383,23 @@ export const sumAllMeasureFields = (
     });
 
   const ast: any = alasql.parse(sql);
-  const nonAggregatedColumns: string[] = ast.statements[0].columns
+  const selectedColumns = ast.statements[0].columns;
+  const nonAggregatedColumns: string[] = selectedColumns
     .filter((column: any) => !column.aggregatorid)
     .map((column: any) => column.columnid);
-  const groupByColumns: string[] = ast.statements[0].group.map((column: any) => column.columnid);
 
-  //aggregate columns that is not in group by statement
-  const needAggregateColumns = nonAggregatedColumns
-    //filter all the measure fields
-    .filter(column => measureFieldsInSql.includes(column))
-    //filter measure fields that is not in groupby
-    .filter(column => !groupByColumns.includes(column));
+  const groupByColumns: string[] = (ast.statements[0].group ?? []).map((column: any) => column.columnid);
+
+  //if there exist some aggregated columns in sql and there exist GROUP BY statement in sql, then aggregate all the measure columns
+  let needAggregateColumns: string[] = [];
+  if (groupByColumns.length > 0 && nonAggregatedColumns.length !== selectedColumns.length) {
+    //aggregate columns that is not in group by statement
+    needAggregateColumns = nonAggregatedColumns
+      //filter all the measure fields
+      .filter(column => measureFieldsInSql.includes(column))
+      //filter measure fields that is not in groupby
+      .filter(column => !groupByColumns.includes(column));
+  }
 
   const patchedFields = needAggregateColumns.map(column => `SUM(\`${column}\`) as ${column}`);
 

packages/vmind/src/common/utils.ts

@@ -11,3 +11,13 @@ export const calculateTokenUsage = (usageList: any[]) => {
   });
   return totalUsage;
 };
+
+export const execPipeline = <PipelineContext>(
+  src: any,
+  pipes: ((src: any, context: PipelineContext) => any)[],
+  context: PipelineContext
+) =>
+  pipes.reduce((pre: any, pipe: (src: any, context: PipelineContext) => any) => {
+    const result = pipe(pre, context);
+    return result;
+  }, src);

packages/vmind/src/common/vizDataToSpec/utils.ts

@@ -1,4 +1,6 @@
+import { Cell, DataItem, DataType, ROLE, SimpleFieldInfo } from 'src/typings';
 import { VIDEO_LENGTH_BY_CHART_TYPE, DEFAULT_VIDEO_LENGTH } from './constants';
+import { FOLD_NAME, FOLD_VALUE, fold } from '@visactor/chart-advisor';
 
 export const detectAxesType = (values: any[], field: string) => {
   const isNumber = values.every(d => !d[field] || !isNaN(Number(d[field])));
@@ -43,3 +45,23 @@ export const estimateVideoTime = (chartType: string, spec: any, parsedTime?: num
     frameArr: []
   };
 };
+
+export const getRemainedFields = (cell: Cell, fieldInfo: SimpleFieldInfo[]) => {
+  const usedFields = Object.values(cell).flat();
+  const remainedFields = fieldInfo.filter(f => !usedFields.includes(f.fieldName));
+  return remainedFields;
+};
+
+export const getFieldByRole = (fields: SimpleFieldInfo[], role: ROLE) => {
+  return fields.find(f => f.role === role);
+};
+
+export const getFieldByDataType = (fields: SimpleFieldInfo[], dataTypeList: DataType[]) => {
+  return fields.find(f => dataTypeList.includes(f.type));
+};
+
+export const foldDatasetByYField = (dataset: DataItem[], yFieldList: string[], fieldInfo: SimpleFieldInfo[]) => {
+  const aliasMap = Object.fromEntries(fieldInfo.map(d => [d.fieldName, d.fieldName]));
+
+  return fold(dataset as any, yFieldList, FOLD_NAME, FOLD_VALUE, aliasMap, false);
+};

packages/vmind/src/common/vizDataToSpec/vizDataToSpec.ts

@@ -45,9 +45,10 @@ import {
   boxPlotField,
   boxPlotStyle
 } from './pipes';
-import { Cell, ChartType, Context, FieldInfo, Pipe, SimpleFieldInfo } from '../../typings';
-import { CARTESIAN_CHART_LIST, detectAxesType } from './utils';
+import { Cell, ChartType, Context, SimpleFieldInfo } from '../../typings';
 import { isArray } from 'lodash';
+import { execPipeline } from '../utils';
+
 export const vizDataToSpec = (
   dataset: any[],
   chartType: ChartType,
@@ -56,7 +57,7 @@ export const vizDataToSpec = (
   totalTime?: number
 ) => {
   const pipelines = pipelineMap[chartType];
-  const spec = execPipeline({}, pipelines, {
+  const spec = execPipeline<Context>({}, pipelines, {
     chartType,
     dataset,
     cell,
@@ -160,9 +161,3 @@ export const pipelineMap: { [chartType: string]: any } = {
   'WATERFALL CHART': pipelineWaterfall,
   'BOX PLOT': pipelineBoxPlot
 };
-
-export const execPipeline = (src: any, pipes: Pipe[], context: Context) =>
-  pipes.reduce((pre: any, pipe: Pipe) => {
-    const result = pipe(pre, context);
-    return result;
-  }, src);

packages/vmind/src/gpt/chart-generation/NLToChart.ts

@@ -2,14 +2,15 @@ import { SUPPORTED_CHART_LIST } from '../../common/vizDataToSpec/constants';
 import { DataItem, GPTChartAdvisorResult, ILLMOptions, LOCATION, SimpleFieldInfo, VizSchema } from '../../typings';
 import { checkChartTypeAndCell, vizDataToSpec } from '../../common/vizDataToSpec';
 import { parseGPTResponse, requestGPT } from '../utils';
-import { patchChartTypeAndCell, patchUserInput } from './utils';
+import { patchUserInput } from './utils';
 import { ChartAdvisorPromptEnglish } from './prompts';
 import { chartAdvisorHandler } from '../../common/chartAdvisor';
 import { estimateVideoTime } from '../../common/vizDataToSpec/utils';
 import { getSchemaFromFieldInfo } from '../../common/schema';
 import { queryDatasetWithGPT } from '../dataProcess/query/queryDataset';
 import { calculateTokenUsage } from '../..//common/utils';
 import { pick } from 'lodash';
+import { patchChartTypeAndCell } from './patch';
 
 export const generateChartWithGPT = async (
   userPrompt: string, //user's intent of visualization, usually aspect in data that they want to visualize
@@ -53,10 +54,11 @@ export const generateChartWithGPT = async (
     const chartTypeRes = resJson['CHART_TYPE'].toUpperCase();
     const cellRes = resJson['FIELD_MAP'];
     advisorUsage = resJson['usage'];
-    const patchResult = patchChartTypeAndCell(chartTypeRes, cellRes, dataset);
+    const patchResult = patchChartTypeAndCell(chartTypeRes, cellRes, dataset, fieldInfo);
     if (checkChartTypeAndCell(patchResult.chartTypeNew, patchResult.cellNew, fieldInfo)) {
       chartType = patchResult.chartTypeNew;
       cell = patchResult.cellNew;
+      dataset = patchResult.datasetNew;
     }
   } catch (err) {
     console.warn(err);