feat: aggregate all measure columns in sql

Author: da730

packages/vmind/src/common/dataProcess/dataQuery.ts

@@ -9,7 +9,9 @@ import {
   replaceInvalidWords,
   swapMap,
   replaceBlankSpace,
-  replaceString
+  replaceString,
+  sumAllMeasureFields,
+  convertGroupByToString
 } from './utils';
 import alasql from 'alasql';
 
@@ -25,48 +27,35 @@ export const VMIND_DATA_SOURCE = 'VMind_data_source';
  */
 export const queryDataset = (sql: string, sourceDataset: DataItem[], fieldInfo: SimpleFieldInfo[]) => {
   const fieldNames = fieldInfo.map(field => field.fieldName);
-  console.log(sql);
   const { validStr, sqlReplaceMap, columnReplaceMap } = replaceInvalidWords(sql, fieldNames);
-  console.log(validStr);
-
-  //const { validStr: sqlWithNoASCII, replaceMap: ASCIIReplaceMap } = replaceNonASCIICharacters(sqlWithNoOperator);
-  //console.log(sqlWithNoASCII);
-
-  //const replaceMap = mergeMap(ASCIIReplaceMap, operatorReplaceMap);
-  console.log(columnReplaceMap);
-  console.log(sqlReplaceMap);
 
   //replace field names according to replaceMap
   const validColumnDataset = replaceDataset(sourceDataset, columnReplaceMap, true);
-  console.log(validColumnDataset);
 
   //replace field names and data values according to replaceMap
   const validDataset = replaceDataset(validColumnDataset, sqlReplaceMap, false);
-  console.log(validDataset);
 
   //replace blank spaces in column name
   const replacedFieldNames = fieldNames
     .map(field => replaceString(field, columnReplaceMap))
     .map(field => replaceString(field, sqlReplaceMap));
-  const finalSql = replaceBlankSpace(validStr, replacedFieldNames as string[]);
-  console.log(finalSql);
+  const validSql = replaceBlankSpace(validStr, replacedFieldNames as string[]);
+
+  const finalSql = sumAllMeasureFields(validSql, fieldInfo, columnReplaceMap, sqlReplaceMap);
+  //convertGroupByToString(finalSql, validDataset)
 
   //replace VMIND_DATA_SOURCE with placeholder "?"
   const sqlParts = (finalSql + ' ').split(VMIND_DATA_SOURCE);
   const sqlCount = sqlParts.length - 1;
   const alasqlQuery = sqlParts.join('?');
   //do the query
   const alasqlDataset = alasql(alasqlQuery, new Array(sqlCount).fill(validDataset));
-  console.log(alasqlDataset);
 
   //restore the dataset
   const columnReversedMap = swapMap(columnReplaceMap);
   const columnRestoredDataset = replaceDataset(alasqlDataset, columnReversedMap, true);
-  console.log(columnRestoredDataset);
   const sqlReversedMap = swapMap(sqlReplaceMap);
   const sqlRestoredDataset = replaceDataset(columnRestoredDataset, sqlReversedMap, false);
 
-  console.log(sqlRestoredDataset);
-
   return sqlRestoredDataset;
 };

packages/vmind/src/common/dataProcess/utils.ts

@@ -347,7 +347,6 @@ export const replaceBlankSpace = (sql: string, fieldNames: string[]) => {
     const matchedFieldName = fieldNames.find(field => matchColumnName(column, field));
     return matchedFieldName ?? column;
   });
-  console.log(columnsInSql, validColumnNames);
 
   const finalSql = columnsInSql.reduce((prev, _cur, index) => {
     const originColumnName = columnsInSql[index];
@@ -360,3 +359,62 @@ export const replaceBlankSpace = (sql: string, fieldNames: string[]) => {
   }, sql);
   return finalSql;
 };
+
+/**
+ * sometimes skylark2 pro will return a sql statement with some measure fields not being aggregated
+ * this will make an empty field in dataset
+ * so we need to aggregate these fields.
+ *
+ */
+export const sumAllMeasureFields = (
+  sql: string,
+  fieldInfo: SimpleFieldInfo[],
+  columnReplaceMap: Map<string, string>,
+  sqlReplaceMap: Map<string, string>
+) => {
+  const measureFieldsInSql = fieldInfo
+    .filter(field => field.role === ROLE.MEASURE)
+    .map(field => {
+      const { fieldName } = field;
+      const replacedName1 = replaceString(fieldName, columnReplaceMap);
+      const replacedName2 = replaceString(replacedName1, sqlReplaceMap);
+
+      return replacedName2;
+    });
+
+  const ast: any = alasql.parse(sql);
+  const nonAggregatedColumns: string[] = ast.statements[0].columns
+    .filter((column: any) => !column.aggregatorid)
+    .map((column: any) => column.columnid);
+  const groupByColumns: string[] = ast.statements[0].group.map((column: any) => column.columnid);
+
+  //aggregate columns that is not in group by statement
+  const needAggregateColumns = nonAggregatedColumns
+    //filter all the measure fields
+    .filter(column => measureFieldsInSql.includes(column))
+    //filter measure fields that is not in groupby
+    .filter(column => !groupByColumns.includes(column));
+
+  const patchedFields = needAggregateColumns.map(column => `SUM(\`${column}\`) as ${column}`);
+
+  const finalSql = needAggregateColumns.reduce((prev, cur, index) => {
+    const regexStr = `\`?${cur}\`?`;
+    const regex = new RegExp(regexStr, 'g');
+    return prev.replace(regex, patchedFields[index]);
+  }, sql);
+
+  return finalSql;
+};
+
+/**
+ * convert group by columns to string
+ */
+export const convertGroupByToString = (sql: string, dataset: DataItem[]) => {
+  const ast: any = alasql.parse(sql);
+  const groupByColumns: string[] = ast.statements[0].group.map((column: any) => column.columnid);
+  dataset.forEach(item => {
+    groupByColumns.forEach(column => {
+      item[column] = item[column].toString();
+    });
+  });
+};

packages/vmind/src/skylark/dataProcess/query/prompts.ts

@@ -18,7 +18,7 @@ export const getQueryDatasetPrompt = (
 1. 从用户的指令中提取与数据相关的部分。忽略其他与数据无关的部分。
 2. 根据列的名称和类型，推断${VMIND_DATA_SOURCE}中与用户指令有关的列，并将其添加到SELECT中。尽可能多地选择相关列，不要遗漏一些关键的列，比如日期类型的维度等。你只能使用Column Information中提到的列，不要假设不存在的列。如果现有的列不能满足用户的命令，选择Column Information中最相关的列。
 3. 不论用户指定了哪种图表类型，将所选择的度量列使用聚合函数聚合，即使你推断它们不适合被聚合，即使用户没有要求你这样做。如果你不确定使用哪个聚合函数，使用SUM()。不要使用不支持的聚合函数。
-4. 使用你所选择的维度列对数据进行分组，将其添加到GROUP BY中。没有被聚合的度量列也必须被放到GROUP BY中。
+4. 使用维度列对数据进行分组。
 5. 在您的sql中，如有必要，您也可以使用WHERE, HAVING, ORDER BY, LIMIT。使用支持的操作符完成WHERE和HAVING。只能使用如columnA = value1，sum_b > 0的二元表达式。在您的表达式中，只能使用在维度列的domain中出现的维度值。
 
 让我们一步一步思考。不要忘了将所有度量列聚合。