feat: Adding support for having clause (#56)

* having support added

* reverted test case changes

* reusing traverse and filter

* removed unneeded function

* code refactor

* bumped up versions

* fixed test cases

* fixed test cases

* self review

* added test case for having/where filter

Author: zaidjan-devrev

meerkat-core/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@devrev/meerkat-core",
-  "version": "0.0.62",
+  "version": "0.0.63",
   "dependencies": {
     "@swc/helpers": "~0.5.0"
   },

meerkat-core/src/ast-builder/ast-builder.ts

@@ -2,15 +2,44 @@ import { cubeFilterToDuckdbAST } from '../cube-filter-transformer/factory';
 import { cubeDimensionToGroupByAST } from '../cube-group-by-transformer/cube-group-by-transformer';
 import { cubeLimitOffsetToAST } from '../cube-limit-offset-transformer/cube-limit-offset-transformer';
 import { cubeOrderByToAST } from '../cube-order-by-transformer/cube-order-by-transformer';
-import { QueryFiltersWithInfo } from '../cube-to-duckdb/cube-filter-to-duckdb';
-import { FilterType, Query } from '../types/cube-types/query';
+import { QueryFiltersWithInfo, QueryFiltersWithInfoSingular } from '../cube-to-duckdb/cube-filter-to-duckdb';
+import { traverseAndFilter } from '../filter-params/filter-params-ast';
+import { FilterType, MeerkatQueryFilter, Query } from '../types/cube-types/query';
 import { TableSchema } from '../types/cube-types/table';
+import { SelectStatement } from '../types/duckdb-serialization-types';
 import { SelectNode } from '../types/duckdb-serialization-types/serialization/QueryNode';
 import { getBaseAST } from '../utils/base-ast';
 import { cubeFiltersEnrichment } from '../utils/cube-filter-enrichment';
 import { modifyLeafMeerkatFilter } from '../utils/modify-meerkat-filter';
 
 
+const formatFilters = (queryFiltersWithInfo: QueryFiltersWithInfo, filterType?: FilterType) => {
+  /*
+  * If the type of filter is set to base filter where 
+  */
+  return filterType === 'BASE_FILTER' ? queryFiltersWithInfo : modifyLeafMeerkatFilter(queryFiltersWithInfo, (item) => {
+    return {
+      ...item,
+      member: item.member.split('.').join('__')
+    };
+  }) as QueryFiltersWithInfo;
+}
+
+
+const getFormattedFilters = ({ queryFiltersWithInfo, filterType, mapperFn, baseAST }: {
+  queryFiltersWithInfo: QueryFiltersWithInfo,
+  filterType?: FilterType,
+  baseAST: SelectStatement,
+  mapperFn: (val: QueryFiltersWithInfoSingular) => MeerkatQueryFilter | null
+}) => {
+  const filters = queryFiltersWithInfo.map(item => mapperFn(item)).filter(Boolean) as QueryFiltersWithInfoSingular[];
+  const formattedFilters = formatFilters(filters, filterType);
+  return cubeFilterToDuckdbAST(
+    formattedFilters,
+    baseAST
+  );
+}
+
 export const cubeToDuckdbAST = (query: Query, tableSchema: TableSchema, options?: { filterType: FilterType }
 ) => {
   /**
@@ -35,23 +64,24 @@ export const cubeToDuckdbAST = (query: Query, tableSchema: TableSchema, options?
       return null;
     }
 
-    /*
-    * If the type of filter is set to base filter where 
-    */
-    const finalFilters = options?.filterType === 'BASE_FILTER' ? queryFiltersWithInfo : modifyLeafMeerkatFilter(queryFiltersWithInfo, (item) => {
-      return {
-        ...item,
-        member: item.member.split('.').join('__')
-      };
-    }) as QueryFiltersWithInfo; 
+    const whereClause = getFormattedFilters({
+      baseAST,
+      mapperFn: (item) => traverseAndFilter(item, (value) =>  !query.measures.includes(value.member)),
+      queryFiltersWithInfo,
+      filterType: options?.filterType
+    })
+
+    const havingClause = getFormattedFilters({
+      baseAST,
+      mapperFn: (item) => traverseAndFilter(item, (value) =>  query.measures.includes(value.member)),
+      queryFiltersWithInfo,
+      filterType: options?.filterType
+    })
 
-    const duckdbWhereClause = cubeFilterToDuckdbAST(
-      finalFilters,
-      baseAST
-    );
     // eslint-disable-next-line @typescript-eslint/ban-ts-comment
     //@ts-ignore
-    node.where_clause = duckdbWhereClause;
+    node.where_clause = whereClause;
+    node.having = havingClause
   }
 
   if (query.dimensions && query.dimensions?.length > 0) {

meerkat-core/src/cube-to-duckdb/cube-filter-to-duckdb.ts

@@ -1,5 +1,5 @@
-import { Dimension, Measure } from '../types/cube-types/table';
 import { QueryFilter } from '../types/cube-types/query';
+import { Dimension, Measure } from '../types/cube-types/table';
 
 export type QueryOperatorsWithInfo = QueryFilter & {
   memberInfo: Measure | Dimension;
@@ -28,8 +28,8 @@ export type QueryFilterWithInfo =
       | LogicalOrFilterWithInfo
     )[];
 
-export type QueryFiltersWithInfo = (
-  | QueryOperatorsWithInfo
-  | LogicalAndFilterWithInfo
-  | LogicalOrFilterWithInfo
-)[];
+export type QueryFiltersWithInfoSingular = QueryOperatorsWithInfo
+| LogicalAndFilterWithInfo
+| LogicalOrFilterWithInfo;
+
+export type QueryFiltersWithInfo = QueryFiltersWithInfoSingular[];

meerkat-core/src/filter-params/filter-params-ast.ts

@@ -5,24 +5,26 @@ import {
   LogicalOrFilter,
   MeerkatQueryFilter,
   Query,
+  QueryFilter,
   TableSchema
 } from '../types/cube-types';
 import { SelectStatement } from '../types/duckdb-serialization-types/serialization/Statement';
 
 /**
  * Get the query filter with only where filterKey matches
  */
-const traverseAndFilter = (
+
+export const traverseAndFilter = (
   filter: MeerkatQueryFilter,
-  memberKey: string
+  callback: (value: QueryFilter) => boolean
 ): MeerkatQueryFilter | null => {
   if ('member' in filter) {
-    return filter.member === memberKey ? filter : null;
+    return callback(filter) ? filter : null;
   }
 
   if ('and' in filter) {
     const filteredAndFilters = filter.and
-      .map((subFilter) => traverseAndFilter(subFilter, memberKey))
+      .map((subFilter) => traverseAndFilter(subFilter, callback))
       .filter(Boolean) as MeerkatQueryFilter[];
     const obj =
       filteredAndFilters.length > 0 ? { and: filteredAndFilters } : null;
@@ -31,7 +33,7 @@ const traverseAndFilter = (
 
   if ('or' in filter) {
     const filteredOrFilters = filter.or
-      .map((subFilter) => traverseAndFilter(subFilter, memberKey))
+      .map((subFilter) => traverseAndFilter(subFilter, callback))
       .filter(Boolean);
     const obj = filteredOrFilters.length > 0 ? { or: filteredOrFilters } : null;
     return obj as LogicalOrFilter;
@@ -47,7 +49,7 @@ export const getFilterByMemberKey = (
 ): MeerkatQueryFilter[] => {
   if (!filters) return [];
   return filters
-    .map((filter) => traverseAndFilter(filter, memberKey))
+    .map((filter) => traverseAndFilter(filter, (value) =>  value.member === memberKey))
     .filter(Boolean) as MeerkatQueryFilter[];
 };
 

meerkat-core/src/get-projection-clause/get-projection-clause.spec.ts

@@ -1,22 +1,20 @@
-import { get } from "http";
 import { TableSchema } from "../types/cube-types";
-import { getMemberProjection, getProjectionClause } from './get-projection-clause';
+import { getDimensionProjection, getFilterMeasureProjection, getProjectionClause } from './get-projection-clause';
 
 
 const TABLE_SCHEMA: TableSchema = {
     dimensions: [{ name: 'a', sql: 'others', type: 'number' }, { name: 'c', sql: 'any', type: 'string' }],
-    measures: [],
+    measures: [{ name: 'x', sql: 'x', type: 'number' }, { name: 'y', sql: 'y', type: 'number' }, { name: 'z', sql: 'z', type: 'number' }],
     name: 'test',
     sql: 'SELECT * from test'
     // Define your table schema here
 };
 describe("get-projection-clause", () => {
-    describe("getMemberProjection", () => {
+    describe("getDimensionProjection", () => {
         it("should return the member projection when the key exists in the table schema", () => {
             const key = "test.a";
-            
 
-            const result = getMemberProjection({ key, tableSchema: TABLE_SCHEMA });
+            const result = getDimensionProjection({ key, tableSchema: TABLE_SCHEMA });
             expect(result).toEqual({ aliasKey: "test__a", foundMember: {"name": "a", "sql": "others", "type": "number"}, sql: "others AS test__a"});
         });
 
@@ -27,25 +25,48 @@ describe("get-projection-clause", () => {
                 dimensions: [{ name: 'b', sql: 'others', type: 'number' }],
             };
 
-            const result = getMemberProjection({ key, tableSchema });
+            const result = getDimensionProjection({ key, tableSchema });
+            expect(result).toEqual({ aliasKey: undefined, foundMember: undefined, sql: undefined });
+        });
+    })
+
+    describe("getFilterMeasureProjection", () => {
+        it("should return the member projection when the key exists in the table schema", () => {
+            const key = "test.x";
+            const result = getFilterMeasureProjection({ key, tableSchema: TABLE_SCHEMA, measures: ['test.a']});
+            expect(result).toEqual({ aliasKey: "test__x", foundMember: {"name": "x", "sql": "x", "type": "number"}, sql: "test.x AS test__x"});
+        });
+
+        it("should not create alias when item in measure list", () => {
+            const key = "test.x";
+            const result = getFilterMeasureProjection({ key, tableSchema: TABLE_SCHEMA, measures: ['test.x']});
+            expect(result).toEqual({ aliasKey: undefined, foundMember: undefined, sql: undefined});
+        });
+    
+        it("should return the object with undefined values when the key doesn't exist in the table schema", () => {
+            const key = "test.a";
+            const tableSchema: TableSchema = {
+                ...TABLE_SCHEMA,
+                measures: [{ name: 'b', sql: 'others', type: 'number' }],
+            };
+    
+            const result = getFilterMeasureProjection({ key, tableSchema, measures: ['test.b'] });
             expect(result).toEqual({ aliasKey: undefined, foundMember: undefined, sql: undefined });
         });
     })
 
     describe("getProjectionClause", () => {
         it('should return the projection clause when the members are present in the table schema', () => {
             const members = ['test.a', 'test.c'];
-            const tableSchema = TABLE_SCHEMA;
             const aliasedColumnSet = new Set<string>();
-            const result = getProjectionClause(members, tableSchema, aliasedColumnSet);
-            expect(result).toEqual(', others AS test__a, any AS test__c');
+            const result = getProjectionClause([], members, TABLE_SCHEMA, aliasedColumnSet);
+            expect(result).toEqual('others AS test__a, any AS test__c');
         })
         it('should skip aliased items present in already seen', () => {
             const members = ['test.a', 'test.c'];
-            const tableSchema = TABLE_SCHEMA;
             const aliasedColumnSet = new Set<string>(['test.c']);
-            const result = getProjectionClause(members, tableSchema, aliasedColumnSet);
-            expect(result).toEqual(', others AS test__a');
+            const result = getProjectionClause([], members, TABLE_SCHEMA, aliasedColumnSet);
+            expect(result).toEqual('others AS test__a, ');
         })
     })
 

meerkat-core/src/get-projection-clause/get-projection-clause.ts

@@ -1,36 +1,85 @@
 import { TableSchema } from "../types/cube-types";
-import { findInSchema } from "../utils/find-in-table-schema";
+import { findInDimensionSchema, findInMeasureSchema } from "../utils/find-in-table-schema";
 import { memberKeyToSafeKey } from "../utils/member-key-to-safe-key";
 
-export const getMemberProjection = ({ key, tableSchema }: {
-    key: string;
-    tableSchema: TableSchema;
-  }) => {
-    // Find the table access key
-    const measureWithoutTable = key.split('.')[1];
-  
-    const foundMember = findInSchema(measureWithoutTable, tableSchema)
-    if (!foundMember) {
-      // If the selected member is not found in the table schema or if it is already selected, continue.
-      return {
-        sql: undefined,
-        foundMember: undefined,
-        aliasKey: undefined
-      }
+
+export const getFilterMeasureProjection = ({ key, tableSchema, measures }: {
+  key: string;
+  tableSchema: TableSchema;
+  measures: string[]
+}) => {
+  const measureWithoutTable = key.split('.')[1];
+  const foundMember = findInMeasureSchema(measureWithoutTable, tableSchema);
+  const isMeasure = measures.includes(key);
+  if (!foundMember || isMeasure) {
+    // If the selected member is not found in the table schema or if it is already selected, continue.
+    // If the selected member is a measure, don't create an alias. Since measure computation is done in the outermost level of the query
+    return {
+      sql: undefined,
+      foundMember: undefined,
+      aliasKey: undefined
     }
-    const aliasKey = memberKeyToSafeKey(key);
-    // Add the alias key to the set. So we have a reference to all the previously selected members.
-    return { sql: `${foundMember.sql} AS ${aliasKey}` , foundMember, aliasKey }
+  }
+  const aliasKey = memberKeyToSafeKey(key);
+  return { sql: `${key} AS ${aliasKey}` , foundMember, aliasKey }
 }
 
+export const getDimensionProjection = ({ key, tableSchema }: {
+  key: string;
+  tableSchema: TableSchema;
+}) => {
+  // Find the table access key
+  const measureWithoutTable = key.split('.')[1];
 
-export const getProjectionClause = (members: string[], tableSchema: TableSchema, aliasedColumnSet: Set<string>) => {
-  return members.reduce((acc, member) => {
-    const { sql: memberSql }  = getMemberProjection({ key: member, tableSchema })
-    if (aliasedColumnSet.has(member)) {
-      return acc
+  const foundMember = findInDimensionSchema(measureWithoutTable, tableSchema)
+  if (!foundMember) {
+    // If the selected member is not found in the table schema or if it is already selected, continue.
+    return {
+      sql: undefined,
+      foundMember: undefined,
+      aliasKey: undefined
     }
-    acc += `, ${memberSql}`
+  }
+  const aliasKey = memberKeyToSafeKey(key);
+  // Add the alias key to the set. So we have a reference to all the previously selected members.
+  return { sql: `${foundMember.sql} AS ${aliasKey}` , foundMember, aliasKey }
+}
+
+const aggregator = ({
+  member,
+  aliasedColumnSet,
+  acc,
+  currentIndex,
+  members,
+  sql
+}: {
+  member: string;
+  aliasedColumnSet: Set<string>;
+  acc: string;
+  sql?: string;
+  currentIndex: number,
+  members: string[]
+}) => {
+  if (aliasedColumnSet.has(member) || !sql) {
     return acc
+  } 
+  aliasedColumnSet.add(member)
+  acc += sql
+  if (currentIndex !== members.length - 1) {
+    acc += `, `
+  }
+  return acc
+}
+
+
+export const getProjectionClause = (measures: string[], dimensions: string[], tableSchema: TableSchema, aliasedColumnSet: Set<string>) => {
+  const dimensionsProjections =  dimensions.reduce((acc, member, currentIndex, members) => {
+    const { sql: memberSql }  = getDimensionProjection({ key: member, tableSchema })
+    return aggregator({ member, aliasedColumnSet, acc, currentIndex, members, sql: memberSql })
   }, '')
-}
+  const measureProjections = measures.reduce((acc, member, currentIndex, members) => {
+    const { sql: memberSql } = getFilterMeasureProjection({ key: member, tableSchema, measures })
+    return aggregator({ member, aliasedColumnSet, acc, currentIndex, members, sql: memberSql })
+  }, '')
+  return dimensionsProjections + (dimensionsProjections.length && measureProjections.length ? ', ' : '') + measureProjections
+}