Skip to content

Commit 2ad2540

Browse files
zaidjan-devrevzaidjan-devrev
authored
refactor: unifying node and browser code (#115)
* unifiying node and browser code * bumped package versions * formatting fix * added test case for get-filter-params-sql * added test cases * reverted uneeded changes
1 parent 61708d0 commit 2ad2540

File tree

10 files changed

+307
-145
lines changed

10 files changed

+307
-145
lines changed

‎meerkat-browser/package.json

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
{
22
"name": "@devrev/meerkat-browser",
3-
"version": "0.0.83",
3+
"version": "0.0.84",
44
"dependencies": {
55
"@swc/helpers": "~0.5.0",
66
"@devrev/meerkat-core": "*",

‎meerkat-browser/src/browser-cube-to-sql/browser-cube-to-sql.ts

+18-75
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,6 @@
11
import {
22
BASE_TABLE_NAME,
33
ContextParams,
4-
FilterType,
54
Query,
65
TableSchema,
76
applyFilterParamsToBaseSQL,
@@ -11,95 +10,39 @@ import {
1110
deserializeQuery,
1211
detectApplyContextParamsToBaseSQL,
1312
getCombinedTableSchema,
14-
getFilterParamsAST,
15-
getWrappedBaseQueryWithProjections,
13+
getFilterParamsSQL,
14+
getFinalBaseSQL
1615
} from '@devrev/meerkat-core';
1716
import { AsyncDuckDBConnection } from '@duckdb/duckdb-wasm';
1817

19-
const getFilterParamsSQL = async ({
20-
query,
21-
tableSchema,
22-
filterType,
23-
connection,
24-
}: {
25-
query: Query;
26-
tableSchema: TableSchema;
27-
filterType?: FilterType;
28-
connection: AsyncDuckDBConnection;
29-
}) => {
30-
const filterParamsAST = getFilterParamsAST(
31-
query,
32-
tableSchema,
33-
filterType
34-
);
35-
const filterParamsSQL = [];
36-
37-
for (const filterParamAST of filterParamsAST) {
38-
if (!filterParamAST.ast) {
39-
continue;
40-
}
4118

42-
const queryOutput = await connection.query(
43-
astDeserializerQuery(filterParamAST.ast)
44-
);
45-
const parsedOutputQuery = queryOutput.toArray().map((row) => row.toJSON());
19+
const getQueryOutput = async (query: string, connection: AsyncDuckDBConnection) => {
20+
const queryOutput = await connection.query(query);
21+
const parsedOutputQuery = queryOutput.toArray().map((row) => row.toJSON());
22+
return parsedOutputQuery;
23+
}
4624

47-
const sql = deserializeQuery(parsedOutputQuery);
4825

49-
filterParamsSQL.push({
50-
memberKey: filterParamAST.memberKey,
51-
sql: sql,
52-
matchKey: filterParamAST.matchKey,
53-
});
54-
}
55-
return filterParamsSQL;
56-
};
57-
58-
const getFinalBaseSQL = async (
26+
interface CubeQueryToSQLParams {
27+
connection: AsyncDuckDBConnection,
5928
query: Query,
60-
tableSchema: TableSchema,
61-
connection: AsyncDuckDBConnection
62-
) => {
63-
/**
64-
* Apply transformation to the supplied base query.
65-
* This involves updating the filter placeholder with the actual filter values.
66-
*/
67-
const baseFilterParamsSQL = await getFilterParamsSQL({
68-
query: query,
69-
tableSchema,
70-
filterType: 'BASE_FILTER',
71-
connection,
72-
});
73-
const baseSQL = applyFilterParamsToBaseSQL(
74-
tableSchema.sql,
75-
baseFilterParamsSQL
76-
);
77-
const baseSQLWithFilterProjection = getWrappedBaseQueryWithProjections({
78-
baseQuery: baseSQL,
79-
tableSchema,
80-
query: query,
81-
});
82-
return baseSQLWithFilterProjection;
83-
};
29+
tableSchemas: TableSchema[],
30+
contextParams?: ContextParams,
31+
}
8432

8533
export const cubeQueryToSQL = async ({
8634
connection,
8735
query,
8836
tableSchemas,
8937
contextParams
90-
}: {
91-
connection: AsyncDuckDBConnection,
92-
query: Query,
93-
tableSchemas: TableSchema[],
94-
contextParams?: ContextParams
95-
}) => {
38+
}: CubeQueryToSQLParams) => {
9639
const updatedTableSchemas: TableSchema[] = await Promise.all(
9740
tableSchemas.map(async (schema: TableSchema) => {
98-
const baseFilterParamsSQL = await getFinalBaseSQL(
41+
const baseFilterParamsSQL = await getFinalBaseSQL({
9942
query,
100-
schema,
101-
connection
102-
);
43+
tableSchema: schema,
44+
getQueryOutput: (query) => getQueryOutput(query, connection)
45+
});
10346
return {
10447
...schema,
10548
sql: baseFilterParamsSQL,
@@ -124,7 +67,7 @@ export const cubeQueryToSQL = async ({
12467

12568
const preBaseQuery = deserializeQuery(parsedOutputQuery);
12669
const filterParamsSQL = await getFilterParamsSQL({
127-
connection,
70+
getQueryOutput: (query) => getQueryOutput(query, connection),
12871
query,
12972
tableSchema: updatedTableSchema,
13073
filterType: 'BASE_FILTER',

‎meerkat-core/package.json

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
{
22
"name": "@devrev/meerkat-core",
3-
"version": "0.0.83",
3+
"version": "0.0.84",
44
"dependencies": {
55
"@swc/helpers": "~0.5.0"
66
},

‎meerkat-core/src/get-filter-params-sql/get-filter-params-sql.spec.ts

+132
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,132 @@
1+
import { Database } from 'duckdb';
2+
import { TableSchema } from '../types/cube-types';
3+
import { getFilterParamsSQL } from './get-filter-params-sql';
4+
5+
const getQueryOutput = async (sql: string) => {
6+
const db = new Database(':memory:')
7+
return new Promise((resolve, reject) => {
8+
db.all(sql, (err, res) => {
9+
if (err) {
10+
reject(err);
11+
}
12+
resolve(res);
13+
});
14+
});
15+
}
16+
17+
const TABLE_SCHEMA: TableSchema = {
18+
name: 'orders',
19+
sql: "select * from orders WHERE ${FILTER_PARAMS.orders.status.filter('status')}",
20+
measures: [],
21+
dimensions: [
22+
{ name: 'id', sql: 'id', type: 'number' },
23+
{ name: 'date', sql: 'date', type: 'string' },
24+
{ name: 'status', sql: 'status', type: 'string' },
25+
{ name: 'amount', sql: 'amount', type: 'number' }
26+
]
27+
}
28+
describe('getFilterParamsSQL', () => {
29+
it('should find filter params when there are filters of base filter type', async () => {
30+
const result = await getFilterParamsSQL({
31+
filterType: 'BASE_FILTER',
32+
query: {
33+
measures: [ '*' ],
34+
filters: [
35+
{
36+
and: [
37+
{ member: 'orders.amount', operator: 'notSet' },
38+
{ member: 'orders.status', operator: 'set' }
39+
]
40+
}
41+
],
42+
dimensions: []
43+
},
44+
tableSchema: TABLE_SCHEMA,
45+
getQueryOutput,
46+
});
47+
expect(result).toEqual([
48+
{
49+
"matchKey": "${FILTER_PARAMS.orders.status.filter('status')}",
50+
"memberKey": "orders.status",
51+
"sql": "SELECT * FROM REPLACE_BASE_TABLE WHERE ((orders.status IS NOT NULL))",
52+
},
53+
]);
54+
});
55+
it('should not find filter params when there are no filters of base filter type', async () => {
56+
const result = await getFilterParamsSQL({
57+
filterType: 'BASE_FILTER',
58+
query: {
59+
measures: [ '*' ],
60+
filters: [],
61+
dimensions: []
62+
},
63+
tableSchema: TABLE_SCHEMA,
64+
getQueryOutput,
65+
});
66+
expect(result).toEqual([]);
67+
});
68+
it('should find filter params when there are filters of projection filter type', async () => {
69+
const result = await getFilterParamsSQL({
70+
filterType: 'PROJECTION_FILTER',
71+
query: {
72+
measures: [ '*' ],
73+
filters: [
74+
{
75+
and: [
76+
{ member: 'orders.amount', operator: 'notSet' },
77+
{ member: 'orders.status', operator: 'set' }
78+
]
79+
}
80+
],
81+
dimensions: []
82+
},
83+
tableSchema: TABLE_SCHEMA,
84+
getQueryOutput,
85+
});
86+
expect(result).toEqual([
87+
{
88+
"matchKey": "${FILTER_PARAMS.orders.status.filter('status')}",
89+
"memberKey": "orders.status",
90+
"sql": "SELECT * FROM REPLACE_BASE_TABLE WHERE ((orders__status IS NOT NULL))",
91+
},
92+
]);
93+
});
94+
it('should not find filter params when there are no filters', async () => {
95+
const result = await getFilterParamsSQL({
96+
filterType: 'PROJECTION_FILTER',
97+
query: {
98+
measures: [ '*' ],
99+
filters: [],
100+
dimensions: []
101+
},
102+
tableSchema: TABLE_SCHEMA,
103+
getQueryOutput,
104+
});
105+
expect(result).toEqual([]);
106+
});
107+
it('should find filter params when there are filters of no defined type', async () => {
108+
const result = await getFilterParamsSQL({
109+
query: {
110+
measures: [ '*' ],
111+
filters: [
112+
{
113+
and: [
114+
{ member: 'orders.amount', operator: 'notSet' },
115+
{ member: 'orders.status', operator: 'set' }
116+
]
117+
}
118+
],
119+
dimensions: []
120+
},
121+
tableSchema: TABLE_SCHEMA,
122+
getQueryOutput,
123+
});
124+
expect(result).toEqual([
125+
{
126+
"matchKey": "${FILTER_PARAMS.orders.status.filter('status')}",
127+
"memberKey": "orders.status",
128+
"sql": "SELECT * FROM REPLACE_BASE_TABLE WHERE ((orders__status IS NOT NULL))",
129+
},
130+
]);
131+
});
132+
});

‎meerkat-core/src/get-filter-params-sql/get-filter-params-sql.ts

+37
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
import { astDeserializerQuery, deserializeQuery } from "../ast-deserializer/ast-deserializer";
2+
import { getFilterParamsAST } from "../filter-params/filter-params-ast";
3+
import { FilterType, Query, TableSchema } from "../types/cube-types";
4+
5+
export const getFilterParamsSQL = async ({
6+
query,
7+
tableSchema,
8+
filterType,
9+
getQueryOutput
10+
}: {
11+
query: Query;
12+
tableSchema: TableSchema;
13+
filterType?: FilterType;
14+
getQueryOutput: (query: string) => Promise<any>;
15+
}) => {
16+
const filterParamsAST = getFilterParamsAST(
17+
query,
18+
tableSchema,
19+
filterType
20+
);
21+
const filterParamsSQL = [];
22+
for (const filterParamAST of filterParamsAST) {
23+
if (!filterParamAST.ast) {
24+
continue;
25+
}
26+
27+
const queryOutput = await getQueryOutput(astDeserializerQuery(filterParamAST.ast))
28+
const sql = deserializeQuery(queryOutput);
29+
30+
filterParamsSQL.push({
31+
memberKey: filterParamAST.memberKey,
32+
sql: sql,
33+
matchKey: filterParamAST.matchKey,
34+
});
35+
}
36+
return filterParamsSQL;
37+
};

‎meerkat-core/src/get-final-base-sql/get-final-base-sql.spec.ts

+69
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,69 @@
1+
import { Database } from 'duckdb';
2+
import { TableSchema } from '../types/cube-types';
3+
import { getFinalBaseSQL } from './get-final-base-sql';
4+
5+
const getQueryOutput = async (sql: string) => {
6+
const db = new Database(':memory:')
7+
return new Promise((resolve, reject) => {
8+
db.all(sql, (err, res) => {
9+
if (err) {
10+
reject(err);
11+
}
12+
resolve(res);
13+
});
14+
});
15+
}
16+
17+
const TABLE_SCHEMA: TableSchema = {
18+
name: 'orders',
19+
sql: "select * from orders WHERE ${FILTER_PARAMS.orders.status.filter('status')}",
20+
measures: [
21+
{ name: 'count', sql: 'COUNT(*)', type: 'number' }
22+
],
23+
dimensions: [
24+
{ name: 'id', sql: 'id', type: 'number' },
25+
{ name: 'date', sql: 'date', type: 'string' },
26+
{ name: 'status', sql: 'status', type: 'string' },
27+
{ name: 'amount', sql: 'amount', type: 'number' }
28+
]
29+
}
30+
describe('get final base sql', () => {
31+
it('should not return measures in the projected base sql when filter param passed', async () => {
32+
const result = await getFinalBaseSQL({
33+
query: {
34+
measures: ['orders.count'],
35+
filters: [
36+
{
37+
and: [
38+
{ member: 'orders.amount', operator: 'notSet' },
39+
{ member: 'orders.status', operator: 'set' }
40+
]
41+
}
42+
],
43+
dimensions: ['orders.status']
44+
},
45+
tableSchema: TABLE_SCHEMA,
46+
getQueryOutput,
47+
});
48+
expect(result).toEqual('SELECT *, amount AS orders__amount, status AS orders__status FROM (select * from orders WHERE ((orders.status IS NOT NULL))) AS orders');
49+
});
50+
it('should not return measures in the projected base sql when filter param not passed', async () => {
51+
const result = await getFinalBaseSQL({
52+
query: {
53+
measures: ['orders.count'],
54+
filters: [
55+
{
56+
and: [
57+
{ member: 'orders.amount', operator: 'notSet' },
58+
]
59+
}
60+
],
61+
dimensions: ['orders.status']
62+
},
63+
tableSchema: TABLE_SCHEMA,
64+
getQueryOutput,
65+
});
66+
expect(result).toEqual('SELECT *, amount AS orders__amount, status AS orders__status FROM (select * from orders WHERE TRUE) AS orders');
67+
});
68+
});
69+

0 commit comments

Comments
 (0)