Skip to content
This repository was archived by the owner on Dec 11, 2022. It is now read-only.

Commit b5f2daf

Browse files
ofir5300ofir5300
authored
Handle time range filtering issue for both partitioned and non-partitioned (#385)
* Handle time range filtering issue for both partitioned and non-partitioned (fixes #321, #325, #334, #362) * Fix UTC conversion & usage
1 parent 370b61a commit b5f2daf

File tree

2 files changed

+62
-49
lines changed

2 files changed

+62
-49
lines changed

‎src/bigquery_query.ts‎

Lines changed: 43 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -22,8 +22,12 @@ export default class BigQueryQuery {
2222
return res;
2323
}
2424

25-
public static formatDateToString(inputDate, separator = '', addtime = false) {
26-
const date = new Date(inputDate);
25+
public static formatDateToString(inputDate, convertToUTC = false, separator = '', addtime = false) {
26+
let date = new Date(inputDate);
27+
if (convertToUTC) {
28+
// check if should convert string to UTC time (relevant whenever addTime = true)
29+
date = BigQueryQuery.convertToUtc(date);
30+
}
2731
// 01, 02, 03, ... 29, 30, 31
2832
const DD = (date.getDate() < 10 ? '0' : '') + date.getDate();
2933
// 01, 02, 03, ... 10, 11, 12
@@ -89,6 +93,8 @@ export default class BigQueryQuery {
8993
public static replaceTimeShift(q) {
9094
return q.replace(/(\$__timeShifting\().*?(?=\))./g, '');
9195
}
96+
97+
// Note: converts to UTC time BUT - Date object always represented in local time (so probably relevant when handling date/time strings)
9298
static convertToUtc(d) {
9399
return new Date(d.getTime() + d.getTimezoneOffset() * 60000);
94100
}
@@ -378,22 +384,32 @@ export default class BigQueryQuery {
378384
return query;
379385
}
380386

387+
// Detects either date or timestamp, only if not comment out
388+
static hasDateFilter(whereClause) {
389+
return whereClause.match(/((?<!--.*)([1-2]\d{3}-[0-1]\d-[0-3]\d)|([\D](\d{13})[\D]).*\n)/gi);
390+
}
391+
381392
public buildWhereClause() {
382-
let query = '';
393+
let query = '', hasMacro = false, hasDateFilter = false;
383394
const conditions = _.map(this.target.where, (tag, index) => {
384395
switch (tag.type) {
385396
case 'macro':
397+
hasMacro = true;
386398
return tag.name + '(' + this.target.timeColumn + ')';
387399
case 'expression':
388-
return tag.params.join(' ');
400+
const expression = tag.params.join(' ');
401+
hasDateFilter = BigQueryQuery.hasDateFilter(expression) ? true : hasDateFilter;
402+
return expression;
389403
}
390404
});
405+
const hasTimeFilter = !!(hasMacro || hasDateFilter);
391406
if (this.target.partitioned) {
392407
const partitionedField = this.target.partitionedField ? this.target.partitionedField : '_PARTITIONTIME';
393-
if (this.target.timeColumn !== partitionedField) {
408+
if (this.target.timeColumn !== partitionedField && !hasTimeFilter) {
394409
if (this.templateSrv.timeRange && this.templateSrv.timeRange.from) {
395410
const from = `${partitionedField} >= '${BigQueryQuery.formatDateToString(
396411
this.templateSrv.timeRange.from._d,
412+
this.target.convertToUTC,
397413
'-',
398414
true
399415
)}'`;
@@ -402,6 +418,7 @@ export default class BigQueryQuery {
402418
if (this.templateSrv.timeRange && this.templateSrv.timeRange.to) {
403419
const to = `${partitionedField} < '${BigQueryQuery.formatDateToString(
404420
this.templateSrv.timeRange.to._d,
421+
this.target.convertToUTC,
405422
'-',
406423
true
407424
)}'`;
@@ -410,8 +427,8 @@ export default class BigQueryQuery {
410427
}
411428
}
412429
if (this.target.sharded) {
413-
const from = BigQueryQuery.formatDateToString(this.templateSrv.timeRange.from._d);
414-
const to = BigQueryQuery.formatDateToString(this.templateSrv.timeRange.to._d);
430+
const from = BigQueryQuery.formatDateToString(this.templateSrv.timeRange.from._d, this.target.convertToUTC);
431+
const to = BigQueryQuery.formatDateToString(this.templateSrv.timeRange.to._d, this.target.convertToUTC);
415432
const sharded = "_TABLE_SUFFIX BETWEEN '" + from + "' AND '" + to + "' ";
416433
conditions.push(sharded);
417434
}
@@ -536,24 +553,18 @@ export default class BigQueryQuery {
536553
public expend_macros(options) {
537554
if (this.target.rawSql) {
538555
let q = this.target.rawSql;
539-
q = BigQueryQuery.replaceTimeShift(q);
540-
q = this.replaceTimeFilters(q, options);
541-
q = this.replacetimeGroupAlias(q, true, options);
542-
q = this.replacetimeGroupAlias(q, false, options);
543-
return q;
556+
let hasTimeFilter, hasTimeGroup, hasTimeGroupAlias = false;
557+
q = BigQueryQuery.replaceTimeShift(q); // should also block additional partition time filtering?
558+
[q, hasTimeFilter] = this.replaceTimeFilters(q, options);
559+
[q, hasTimeGroup] = this.replacetimeGroupAlias(q, true, options);
560+
[q, hasTimeGroupAlias] = this.replacetimeGroupAlias(q, false, options);
561+
return [q, hasTimeFilter || hasTimeGroup || hasTimeGroupAlias, this.target.convertToUTC];
544562
}
545563
}
546564
public replaceTimeFilters(q, options) {
547-
let fromD = options.range.from;
548-
let toD = options.range.to;
549-
if (this.target.convertToUTC === true) {
550-
fromD = BigQueryQuery.convertToUtc(options.range.from._d);
551-
toD = BigQueryQuery.convertToUtc(options.range.to._d);
552-
}
553-
let to = '';
554-
let from = '';
555-
from = this._getDateRangePart(fromD);
556-
to = this._getDateRangePart(toD);
565+
const { from: fromD, to: toD } = options.range;
566+
const from = this._getDateRangePart(fromD, this.target.convertToUTC);
567+
const to = this._getDateRangePart(toD, this.target.convertToUTC);
557568
if (this.target.timeColumn === '-- time --') {
558569
const myRegexp = /\$__timeFilter\(([\w_.]+)\)/g;
559570
const tf = myRegexp.exec(q);
@@ -564,26 +575,27 @@ export default class BigQueryQuery {
564575
const range = BigQueryQuery.quoteFiledName(this.target.timeColumn) + ' BETWEEN ' + from + ' AND ' + to;
565576
const fromRange = BigQueryQuery.quoteFiledName(this.target.timeColumn) + ' > ' + from + ' ';
566577
const toRange = BigQueryQuery.quoteFiledName(this.target.timeColumn) + ' < ' + to + ' ';
578+
const hasMacro = q.match(/(\b__timeFilter\b)|(\b__timeFrom\b)|(\b__timeTo\b)|(\b__millisTimeTo\b)|(\b__millisTimeFrom\b)/g)
567579
q = q.replace(/\$__timeFilter\((.*?)\)/g, range);
568580
q = q.replace(/\$__timeFrom\(([\w_.]+)\)/g, fromRange);
569581
q = q.replace(/\$__timeTo\(([\w_.]+)\)/g, toRange);
570582
q = q.replace(/\$__millisTimeTo\(([\w_.]+)\)/g, to);
571583
q = q.replace(/\$__millisTimeFrom\(([\w_.]+)\)/g, from);
572-
return q;
584+
return [q, hasMacro];
573585
}
574586

575587
public replacetimeGroupAlias(q, alias: boolean, options) {
576588
const res = BigQueryQuery._getInterval(q, alias);
577589
const interval = res[0];
578590
const mininterval = res[1];
579591
if (!interval) {
580-
return q;
592+
return [q, false];
581593
}
582594
const intervalStr = this.getIntervalStr(interval, mininterval, options);
583595
if (alias) {
584-
return q.replace(/\$__timeGroupAlias\(([\w_.]+,+[a-zA-Z0-9_ ]+.*\))/g, intervalStr);
596+
return [q.replace(/\$__timeGroupAlias\(([\w_.]+,+[a-zA-Z0-9_ ]+.*\))/g, intervalStr), q.match(/(\b__timeGroupAlias\b)/g)];
585597
} else {
586-
return q.replace(/\$__timeGroup\(([\w_.]+,+[a-zA-Z0-9_ ]+.*\))/g, intervalStr);
598+
return [q.replace(/\$__timeGroup\(([\w_.]+,+[a-zA-Z0-9_ ]+.*\))/g, intervalStr), q.match(/(\b__timeGroup\b)/g)];
587599
}
588600
}
589601

@@ -597,13 +609,14 @@ export default class BigQueryQuery {
597609
const seconds = (this.templateSrv.timeRange.to._d - this.templateSrv.timeRange.from._d) / 1000;
598610
return Math.ceil(seconds / options.maxDataPoints) + 's';
599611
}
600-
private _getDateRangePart(part) {
612+
private _getDateRangePart(part, convertToUTC = false) {
613+
// if its TIMESTAMP no need conversion (same value for utc or local), else - need conversion
601614
if (this.target.timeColumnType === 'DATE') {
602-
return "'" + BigQueryQuery.formatDateToString(part, '-') + "'";
615+
return "'" + BigQueryQuery.formatDateToString(part, convertToUTC, '-') + "'"; // "'2021-01-31'"
603616
} else if (this.target.timeColumnType === 'DATETIME') {
604-
return "'" + BigQueryQuery.formatDateToString(part, '-', true) + "'";
617+
return "'" + BigQueryQuery.formatDateToString(part, convertToUTC, '-', true) + "'"; // "'2021-01-31 19:41:45'"
605618
} else {
606-
return 'TIMESTAMP_MILLIS (' + part.valueOf().toString() + ')';
619+
return 'TIMESTAMP_MILLIS (' + part.valueOf().toString() + ')'; // "TIMESTAMP_MILLIS (1612056873199)"
607620
}
608621
}
609622
}

‎src/datasource.ts‎

Lines changed: 19 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -128,7 +128,7 @@ export class BigQueryDatasource {
128128
return copy;
129129
}
130130

131-
private static _updatePartition(q, options) {
131+
private static _updatePartition(q, options, convertToUTC = false) {
132132
if (q.indexOf('AND _PARTITIONTIME >= ') < 1) {
133133
return q;
134134
}
@@ -137,26 +137,26 @@ export class BigQueryDatasource {
137137
}
138138
const from = q.substr(q.indexOf('AND _PARTITIONTIME >= ') + 22, 21);
139139

140-
const newFrom = "'" + BigQueryQuery.formatDateToString(options.range.from._d, '-', true) + "'";
140+
const newFrom = "'" + BigQueryQuery.formatDateToString(options.range.from._d, convertToUTC, '-', true) + "'";
141141
q = q.replace(from, newFrom);
142142
const to = q.substr(q.indexOf('AND _PARTITIONTIME < ') + 21, 21);
143-
const newTo = "'" + BigQueryQuery.formatDateToString(options.range.to._d, '-', true) + "'";
143+
const newTo = "'" + BigQueryQuery.formatDateToString(options.range.to._d, convertToUTC, '-', true) + "'";
144144

145145
q = q.replace(to, newTo) + '\n ';
146146
return q;
147147
}
148148

149-
private static _updateTableSuffix(q, options) {
149+
private static _updateTableSuffix(q, options, convertToUTC = false) {
150150
const ind = q.indexOf('AND _TABLE_SUFFIX BETWEEN ');
151151
if (ind < 1) {
152152
return q;
153153
}
154154
const from = q.substr(ind + 28, 8);
155155

156-
const newFrom = BigQueryQuery.formatDateToString(options.range.from._d);
156+
const newFrom = BigQueryQuery.formatDateToString(options.range.from._d, convertToUTC);
157157
q = q.replace(from, newFrom);
158158
const to = q.substr(ind + 43, 8);
159-
const newTo = BigQueryQuery.formatDateToString(options.range.to._d);
159+
const newTo = BigQueryQuery.formatDateToString(options.range.to._d, convertToUTC);
160160
q = q.replace(to, newTo) + '\n ';
161161
return q;
162162
}
@@ -270,7 +270,7 @@ export class BigQueryDatasource {
270270
});
271271
this.queryModel.target.rawSql = query.rawSql;
272272
modOptions = BigQueryDatasource._setupTimeShiftQuery(query, options);
273-
const q = this.setUpQ(modOptions, options, query);
273+
const q = this.setUpQ(modOptions, options, query); // TODO hanle raw sql WHERE clause!
274274
console.log(q);
275275
return this.doQuery(q, options.panelId + query.refId, query.queryPriority).then((response) => {
276276
return ResponseParser.parseDataQuery(response, query.format);
@@ -426,7 +426,7 @@ export class BigQueryDatasource {
426426
refId: options.annotation.name,
427427
};
428428
this.queryModel.target.rawSql = query.rawSql;
429-
query.rawSql = this.queryModel.expend_macros(options);
429+
[query.rawSql,] = this.queryModel.expend_macros(options);
430430
return this.backendSrv
431431
.datasourceRequest({
432432
data: {
@@ -444,11 +444,11 @@ export class BigQueryDatasource {
444444
.then((data) => this.responseParser.transformAnnotationResponse(options, data));
445445
}
446446
private setUpQ(modOptions, options, query) {
447-
let q = this.queryModel.expend_macros(modOptions);
447+
let [q, hasMacro, convertToUTC] = this.queryModel.expend_macros(modOptions);
448448
if (q) {
449-
q = this.setUpPartition(q, query.partitioned, query.partitionedField, modOptions);
450-
q = BigQueryDatasource._updatePartition(q, modOptions);
451-
q = BigQueryDatasource._updateTableSuffix(q, modOptions);
449+
q = this.setUpPartition(q, query.partitioned, query.partitionedField, modOptions, hasMacro, convertToUTC);
450+
q = BigQueryDatasource._updatePartition(q, modOptions, convertToUTC);
451+
q = BigQueryDatasource._updateTableSuffix(q, modOptions, convertToUTC);
452452
if (query.refId.search(Shifted) > -1) {
453453
q = this._updateAlias(q, modOptions, query.refId);
454454
}
@@ -466,19 +466,19 @@ export class BigQueryDatasource {
466466
return q;
467467
}
468468
/**
469-
* Add partition to query unless it has one
469+
* Add partition to query unless it has one OR already being ranged by other condition
470470
* @param query
471471
* @param isPartitioned
472472
* @param partitionedField
473473
* @param options
474474
*/
475-
private setUpPartition(query, isPartitioned, partitionedField, options) {
475+
private setUpPartition(query, isPartitioned, partitionedField, options, hasMacro = false, convertToUTC = false) {
476476
partitionedField = partitionedField ? partitionedField : '_PARTITIONTIME';
477-
if (isPartitioned && !query.match(new RegExp(partitionedField, "i"))) {
478-
const fromD = BigQueryQuery.convertToUtc(options.range.from._d);
479-
const toD = BigQueryQuery.convertToUtc(options.range.to._d);
480-
const from = `${partitionedField} >= '${BigQueryQuery.formatDateToString(fromD, '-', true)}'`;
481-
const to = `${partitionedField} < '${BigQueryQuery.formatDateToString(toD, '-', true)}'`;
477+
const hasTimeFilter = !!(BigQueryQuery.hasDateFilter(query.split(/where/gi)[1] || "") || hasMacro);
478+
if (isPartitioned && !hasTimeFilter) {
479+
const { from: { _d: fromD }, to: { _d: toD } } = options.range;
480+
const from = `${partitionedField} >= '${BigQueryQuery.formatDateToString(fromD, convertToUTC, '-', true)}'`;
481+
const to = `${partitionedField} < '${BigQueryQuery.formatDateToString(toD, convertToUTC, '-', true)}'`;
482482
const partition = `where ${from} AND ${to} AND `;
483483
if (query.match(/where/i)) query = query.replace(/where/i, partition);
484484
else {

0 commit comments

Comments
 (0)