From 59cb26b64d1f069eb0625d7948562fcf80a71b80 Mon Sep 17 00:00:00 2001 From: Drew Davis Date: Wed, 14 Oct 2026 12:41:31 -0400 Subject: [PATCH] feat: Broadcast adhoc dashboard filter conditions --- .changeset/adhoc-filter-broadcast.md | 6 + packages/app/src/DBDashboardPage.tsx | 46 +++-- .../app/src/__tests__/searchFilters.test.ts | 3 - .../AdhocDashboardFilter.tsx | 5 + .../__tests__/AdhocDashboardFilter.test.tsx | 21 ++- .../__tests__/useAdhocFilterOptions.test.tsx | 6 + .../useAdhocFilterOptions.ts | 19 +- .../DBEditTimeChartForm/EditTimeChartForm.tsx | 8 +- .../__tests__/utils.test.ts | 60 +++++- .../components/DBEditTimeChartForm/utils.ts | 35 +++- .../components/FilterPill/filterOperators.ts | 21 ++- .../__tests__/useDashboardFilters.test.tsx | 176 +++++++++++++++++- .../usePresetDashboardFilters.test.tsx | 4 +- .../app/src/hooks/useDashboardFilters.tsx | 55 ++++-- .../features/dashboard-adhoc-filters.spec.ts | 149 ++++++++++++++- packages/common-utils/package.json | 3 + .../src/__tests__/filters.test.ts | 150 +++++++++++++++ .../common-utils/src/__tests__/macros.test.ts | 74 ++++++++ .../src/__tests__/renderChartConfig.test.ts | 59 ++++++ .../src/__tests__/variables.test.ts | 4 + .../__tests__/promqlLabelInjection.test.ts | 45 +++++ .../src/core/promqlLabelInjection.ts | 93 +++++++++ packages/common-utils/src/filters.ts | 141 +++++++++++++- packages/common-utils/src/macros.ts | 48 ++++- packages/common-utils/src/types.ts | 52 ++++-- packages/common-utils/src/variables.ts | 9 +- yarn.lock | 31 +-- 27 files changed, 1211 insertions(+), 112 deletions(-) create mode 100644 .changeset/adhoc-filter-broadcast.md create mode 100644 packages/common-utils/src/core/__tests__/promqlLabelInjection.test.ts create mode 100644 packages/common-utils/src/core/promqlLabelInjection.ts diff --git a/.changeset/adhoc-filter-broadcast.md b/.changeset/adhoc-filter-broadcast.md new file mode 100644 index 0000000000..c4f2cbb576 --- /dev/null +++ b/.changeset/adhoc-filter-broadcast.md @@ -0,0 +1,6 @@ +--- +'@hyperdx/common-utils': patch +'@hyperdx/app': patch +--- + +feat: Broadcast adhoc dashboard filter conditions diff --git a/packages/app/src/DBDashboardPage.tsx b/packages/app/src/DBDashboardPage.tsx index ada2e7e680..61f78b91ef 100644 --- a/packages/app/src/DBDashboardPage.tsx +++ b/packages/app/src/DBDashboardPage.tsx @@ -35,6 +35,10 @@ import { isTimeSeriesDisplayType, } from '@hyperdx/common-utils/dist/core/utils'; import { getBlockingRequiredFilterNames } from '@hyperdx/common-utils/dist/dashboardFilterValues'; +import { + getPromqlFilters, + getSqlFilters, +} from '@hyperdx/common-utils/dist/filters'; import { displayTypeRequiresSource, isBuilderChartConfig, @@ -51,12 +55,12 @@ import { import { isMissingFiltersMacro } from '@hyperdx/common-utils/dist/macros'; import { AlertState, + BroadcastFilter, ChartConfigWithDateRange, ChartVariable, DashboardContainer as DashboardContainerSchema, DashboardFilter, DisplayType, - Filter, getSampleWeightExpression, isLogSource, isTraceSource, @@ -467,7 +471,7 @@ const Tile = ({ onSettled?: () => void; granularity: SQLInterval | undefined; onTimeRangeSelect: (start: Date, end: Date) => void; - filters?: Filter[]; + filters?: BroadcastFilter[]; variables?: ChartVariable[]; /** The dashboard's required filters that have nothing selected. */ unsatisfiedRequiredFilters?: DashboardFilter[]; @@ -650,6 +654,15 @@ const Tile = ({ missingRequiredFilterNames.length > 0 && displayTypeRequiresSource(chart.config.displayType); + const sqlFilters = useMemo( + () => (filters ? getSqlFilters(filters) : undefined), + [filters], + ); + const promqlFilters = useMemo( + () => (filters ? getPromqlFilters(filters) : undefined), + [filters], + ); + useEffect(() => { if (isPromqlSavedChartConfig(chart.config)) { if (source != null) { @@ -660,6 +673,7 @@ const Tile = ({ dateRange, granularity, variables: tileVariables, + filters: promqlFilters, }); } return; @@ -672,7 +686,7 @@ const Tile = ({ ...chart.config, dateRange, granularity, - filters, + filters: sqlFilters, variables: tileVariables, }); } else if (source != null) { @@ -691,7 +705,7 @@ const Tile = ({ sampleWeightExpression: getSampleWeightExpression(source), dateRange, granularity, - filters, + filters: sqlFilters, variables: tileVariables, }); } @@ -732,13 +746,21 @@ const Tile = ({ ? source.bodyExpression : undefined, sampleWeightExpression: getSampleWeightExpression(source), - filters, + filters: sqlFilters, variables: tileVariables, metricTables: isMetricSource ? source.metricTables : undefined, }); } } - }, [source, chart, dateRange, granularity, filters, tileVariables]); + }, [ + source, + chart, + dateRange, + granularity, + sqlFilters, + promqlFilters, + tileVariables, + ]); const [hovered, setHovered] = useState(false); @@ -809,7 +831,7 @@ const Tile = ({ where: builderConfig?.where, whereLanguage: builderConfig?.whereLanguage, select: builderConfig?.select, - filters, + filters: sqlFilters, }, ); @@ -819,10 +841,10 @@ const Tile = ({ ); const filterWarning = useMemo(() => { - const doFiltersExist = !!filters?.filter( + const doFiltersExist = !!sqlFilters?.filter( f => (f.type === 'lucene' || f.type === 'sql') && f.condition.trim(), )?.length; - const doLuceneFiltersExist = !!filters?.filter( + const doLuceneFiltersExist = !!sqlFilters?.filter( f => f.type === 'lucene' && f.condition.trim(), )?.length; @@ -858,7 +880,7 @@ const Tile = ({ ); - }, [filters, queriedConfig, source]); + }, [sqlFilters, queriedConfig, source]); const replaySearchUrl = useMemo(() => { return buildDashboardReplaySearchUrl({ @@ -1548,7 +1570,7 @@ const EditTileModal = ({ isSaving?: boolean; onSave: (chart: Tile) => void; variables?: ChartVariable[]; - getDashboardFilters: (sourceId: string | undefined) => Filter[]; + getDashboardFilters: (sourceId: string | undefined) => BroadcastFilter[]; unsatisfiedRequiredFilters?: DashboardFilter[]; }) => { const contextZIndex = useZIndex(); @@ -2304,7 +2326,7 @@ function DBDashboardPage({ // `sourceId`. Shared with the tile editor so its preview queries what the // tile does. const getTileFilters = useCallback( - (sourceId: string | undefined): Filter[] => [ + (sourceId: string | undefined): BroadcastFilter[] => [ { type: whereLanguage === 'sql' ? 'sql' : 'lucene', condition: where, diff --git a/packages/app/src/__tests__/searchFilters.test.ts b/packages/app/src/__tests__/searchFilters.test.ts index 9cc571438d..a6eec0c225 100644 --- a/packages/app/src/__tests__/searchFilters.test.ts +++ b/packages/app/src/__tests__/searchFilters.test.ts @@ -886,7 +886,6 @@ describe('searchFilters', () => { const query = filtersToQuery(originalFilters, { dateTimeColumns }); // Sanity: produces the DateTime (non-64) wrapper. - // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion expect((query[0] as ConditionFilter).condition).toBe( "TimestampTime IN (parseDateTimeBestEffort('2026-06-17T11:56:41Z'))", ); @@ -930,7 +929,6 @@ describe('searchFilters', () => { const query = filtersToQuery(originalFilters, { dateTimeColumns: aliasColumns, }); - // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion expect((query[0] as ConditionFilter).condition).toBe( "time IN (parseDateTime64BestEffort('2026-06-18T10:33:55Z', 9))", ); @@ -957,7 +955,6 @@ describe('searchFilters', () => { const query = filtersToQuery(originalFilters, { dateTimeColumns: exprColumns, }); - // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion expect((query[0] as ConditionFilter).condition).toBe( "toDate(TimestampTime) IN (toDate('2026-06-18'))", ); diff --git a/packages/app/src/components/AdhocDashboardFilter/AdhocDashboardFilter.tsx b/packages/app/src/components/AdhocDashboardFilter/AdhocDashboardFilter.tsx index a6c2bae9ba..959f1631e2 100644 --- a/packages/app/src/components/AdhocDashboardFilter/AdhocDashboardFilter.tsx +++ b/packages/app/src/components/AdhocDashboardFilter/AdhocDashboardFilter.tsx @@ -1,4 +1,5 @@ import { useState } from 'react'; +import { isAdhocConditionSupported } from '@hyperdx/common-utils/dist/filters'; import { ADHOC_FILTER_MAX_CONDITIONS, AdhocDashboardFilter as AdhocDashboardFilterType, @@ -103,6 +104,10 @@ export function AdhocDashboardFilter({ operator={operator.label} value={condition.value} isExcluded={operator.isExcluded} + isInvalid={ + !isAdhocConditionSupported(filter.sourceType, condition) + } + invalidReason={`Filter not applied: PromQL has no ${condition.operator} operator. Remove this condition and add it again with a supported operator.`} onRemove={() => remove(index)} renderPopover={close => ( screen.getByTestId('adhoc-filter-add-Conditions'); -const renderFilter = (conditions: AdhocFilterCondition[]) => { +const renderFilter = ( + conditions: AdhocFilterCondition[], + definition: AdhocDashboardFilterType = filter, +) => { const onChange = jest.fn(); renderWithMantine( { expect(pills[1]).toHaveTextContent('Level regex ^err'); }); + it('marks conditions a PromQL filter cannot apply as invalid', () => { + renderFilter( + [ + { key: 'job', operator: 'LIKE', value: '%api%' }, + { key: 'env', operator: '=', value: 'prod' }, + ], + { ...filter, sourceType: 'promql' }, + ); + + const pills = screen.getAllByTestId('adhoc-condition-pill'); + expect(pills[0]).toHaveAttribute('data-invalid', 'true'); + expect(pills[1]).not.toHaveAttribute('data-invalid'); + }); + it('removes a condition', async () => { const onChange = renderFilter([ existing, diff --git a/packages/app/src/components/AdhocDashboardFilter/__tests__/useAdhocFilterOptions.test.tsx b/packages/app/src/components/AdhocDashboardFilter/__tests__/useAdhocFilterOptions.test.tsx index 11dd946059..8d0af7d422 100644 --- a/packages/app/src/components/AdhocDashboardFilter/__tests__/useAdhocFilterOptions.test.tsx +++ b/packages/app/src/components/AdhocDashboardFilter/__tests__/useAdhocFilterOptions.test.tsx @@ -86,6 +86,12 @@ describe('getSqlSourceColumnKeys', () => { ]), ).toEqual(['ServiceName']); }); + + it('quotes column names that are not bare identifiers', () => { + expect(getSqlSourceColumnKeys([column('service-name', 'String')])).toEqual([ + '`service-name`', + ]); + }); }); describe('getValueSuggestions', () => { diff --git a/packages/app/src/components/AdhocDashboardFilter/useAdhocFilterOptions.ts b/packages/app/src/components/AdhocDashboardFilter/useAdhocFilterOptions.ts index e54151862d..fbb2aa54c4 100644 --- a/packages/app/src/components/AdhocDashboardFilter/useAdhocFilterOptions.ts +++ b/packages/app/src/components/AdhocDashboardFilter/useAdhocFilterOptions.ts @@ -45,26 +45,37 @@ const unionSorted = (lists: (string[] | undefined)[]) => Array.from(new Set(lists.flatMap(list => list ?? []))).sort(); /** - * A SQL source's filterable keys, in the search sidebar's key format: scalar - * columns, and the keys inside its map and JSON columns. + * Quote a column name that isn't a bare identifier (e.g. `log-attributes`), so + * the stored key is SQL that broadcasting can use verbatim. + */ +const toQuotedKeys = (keys: string[], columns: ColumnMeta[]) => { + const columnNames = new Set(columns.map(column => column.name)); + return keys.map(key => toQuotedClickHouseKeyExpression(key, columnNames)); +}; + +/** + * A SQL source's filterable keys, as SQL expressions in the search sidebar's + * key format: scalar columns, and the keys inside its map and JSON columns. */ export function getSqlSourceKeys(fields: Field[], columns: ColumnMeta[]) { const columnNames = (types: JSDataType[]) => filterColumnMetaByType(columns, types)?.map(column => column.name) ?? []; const jsonColumns = columnNames([JSDataType.JSON]); const mapColumns = columnNames([JSDataType.Map]); - return fields + const keys = fields .filter(field => field.path.length > 1 || isJSDataTypeScalar(field.jsType)) .map(field => mergePath(field.path, jsonColumns, mapColumns)); + return toQuotedKeys(keys, columns); } /** A SQL source's scalar columns, the only keys known without field metadata. */ export function getSqlSourceColumnKeys(columns: ColumnMeta[]) { - return columns + const keys = columns .filter(column => isJSDataTypeScalar(convertCHDataTypeToJSType(column.type)), ) .map(column => column.name); + return toQuotedKeys(keys, columns); } /** diff --git a/packages/app/src/components/DBEditTimeChartForm/EditTimeChartForm.tsx b/packages/app/src/components/DBEditTimeChartForm/EditTimeChartForm.tsx index bf2043d055..0cc4e83795 100644 --- a/packages/app/src/components/DBEditTimeChartForm/EditTimeChartForm.tsx +++ b/packages/app/src/components/DBEditTimeChartForm/EditTimeChartForm.tsx @@ -21,11 +21,11 @@ import { isRawSqlSavedChartConfig, } from '@hyperdx/common-utils/dist/guards'; import { + BroadcastFilter, ChartConfigWithDateRange, ChartVariable, DashboardFilter, DisplayType, - Filter, PromqlHeatmapMode, SavedChartConfig, SourceKind, @@ -111,6 +111,7 @@ import { displayTypeToActiveTab, resolveTilePreviewFilters, TABS_WITH_GENERATED_SQL, + withBroadcastFilters, zSavedChartConfig, } from './utils'; @@ -120,7 +121,7 @@ type EditTimeChartFormProps = { /** Variables and their selected values, from the parent dashboard (if one exists). */ variables?: ChartVariable[]; /** Function returning the dashboard's broadcast filters for the given source, if any. */ - getDashboardFilters?: (sourceId: string | undefined) => Filter[]; + getDashboardFilters?: (sourceId: string | undefined) => BroadcastFilter[]; /** The dashboard's required filters that have nothing selected, if any */ unsatisfiedRequiredFilters?: DashboardFilter[]; displayedTimeInputValue?: string; @@ -543,8 +544,7 @@ export default function EditTimeChartForm({ return queriedConfig; } return { - ...queriedConfig, - filters: previewDashboardFilters?.filters, + ...withBroadcastFilters(queriedConfig, previewDashboardFilters?.filters), variables: previewDashboardFilters?.variables, }; }, [queriedConfig, previewDashboardFilters]); diff --git a/packages/app/src/components/DBEditTimeChartForm/__tests__/utils.test.ts b/packages/app/src/components/DBEditTimeChartForm/__tests__/utils.test.ts index 3db717adf2..2aa2410988 100644 --- a/packages/app/src/components/DBEditTimeChartForm/__tests__/utils.test.ts +++ b/packages/app/src/components/DBEditTimeChartForm/__tests__/utils.test.ts @@ -3,6 +3,8 @@ import { ChartVariable, DashboardFilter, DisplayType, + Filter, + PromqlConditionFilter, PromqlExpressionList, SavedChartConfig, SourceKind, @@ -23,6 +25,7 @@ import { seriesToFilters, tabQueriesData, TABS_WITH_GENERATED_SQL, + withBroadcastFilters, } from '@/components/DBEditTimeChartForm/utils'; // --------------------------------------------------------------------------- @@ -646,6 +649,10 @@ describe('resolveTilePreviewFilters', () => { { type: 'sql' as const, condition: "ServiceName IN ('api')" }, ]; const variables = [{ name: 'service', values: ['api'] }]; + const promqlFilter: PromqlConditionFilter = { + type: 'promql', + condition: 'job="api"', + }; const promqlConfig: ChartConfigWithDateRange = { configType: 'promql', @@ -704,8 +711,30 @@ describe('resolveTilePreviewFilters', () => { }); }); - it('never hands filters to a PromQL tile', () => { - expect(resolve({ config: promqlConfig }).filters).toBeUndefined(); + it('never hands SQL filters to a PromQL tile', () => { + expect(resolve({ config: promqlConfig }).filters).toEqual([]); + }); + + it('hands each tile the filters in its own query language', () => { + const filters = [...dashboardFilters, promqlFilter]; + + expect(resolve({ config: promqlConfig, filters }).filters).toEqual([ + promqlFilter, + ]); + expect(resolve({ filters }).filters).toEqual(dashboardFilters); + expect( + resolve({ config: rawSqlWithFiltersMacro, filters }).filters, + ).toEqual(dashboardFilters); + }); + + it('drops PromQL filters when turned off', () => { + expect( + resolve({ + config: promqlConfig, + filters: [promqlFilter], + applySelections: false, + }).filters, + ).toBeUndefined(); }); it('drops the filters a raw-SQL template would not apply', () => { @@ -769,6 +798,33 @@ describe('resolveTilePreviewFilters', () => { }); }); +describe('withBroadcastFilters', () => { + const sqlFilter: Filter = { type: 'sql', condition: "ServiceName = 'api'" }; + const promqlFilter: PromqlConditionFilter = { + type: 'promql', + condition: 'job="api"', + }; + const promqlConfig: ChartConfigWithDateRange = { + configType: 'promql', + promqlExpression: 'up', + connection: 'local', + source: 'source-promql', + dateRange, + }; + + it('attaches only PromQL filters to a PromQL config', () => { + expect( + withBroadcastFilters(promqlConfig, [sqlFilter, promqlFilter]), + ).toMatchObject({ filters: [promqlFilter] }); + }); + + it('attaches only SQL and Lucene filters to other configs', () => { + expect( + withBroadcastFilters(builderConfig, [sqlFilter, promqlFilter]), + ).toMatchObject({ filters: [sqlFilter] }); + }); +}); + // --------------------------------------------------------------------------- // buildSampleEventsConfig // --------------------------------------------------------------------------- diff --git a/packages/app/src/components/DBEditTimeChartForm/utils.ts b/packages/app/src/components/DBEditTimeChartForm/utils.ts index 785ad572f9..d49cc65bc1 100644 --- a/packages/app/src/components/DBEditTimeChartForm/utils.ts +++ b/packages/app/src/components/DBEditTimeChartForm/utils.ts @@ -12,6 +12,10 @@ import { configConsumesBroadcastFilters, getBlockingRequiredFilterNames, } from '@hyperdx/common-utils/dist/dashboardFilterValues'; +import { + getPromqlFilters, + getSqlFilters, +} from '@hyperdx/common-utils/dist/filters'; import { isBuilderChartConfig, isPromqlChartConfig, @@ -20,6 +24,7 @@ import { } from '@hyperdx/common-utils/dist/guards'; import { substitutePromqlChartConfigTemplates } from '@hyperdx/common-utils/dist/macros'; import { + BroadcastFilter, BuilderChartConfigWithDateRange, ChartAlertBaseSchema, ChartConfigWithDateRange, @@ -201,8 +206,8 @@ export function resolvePreviewVariables({ /** What the dashboard's filter state contributes to a tile preview. */ export type TilePreviewFilters = { - /** Broadcast filter conditions to query the preview with. */ - filters: Filter[] | undefined; + /** Broadcast filter conditions to query the preview with, in its query language. */ + filters: BroadcastFilter[] | undefined; /** The referenced variables, with or without their selections. */ variables: ChartVariable[] | undefined; /** Names of the required filters that have nothing selected. */ @@ -225,7 +230,7 @@ export function resolveTilePreviewFilters({ }: { config: ChartConfigWithDateRange; sourceId: string | undefined; - filters: Filter[] | undefined; + filters: BroadcastFilter[] | undefined; variables: ChartVariable[] | undefined; unsatisfiedRequiredFilters: DashboardFilter[] | undefined; applySelections: boolean; @@ -244,13 +249,16 @@ export function resolveTilePreviewFilters({ }; } - const consumesBroadcastFilters = configConsumesBroadcastFilters( - config, - sourceId, - ); + // PromQL tiles take only PromQL conditions; `configConsumesBroadcastFilters` + // stays false for them so that required SQL filters don't block them. + const previewFilters = isPromqlChartConfig(config) + ? filters && getPromqlFilters(filters) + : configConsumesBroadcastFilters(config, sourceId) && filters + ? getSqlFilters(filters) + : undefined; return { - filters: consumesBroadcastFilters ? filters : undefined, + filters: previewFilters, variables: previewVariables, missingRequiredFilterNames: getBlockingRequiredFilterNames({ config, @@ -261,6 +269,17 @@ export function resolveTilePreviewFilters({ }; } +/** The config with the broadcast filters in its own query language attached. */ +export function withBroadcastFilters( + config: ChartConfigWithDateRange, + filters: BroadcastFilter[] | undefined, +): ChartConfigWithDateRange { + if (isPromqlChartConfig(config)) { + return { ...config, filters: filters && getPromqlFilters(filters) }; + } + return { ...config, filters: filters && getSqlFilters(filters) }; +} + /** One expression as a PromQL tile queries it. */ type RenderedPromqlEntry = { id: string; diff --git a/packages/app/src/components/FilterPill/filterOperators.ts b/packages/app/src/components/FilterPill/filterOperators.ts index 62ba56ff5d..ece901cbf7 100644 --- a/packages/app/src/components/FilterPill/filterOperators.ts +++ b/packages/app/src/components/FilterPill/filterOperators.ts @@ -1,15 +1,20 @@ +import { + AdhocFilterOperator, + PROMQL_ADHOC_FILTER_OPERATORS, +} from '@hyperdx/common-utils/dist/types'; + import type { FilterOperatorOption } from './FilterConditionEditor'; export type FilterLanguage = 'sql' | 'promql'; /** One spelling for both languages; each preset's labels show its own wording. */ -export type FilterOperator = '=' | '!=' | '=~' | '!~' | 'LIKE' | 'NOT LIKE'; +export type FilterOperator = AdhocFilterOperator; const REGEX_VALUE = { valueLabel: 'Pattern', valuePlaceholder: 'Regular expression', }; -const LIKE_VALUE = { valuePlaceholder: 'Text the value contains' }; +const LIKE_VALUE = { valuePlaceholder: 'Pattern, e.g. %error%' }; export const SQL_FILTER_OPERATORS: FilterOperatorOption[] = [ { value: '=', label: '=' }, @@ -20,12 +25,12 @@ export const SQL_FILTER_OPERATORS: FilterOperatorOption[] = [ { value: 'NOT LIKE', label: 'not like', negated: true, ...LIKE_VALUE }, ]; -export const PROMQL_FILTER_OPERATORS: FilterOperatorOption[] = [ - { value: '=', label: '=' }, - { value: '!=', label: '!=', negated: true }, - { value: '=~', label: '=~', ...REGEX_VALUE }, - { value: '!~', label: '!~', negated: true, ...REGEX_VALUE }, -]; +// Derived so the editor never offers an operator the PromQL broadcast skips. +// PromQL users know the operator syntax, so it doubles as the label. +export const PROMQL_FILTER_OPERATORS: FilterOperatorOption[] = + SQL_FILTER_OPERATORS.filter(op => + PROMQL_ADHOC_FILTER_OPERATORS.includes(op.value), + ).map(op => ({ ...op, label: op.value })); export function getFilterOperators( language: FilterLanguage, diff --git a/packages/app/src/hooks/__tests__/useDashboardFilters.test.tsx b/packages/app/src/hooks/__tests__/useDashboardFilters.test.tsx index 1ea51eb2fc..a84b0ef080 100644 --- a/packages/app/src/hooks/__tests__/useDashboardFilters.test.tsx +++ b/packages/app/src/hooks/__tests__/useDashboardFilters.test.tsx @@ -2,9 +2,9 @@ import { ADHOC_FILTER_MAX_CONDITIONS, AdhocDashboardFilter, AdhocFilterCondition, + BroadcastFilter, DashboardFilter, DashboardFilterValue, - Filter, QueryExpressionDashboardFilter, } from '@hyperdx/common-utils/dist/types'; import { act, renderHook } from '@testing-library/react'; @@ -65,8 +65,8 @@ describe('useDashboardFilters', () => { filterId: string, ) => result.current.selectionByFilterId.get(filterId); - const conditionsFor = (queries: Filter[]) => - queries.map(q => ('condition' in q ? q.condition : '')); + const conditionsFor = (queries: BroadcastFilter[]) => + queries.map(q => q.condition); beforeEach(() => { mockState = null; @@ -1121,14 +1121,180 @@ describe('useDashboardFilters', () => { ]); }); - it('exposes no variable, selection, or broadcast query', () => { + it('exposes no variable or selection', () => { mockState = [{ type: 'adhoc', name: 'conds', conditions: [condition] }]; const { result } = renderHook(() => useDashboardFilters([adhocFilter])); expect(result.current.variables).toEqual([]); expect(result.current.selectionByFilterId.size).toBe(0); - expect(result.current.broadcastedFilters).toEqual([]); + }); + + describe('broadcast', () => { + const conditionSql = "toString(ServiceName) != 'api'"; + + it("applies conditions to tiles on the filter's sources only", () => { + mockState = [{ type: 'adhoc', name: 'conds', conditions: [condition] }]; + + const { result } = renderHook(() => useDashboardFilters([adhocFilter])); + + expect( + conditionsFor(result.current.getFilterQueriesForSource('logs')), + ).toEqual([conditionSql]); + expect(result.current.getFilterQueriesForSource('traces')).toEqual([]); + expect(result.current.getFilterQueriesForSource(undefined)).toEqual([]); + }); + + it('applies conditions to appliesToSourceIds over sources', () => { + mockState = [{ type: 'adhoc', name: 'conds', conditions: [condition] }]; + const filters = [{ ...adhocFilter, appliesToSourceIds: ['traces'] }]; + + const { result } = renderHook(() => useDashboardFilters(filters)); + + expect(result.current.getFilterQueriesForSource('logs')).toEqual([]); + expect( + conditionsFor(result.current.getFilterQueriesForSource('traces')), + ).toEqual([conditionSql]); + }); + + it('applies nothing when broadcast is disabled', () => { + mockState = [{ type: 'adhoc', name: 'conds', conditions: [condition] }]; + const filters = [{ ...adhocFilter, isBroadcastEnabled: false }]; + + const { result } = renderHook(() => useDashboardFilters(filters)); + + expect(result.current.getFilterQueriesForSource('logs')).toEqual([]); + }); + + describe('PromQL', () => { + const promqlFilter: AdhocDashboardFilter = { + ...adhocFilter, + sourceType: 'promql', + sources: ['prom'], + }; + const jobMatcher = 'job="api"'; + + beforeEach(() => { + mockState = [ + { + type: 'adhoc', + name: 'conds', + conditions: [{ key: 'job', operator: '=', value: 'api' }], + }, + ]; + }); + + it("applies conditions as label matchers to the filter's sources only", () => { + const { result } = renderHook(() => + useDashboardFilters([promqlFilter]), + ); + + expect(result.current.getFilterQueriesForSource('prom')).toEqual([ + { type: 'promql', condition: jobMatcher }, + ]); + expect(result.current.getFilterQueriesForSource('prom2')).toEqual([]); + expect(result.current.getFilterQueriesForSource(undefined)).toEqual( + [], + ); + }); + + it('applies conditions to appliesToSourceIds over sources', () => { + const filters = [{ ...promqlFilter, appliesToSourceIds: ['prom2'] }]; + + const { result } = renderHook(() => useDashboardFilters(filters)); + + expect(result.current.getFilterQueriesForSource('prom')).toEqual([]); + expect( + conditionsFor(result.current.getFilterQueriesForSource('prom2')), + ).toEqual([jobMatcher]); + }); + + it('applies nothing when broadcast is disabled', () => { + const filters = [{ ...promqlFilter, isBroadcastEnabled: false }]; + + const { result } = renderHook(() => useDashboardFilters(filters)); + + expect(result.current.getFilterQueriesForSource('prom')).toEqual([]); + }); + + it('combines conditions across filters, deduplicated', () => { + mockState = [ + { + type: 'adhoc', + name: 'conds', + conditions: [{ key: 'job', operator: '=', value: 'api' }], + }, + { + type: 'adhoc', + name: 'conds2', + conditions: [ + { key: 'job', operator: '=', value: 'api' }, + { key: 'env', operator: '!=', value: 'dev' }, + ], + }, + ]; + const filters = [ + promqlFilter, + { ...promqlFilter, id: 'adhoc2', variableName: 'conds2' }, + ]; + + const { result } = renderHook(() => useDashboardFilters(filters)); + + expect( + conditionsFor(result.current.getFilterQueriesForSource('prom')), + ).toEqual([jobMatcher, 'env!="dev"']); + }); + + it('skips conditions PromQL cannot express rather than throwing', () => { + // Kept from before the filter was switched from SQL to PromQL + mockState = [ + { + type: 'adhoc', + name: 'conds', + conditions: [ + { key: 'job', operator: 'LIKE', value: '%api%' }, + { key: 'job', operator: '=', value: 'api' }, + ], + }, + ]; + + const { result } = renderHook(() => + useDashboardFilters([promqlFilter]), + ); + + expect(result.current.broadcastedFilters).toEqual([]); + expect( + conditionsFor(result.current.getFilterQueriesForSource('prom')), + ).toEqual([jobMatcher]); + }); + + it('leaves them out of the unscoped broadcast filters', () => { + const { result } = renderHook(() => + useDashboardFilters([promqlFilter]), + ); + + expect(result.current.broadcastedFilters).toEqual([]); + }); + }); + + it('combines conditions with other filters, deduplicated', () => { + mockState = [ + { type: 'sql', condition: "environment IN ('production')" }, + { type: 'adhoc', name: 'conds', conditions: [condition] }, + { type: 'adhoc', name: 'conds2', conditions: [condition] }, + ]; + const filters: DashboardFilter[] = [ + mockFilters[0], + adhocFilter, + { ...adhocFilter, id: 'adhoc2', variableName: 'conds2' }, + ]; + + const { result } = renderHook(() => useDashboardFilters(filters)); + + expect( + conditionsFor(result.current.getFilterQueriesForSource('logs')), + ).toEqual(["toString(environment) IN ('production')", conditionSql]); + }); }); it('lists an ad hoc entry naming no declared filter as ignored', () => { diff --git a/packages/app/src/hooks/__tests__/usePresetDashboardFilters.test.tsx b/packages/app/src/hooks/__tests__/usePresetDashboardFilters.test.tsx index f3770007cc..5c47539263 100644 --- a/packages/app/src/hooks/__tests__/usePresetDashboardFilters.test.tsx +++ b/packages/app/src/hooks/__tests__/usePresetDashboardFilters.test.tsx @@ -1,7 +1,7 @@ import { FilterSelection } from '@hyperdx/common-utils/dist/dashboardFilterValues'; import { + ConditionFilter, DashboardFilter, - Filter, PresetDashboard, PresetDashboardFilter, } from '@hyperdx/common-utils/dist/types'; @@ -63,7 +63,7 @@ describe('usePresetDashboardFilters', () => { ], ); - const mockFilterQueries: Filter[] = [ + const mockFilterQueries: ConditionFilter[] = [ { type: 'sql', condition: "environment = 'production'", diff --git a/packages/app/src/hooks/useDashboardFilters.tsx b/packages/app/src/hooks/useDashboardFilters.tsx index 547a7de988..1e381a7ad6 100644 --- a/packages/app/src/hooks/useDashboardFilters.tsx +++ b/packages/app/src/hooks/useDashboardFilters.tsx @@ -12,12 +12,15 @@ import { VariableEntryValue, } from '@hyperdx/common-utils/dist/dashboardFilterValues'; import { + adhocConditionsToFilters, + adhocConditionsToPromqlFilters, doesFilterApplyToSource, FilterState, filtersToQuery, getDashboardVariableFilters, getFilterExpression, getFilterVariableName, + getSqlFilters, isAdhocFilter, isFilterBroadcastEnabled, isQueryExpressionFilter, @@ -25,9 +28,9 @@ import { import { AdhocFilterCondition, AdhocFilterValueSchema, + BroadcastFilter, ChartVariable, DashboardFilter, - Filter, } from '@hyperdx/common-utils/dist/types'; import { dashboardFilterValuesParser } from '@/utils/queryParsers'; @@ -256,21 +259,10 @@ const useDashboardFilters = (filters: DashboardFilter[]) => { /** Returns the set of rendered filter conditions corresponding to filters matching the given predicate */ const getFiltersQueriesFor = useCallback( - (predicate: (filter: DashboardFilter) => boolean): Filter[] => { + (predicate: (filter: DashboardFilter) => boolean): BroadcastFilter[] => { const seen = new Set(); - const queries: Filter[] = []; - for (const filter of filters) { - if (!predicate(filter)) continue; - const expression = getFilterExpression(filter); - if (expression == null) continue; - const selection = selectionByFilterId.get(filter.id); - if (!selection) continue; - // Wrap keys in `toString()` to support JSON/Dynamic-type columns. - // All keys can be stringified, since filter select values are stringified as well. - const emitted = filtersToQuery( - { [expression]: selection }, - { stringifyKeys: true }, - ); + const queries: BroadcastFilter[] = []; + const emit = (emitted: BroadcastFilter[]) => { for (const query of emitted) { // Two definitions resolving to the same selection would otherwise // emit an identical condition twice, churning react-query keys. @@ -280,14 +272,38 @@ const useDashboardFilters = (filters: DashboardFilter[]) => { queries.push(query); } } + }; + + for (const filter of filters) { + if (!predicate(filter)) continue; + if (isAdhocFilter(filter)) { + const conditions = adhocConditionsByFilterId.get(filter.id); + if (!conditions) continue; + emit( + filter.sourceType === 'promql' + ? adhocConditionsToPromqlFilters(conditions) + : adhocConditionsToFilters(conditions), + ); + continue; + } + const expression = getFilterExpression(filter); + if (expression == null) continue; + const selection = selectionByFilterId.get(filter.id); + if (!selection) continue; + // Wrap keys in `toString()` to support JSON/Dynamic-type columns. + // All keys can be stringified, since filter select values are stringified as well. + emit( + filtersToQuery({ [expression]: selection }, { stringifyKeys: true }), + ); } return queries; }, - [filters, selectionByFilterId], + [filters, selectionByFilterId, adhocConditionsByFilterId], ); + // Its consumers apply it as SQL; PromQL conditions reach tiles by source only. const broadcastedFilters = useMemo( - () => getFiltersQueriesFor(isFilterBroadcastEnabled), + () => getSqlFilters(getFiltersQueriesFor(isFilterBroadcastEnabled)), [getFiltersQueriesFor], ); @@ -296,7 +312,7 @@ const useDashboardFilters = (filters: DashboardFilter[]) => { * broadcast to tiles with the given sourceId **/ const getFilterQueriesForSource = useCallback( - (sourceId: string | undefined): Filter[] => + (sourceId: string | undefined): BroadcastFilter[] => getFiltersQueriesFor(definition => doesFilterApplyToSource(definition, sourceId), ), @@ -338,7 +354,8 @@ const useDashboardFilters = (filters: DashboardFilter[]) => { * source is `sourceId`. Filters with no `appliesToSourceIds` apply to all * tiles. Filters with `appliesToSourceIds` defined apply only to tiles * whose source ID is in the list. Filters with broadcasting disabled - * apply to no tiles at all. + * apply to no tiles at all. PromQL ad hoc filters yield `promql` label + * matchers; the rest yield SQL or Lucene conditions. */ getFilterQueriesForSource, /** The dashboard's variable-enabled filters and their currently selected values. */ diff --git a/packages/app/tests/e2e/features/dashboard-adhoc-filters.spec.ts b/packages/app/tests/e2e/features/dashboard-adhoc-filters.spec.ts index 4c2bfa9172..5ecf45c4f9 100644 --- a/packages/app/tests/e2e/features/dashboard-adhoc-filters.spec.ts +++ b/packages/app/tests/e2e/features/dashboard-adhoc-filters.spec.ts @@ -1,9 +1,13 @@ /** * An `ADHOC` dashboard filter's conditions (`key operator value`) are picked * from the keys of its sources. The filters modal creates and edits them, and - * the dashboard renders their conditions and keeps them in the URL; they are not - * applied to tiles yet. + * the dashboard renders their conditions and keeps them in the URL. Conditions + * of a SQL filter are broadcast to builder tiles and to raw SQL tiles through + * `$__filters`; those of a PromQL filter are added to every selector of PromQL + * tiles. */ +import { DisplayType } from '@hyperdx/common-utils/dist/types'; + import { DashboardPage } from '../page-objects/DashboardPage'; import { SERVICES } from '../seed-clickhouse'; import { expect, test } from '../utils/base-test'; @@ -11,6 +15,7 @@ import { DEFAULT_LOGS_SOURCE_NAME, DEFAULT_METRICS_SOURCE_NAME, DEFAULT_TRACES_SOURCE_NAME, + E2E_PROMQL_METRIC_NAME, PROMQL_SOURCE_NAME, } from '../utils/constants'; import { expectFiltersParam, filtersParam } from '../utils/filters-param'; @@ -514,5 +519,145 @@ test.describe( ).toBeVisible(); }); }); + + test('narrows builder and raw SQL tiles by its conditions', async () => { + const filterName = 'Conditions'; + const sql = `SELECT ServiceName, count() AS count FROM default.e2e_otel_logs WHERE Timestamp >= fromUnixTimestamp64Milli({startDateMilliseconds:Int64}) AND Timestamp <= fromUnixTimestamp64Milli({endDateMilliseconds:Int64}) AND $__filters GROUP BY ServiceName LIMIT 200`; + const expectServices = async (shown: string[], hidden: string[]) => { + for (const tile of [ + dashboardPage.getTile(0), + dashboardPage.getTile(1), + ]) { + for (const service of shown) { + await expect(tile.getByTitle(service, { exact: true })).toBeVisible( + { timeout: 20000 }, + ); + } + for (const service of hidden) { + await expect(tile.getByTitle(service, { exact: true })).toHaveCount( + 0, + ); + } + } + }; + + await test.step('Add a builder tile and a raw SQL tile on the logs source', async () => { + await dashboardPage.createNewDashboard(); + await dashboardPage.addTile(); + await dashboardPage.chartEditor.createTable({ + chartName: 'Builder tile', + sourceName: DEFAULT_LOGS_SOURCE_NAME, + groupBy: 'ServiceName', + }); + await expect(dashboardPage.getTiles()).toHaveCount(1); + + await dashboardPage.addTile(); + await expect(dashboardPage.chartEditor.nameInput).toBeVisible(); + await dashboardPage.chartEditor.waitForDataToLoad(); + await dashboardPage.chartEditor.setChartType(DisplayType.Table); + await dashboardPage.chartEditor.setChartName('Raw SQL tile'); + await dashboardPage.chartEditor.switchToSqlMode(); + await dashboardPage.chartEditor.selectSource(DEFAULT_LOGS_SOURCE_NAME); + await dashboardPage.chartEditor.typeSqlQuery(sql); + await dashboardPage.chartEditor.runQuery(false); + await dashboardPage.saveTile(); + await expect(dashboardPage.getTiles()).toHaveCount(2); + }); + + await test.step('Create the filter on the logs source', async () => { + await dashboardPage.openEditFiltersModal(); + await dashboardPage.addAdhocFilterToDashboard(filterName, [ + DEFAULT_LOGS_SOURCE_NAME, + ]); + await dashboardPage.closeFiltersModal(); + await expectServices(['accounting', 'frontend'], []); + }); + + await test.step('An = condition keeps only the matching service', async () => { + await dashboardPage.openAddAdhocCondition(filterName); + await dashboardPage.fillAdhocConditionEditor(filterName, { + key: 'ServiceName', + value: 'accounting', + }); + await expectServices(['accounting'], ['frontend']); + }); + + await test.step('A != condition excludes the service', async () => { + await dashboardPage.editAdhocCondition( + filterName, + 'ServiceName = accounting', + { operatorLabel: '!=' }, + ); + await expectServices(['frontend'], ['accounting']); + }); + }); + + test('narrows PromQL tiles by its label matchers', async () => { + const filterName = 'Labels'; + // One `` per returned series. + const seriesAreas = dashboardPage.getTile(0).locator('.recharts-area'); + + await test.step('Add a PromQL tile with no matchers', async () => { + await dashboardPage.createNewDashboard(); + await dashboardPage.addTile(); + await expect(dashboardPage.chartEditor.nameInput).toBeVisible(); + await dashboardPage.chartEditor.waitForDataToLoad(); + await dashboardPage.chartEditor.switchToPromqlMode(); + await dashboardPage.chartEditor.selectSource(PROMQL_SOURCE_NAME); + await dashboardPage.chartEditor.setChartName('PromQL tile'); + await dashboardPage.chartEditor.replacePromqlExpression( + E2E_PROMQL_METRIC_NAME, + ); + await dashboardPage.chartEditor.save(); + await expect(dashboardPage.getTiles()).toHaveCount(1); + await expect(seriesAreas).toHaveCount(SERVICES.length, { + timeout: 20000, + }); + }); + + await test.step('Create the filter on the PromQL source', async () => { + await dashboardPage.openEditFiltersModal(); + await dashboardPage.addAdhocFilterToDashboard( + filterName, + [PROMQL_SOURCE_NAME], + { sourceType: 'Prometheus' }, + ); + await dashboardPage.closeFiltersModal(); + }); + + await test.step('An = condition keeps only the matching series', async () => { + await dashboardPage.openAddAdhocCondition(filterName); + await dashboardPage.fillAdhocConditionEditor(filterName, { + key: 'service', + value: 'accounting', + }); + await expect(seriesAreas).toHaveCount(1, { timeout: 20000 }); + }); + + await test.step('A != condition excludes the series', async () => { + await dashboardPage.editAdhocCondition( + filterName, + 'service = accounting', + { operatorLabel: '!=' }, + ); + await expect(seriesAreas).toHaveCount(SERVICES.length - 1, { + timeout: 20000, + }); + // The legend truncates long series names, so match on the tail. + await expect( + dashboardPage.getTile(0).getByText('accounting"}'), + ).toHaveCount(0); + }); + + await test.step('The tile editor previews the added matcher', async () => { + const editor = dashboardPage.chartEditor; + await dashboardPage.editTile(0); + await editor.runQuery(); + await editor.openGeneratedPromql(); + await expect + .poll(() => editor.getGeneratedPromqlText()) + .toBe(`${E2E_PROMQL_METRIC_NAME}{service!="accounting"}`); + }); + }); }, ); diff --git a/packages/common-utils/package.json b/packages/common-utils/package.json index 696473adb6..1a2aee6743 100644 --- a/packages/common-utils/package.json +++ b/packages/common-utils/package.json @@ -15,6 +15,9 @@ "@clickhouse/client-common": "1.23.0-head.fae5998.1", "@clickhouse/client-web": "1.23.0-head.fae5998.1", "@hyperdx/lucene": "^3.1.1", + "@lezer/highlight": "^1.1.2", + "@lezer/lr": "^1.2.3", + "@prometheus-io/lezer-promql": "^0.311.3", "date-fns": "^2.28.0", "date-fns-tz": "^2.0.0", "handlebars": "^4.7.10", diff --git a/packages/common-utils/src/__tests__/filters.test.ts b/packages/common-utils/src/__tests__/filters.test.ts index 050834af73..b3ea3e004f 100644 --- a/packages/common-utils/src/__tests__/filters.test.ts +++ b/packages/common-utils/src/__tests__/filters.test.ts @@ -1,4 +1,6 @@ import { + adhocConditionsToFilters, + adhocConditionsToPromqlFilters, deriveVariableName, doesFilterApplyToSource, FilterState, @@ -11,7 +13,10 @@ import { getFilterExpression, getFilterVariableName, getPendingFilterValuesVariables, + getPromqlFilters, + getSqlFilters, hasFilterEffect, + isAdhocConditionSupported, isFilterBroadcastEnabled, isFilterGlobalRequirement, isFilterRequired, @@ -29,9 +34,12 @@ import { } from '@/filters'; import type { AdhocDashboardFilter, + AdhocFilterOperator, + AdhocFilterSourceType, ChartVariable, DashboardFilter, Filter, + PromqlConditionFilter, QueryExpressionDashboardFilter, StaticListDashboardFilter, } from '@/types'; @@ -1759,6 +1767,148 @@ describe('filters', () => { }); }); + describe('adhocConditionsToFilters', () => { + const conditionFor = ( + operator: AdhocFilterOperator, + value = 'api', + key = 'ServiceName', + ) => adhocConditionsToFilters([{ key, operator, value }]); + + it.each<[AdhocFilterOperator, string]>([ + ['=', "toString(ServiceName) = 'api'"], + ['!=', "toString(ServiceName) != 'api'"], + ['=~', "match(toString(ServiceName), 'api')"], + ['!~', "NOT match(toString(ServiceName), 'api')"], + ['LIKE', "toString(ServiceName) LIKE 'api'"], + ['NOT LIKE', "toString(ServiceName) NOT LIKE 'api'"], + ])('renders %s', (operator, condition) => { + expect(conditionFor(operator)).toEqual([{ type: 'sql', condition }]); + }); + + it('passes LIKE patterns through as written', () => { + expect(conditionFor('LIKE', '%err_r%')).toEqual([ + { type: 'sql', condition: "toString(ServiceName) LIKE '%err_r%'" }, + ]); + }); + + it('escapes quotes and backslashes in values', () => { + expect(conditionFor('=', "it's \\ here")).toEqual([ + { + type: 'sql', + condition: "toString(ServiceName) = 'it''s \\\\ here'", + }, + ]); + }); + + it('uses keys verbatim', () => { + expect(conditionFor('=', 'api', "`log-attrs`['service.name']")).toEqual([ + { + type: 'sql', + condition: "toString(`log-attrs`['service.name']) = 'api'", + }, + ]); + }); + + it('returns no filters for no conditions', () => { + expect(adhocConditionsToFilters([])).toEqual([]); + }); + }); + + describe('adhocConditionsToPromqlFilters', () => { + const matcherFor = ( + operator: AdhocFilterOperator, + value = 'api', + key = 'service', + ) => + adhocConditionsToPromqlFilters([{ key, operator, value }]).map( + filter => filter.condition, + ); + + it.each<[AdhocFilterOperator, string]>([ + ['=', 'service="api"'], + ['!=', 'service!="api"'], + ['=~', 'service=~"api"'], + ['!~', 'service!~"api"'], + ])('renders %s', (operator, matcher) => { + expect(matcherFor(operator)).toEqual([matcher]); + }); + + it.each(['LIKE', 'NOT LIKE'])( + 'skips %s rather than throwing', + operator => { + expect( + adhocConditionsToPromqlFilters([ + { key: 'service', operator, value: 'api' }, + { key: 'env', operator: '=', value: 'prod' }, + ]), + ).toEqual([{ type: 'promql', condition: 'env="prod"' }]); + }, + ); + + it('escapes quotes and backslashes in values', () => { + expect(matcherFor('=', 'say "hi" \\ here')).toEqual([ + 'service="say \\"hi\\" \\\\ here"', + ]); + }); + + it('escapes newlines in values and label names', () => { + expect(matcherFor('=', 'a\nb', 'odd\nlabel')).toEqual([ + '"odd\\nlabel"="a\\nb"', + ]); + }); + + it('quotes label names outside the legacy charset', () => { + expect(matcherFor('=', 'api', 'service.name')).toEqual([ + '"service.name"="api"', + ]); + }); + + it('renders one PromQL filter per condition', () => { + expect( + adhocConditionsToPromqlFilters([ + { key: 'service', operator: '=', value: 'api' }, + { key: 'service', operator: '!=', value: 'web' }, + ]), + ).toEqual([ + { type: 'promql', condition: 'service="api"' }, + { type: 'promql', condition: 'service!="web"' }, + ]); + }); + }); + + describe('isAdhocConditionSupported', () => { + it.each<[AdhocFilterSourceType, AdhocFilterOperator, boolean]>([ + ['sql', 'LIKE', true], + ['sql', 'NOT LIKE', true], + ['promql', 'LIKE', false], + ['promql', 'NOT LIKE', false], + ['promql', '=~', true], + ['promql', '!=', true], + ])('%s supports %s: %s', (sourceType, operator, expected) => { + expect( + isAdhocConditionSupported(sourceType, { + key: 'service', + operator, + value: 'api', + }), + ).toBe(expected); + }); + }); + + describe('getSqlFilters / getPromqlFilters', () => { + const sql: Filter = { type: 'sql', condition: "env = 'prod'" }; + const lucene: Filter = { type: 'lucene', condition: 'level:error' }; + const promql: PromqlConditionFilter = { + type: 'promql', + condition: 'job="api"', + }; + + it('splits broadcast filters by query language', () => { + expect(getSqlFilters([sql, promql, lucene])).toEqual([sql, lucene]); + expect(getPromqlFilters([sql, promql, lucene])).toEqual([promql]); + }); + }); + describe('doesFilterApplyToSource', () => { const filter = ( overrides: Partial = {}, diff --git a/packages/common-utils/src/__tests__/macros.test.ts b/packages/common-utils/src/__tests__/macros.test.ts index f3d7d29f00..d239e68c60 100644 --- a/packages/common-utils/src/__tests__/macros.test.ts +++ b/packages/common-utils/src/__tests__/macros.test.ts @@ -7,6 +7,7 @@ import { substitutePromqlChartConfigTemplates, } from '@/macros'; import type { ChartVariable, MetricTable } from '@/types'; +import { DisplayType } from '@/types'; const variable = (name: string, values: string[]): ChartVariable => ({ name, @@ -677,6 +678,79 @@ describe('substitutePromqlChartConfigTemplates', () => { variables: undefined, }); }); + + it('adds its filters to every selector after expanding variables', () => { + expect( + substitutePromqlChartConfigTemplates({ + configType: 'promql' as const, + connection: 'local', + promqlExpression: [ + { expression: 'up' }, + { expression: 'rate(errors{service=~"$service"}[5m])' }, + ], + variables: [SERVICE], + filters: [ + { type: 'promql', condition: 'job="api"' }, + { type: 'promql', condition: 'env!="dev"' }, + ], + displayType: DisplayType.Line, + dateRange: PROMQL_DATE_RANGE, + }), + ).toMatchObject({ + promqlExpression: [ + { expression: 'up{job="api", env!="dev"}' }, + { + expression: + 'rate(errors{service=~"(api|web)", job="api", env!="dev"}[5m])', + }, + ], + filters: undefined, + }); + }); + + it('rejects an unparseable expression when there are filters', () => { + expect(() => + substitutePromqlChartConfigTemplates({ + ...promqlConfig('up{job=}'), + filters: [{ type: 'promql', condition: 'job="api"' }], + }), + ).toThrow(/failed to parse/); + }); + + it('adds filters only to the expressions the display queries', () => { + expect( + substitutePromqlChartConfigTemplates({ + configType: 'promql' as const, + connection: 'local', + promqlExpression: [{ expression: 'up' }, { expression: 'up{job=}' }], + filters: [{ type: 'promql', condition: 'env="prod"' }], + displayType: DisplayType.Number, + dateRange: PROMQL_DATE_RANGE, + }).promqlExpression, + ).toEqual([{ expression: 'up{env="prod"}' }, { expression: 'up{job=}' }]); + }); + + it('leaves blank rows blank when there are filters', () => { + expect( + substitutePromqlChartConfigTemplates({ + configType: 'promql' as const, + connection: 'local', + promqlExpression: [{ expression: 'up' }, { expression: ' ' }], + filters: [{ type: 'promql', condition: 'env="prod"' }], + displayType: DisplayType.Line, + dateRange: PROMQL_DATE_RANGE, + }).promqlExpression, + ).toEqual([{ expression: 'up{env="prod"}' }, { expression: ' ' }]); + }); + + it('leaves an unparseable expression alone when there are no filters', () => { + expect( + substitutePromqlChartConfigTemplates({ + ...promqlConfig('up{job=}'), + filters: [], + }).promqlExpression, + ).toBe('up{job=}'); + }); }); describe('substitutePromqlChartConfigTemplates macros', () => { diff --git a/packages/common-utils/src/__tests__/renderChartConfig.test.ts b/packages/common-utils/src/__tests__/renderChartConfig.test.ts index 8cb1559bee..6e6dd139a7 100644 --- a/packages/common-utils/src/__tests__/renderChartConfig.test.ts +++ b/packages/common-utils/src/__tests__/renderChartConfig.test.ts @@ -6,6 +6,7 @@ import { timeFilterExpr, } from '@/core/renderChartConfig'; import { convertToCategoricalChartConfig } from '@/core/utils'; +import { adhocConditionsToFilters } from '@/filters'; import { BuilderChartConfig, ChartConfigWithOptDateRange, @@ -1656,6 +1657,35 @@ describe('renderChartConfig', () => { }); }); + it('applies ad hoc filter conditions for every operator to a builder query', async () => { + const sql = parameterizedQueryToSql( + await renderChartConfig( + { + connection: 'test-connection', + from: { databaseName: 'default', tableName: 'otel_logs' }, + select: [{ aggFn: 'count', valueExpression: '' }], + where: '', + whereLanguage: 'sql', + filters: adhocConditionsToFilters([ + { key: 'ServiceName', operator: '=', value: 'api' }, + { key: 'ServiceName', operator: '!=', value: 'web' }, + { key: "LogAttributes['path']", operator: '=~', value: '^/api' }, + { key: "LogAttributes['path']", operator: '!~', value: 'health' }, + { key: 'Body', operator: 'LIKE', value: '%error%' }, + { key: 'Body', operator: 'NOT LIKE', value: "%it's%" }, + ]), + timestampValueExpression: 'Timestamp', + dateRange: [new Date('2025-01-01'), new Date('2025-01-02')], + }, + mockMetadata, + querySettings, + ), + ); + expect(sql).toContain( + "((toString(ServiceName) = 'api') AND (toString(ServiceName) != 'web') AND (match(toString(LogAttributes['path']), '^/api')) AND (NOT match(toString(LogAttributes['path']), 'health')) AND (toString(Body) LIKE '%error%') AND (toString(Body) NOT LIKE '%it''s%'))", + ); + }); + describe('SQL filter KV items direct_read optimization', () => { const stubKvItemsMetadata = () => { mockMetadata.getColumns = jest.fn().mockResolvedValue([ @@ -3523,6 +3553,35 @@ describe('renderChartConfig', () => { ); }); + it('renders ad hoc filter conditions for every operator in $__filters', async () => { + mockMetadata.getMaterializedColumnsLookupTable = jest + .fn() + .mockResolvedValue(new Map()); + const result = await renderChartConfig( + { + configType: 'sql', + sqlTemplate: 'SELECT * FROM logs WHERE $__filters', + connection: 'conn-1', + dateRange: [start, end], + source: 'source-1', + from: { databaseName: 'default', tableName: 'logs' }, + filters: adhocConditionsToFilters([ + { key: 'ServiceName', operator: '=', value: 'api' }, + { key: 'ServiceName', operator: '!=', value: 'web' }, + { key: "LogAttributes['path']", operator: '=~', value: '^/api' }, + { key: "LogAttributes['path']", operator: '!~', value: 'health' }, + { key: 'Body', operator: 'LIKE', value: '%error%' }, + { key: 'Body', operator: 'NOT LIKE', value: "%it's%" }, + ]), + }, + mockMetadata, + undefined, + ); + expect(result.sql).toBe( + "SELECT * FROM logs WHERE ((toString(ServiceName) = 'api') AND (toString(ServiceName) != 'web') AND (match(toString(LogAttributes['path']), '^/api')) AND (NOT match(toString(LogAttributes['path']), 'health')) AND (toString(Body) LIKE '%error%') AND (toString(Body) NOT LIKE '%it''s%'))", + ); + }); + it('renders sql filters raw when source has no tableName (metric source)', async () => { const result = await renderChartConfig( { diff --git a/packages/common-utils/src/__tests__/variables.test.ts b/packages/common-utils/src/__tests__/variables.test.ts index 81d60adf4c..9614f0310d 100644 --- a/packages/common-utils/src/__tests__/variables.test.ts +++ b/packages/common-utils/src/__tests__/variables.test.ts @@ -970,6 +970,10 @@ describe('substituteVariables for promql', () => { expect(matcher(['a\\b'])).toBe('up{service=~"a\\\\\\\\b"}'); }); + it('escapes a newline, which a PromQL string literal cannot contain', () => { + expect(matcher(['a\nb'])).toBe('up{service=~"a\\nb"}'); + }); + it("leaves ' alone, which is not an RE2 metacharacter and invalid to escape here", () => { expect(matcher(["o'brien"])).toBe('up{service=~"o\'brien"}'); }); diff --git a/packages/common-utils/src/core/__tests__/promqlLabelInjection.test.ts b/packages/common-utils/src/core/__tests__/promqlLabelInjection.test.ts new file mode 100644 index 0000000000..503d9bff04 --- /dev/null +++ b/packages/common-utils/src/core/__tests__/promqlLabelInjection.test.ts @@ -0,0 +1,45 @@ +import { injectPromqlLabelMatchers } from '@/core/promqlLabelInjection'; + +describe('injectPromqlLabelMatchers', () => { + const m = 'job="api"'; + + it.each([ + ['up', 'up{job="api"}'], + ['up{}', 'up{job="api"}'], + ['up{env="x"}', 'up{env="x", job="api"}'], + ['up{env="x",}', 'up{env="x",job="api"}'], + ['up{env="x", # note\n}', 'up{env="x", # note\njob="api"}'], + ['up{env="x" # a, b\n}', 'up{env="x" # a, b\n, job="api"}'], + ['up{ # note\n}', 'up{ # note\njob="api"}'], + ['up{# a, b\nenv="x"}', 'up{# a, b\nenv="x", job="api"}'], + ['up{env="x" # a, b\n,}', 'up{env="x" # a, b\n,job="api"}'], + ['up # note', 'up{job="api"} # note'], + ['up @ 123', 'up{job="api"} @ 123'], + ['up[5m] @ end()', 'up{job="api"}[5m] @ end()'], + ['rate(http_requests[5m])', 'rate(http_requests{job="api"}[5m])'], + [ + 'sum by (job) (rate(a{x="1"}[5m])) / on() b offset 1m', + 'sum by (job) (rate(a{x="1", job="api"}[5m])) / on() b{job="api"} offset 1m', + ], + ['max_over_time(up[1h:5m])', 'max_over_time(up{job="api"}[1h:5m])'], + ['{__name__="up"}', '{__name__="up", job="api"}'], + ['{"my.metric"}', '{"my.metric", job="api"}'], + [ + 'label_replace(up, "dst", "$1", "src", "{x}")', + 'label_replace(up{job="api"}, "dst", "$1", "src", "{x}")', + ], + ['1 + 2', '1 + 2'], + ])('%s', (expression, expected) => { + expect(injectPromqlLabelMatchers(expression, m)).toBe(expected); + }); + + it('returns the expression unchanged when there are no matchers', () => { + expect(injectPromqlLabelMatchers('up{', '')).toBe('up{'); + }); + + it('throws when the expression does not parse', () => { + expect(() => injectPromqlLabelMatchers('up{job=}', m)).toThrow( + /failed to parse/, + ); + }); +}); diff --git a/packages/common-utils/src/core/promqlLabelInjection.ts b/packages/common-utils/src/core/promqlLabelInjection.ts new file mode 100644 index 0000000000..97d4ace090 --- /dev/null +++ b/packages/common-utils/src/core/promqlLabelInjection.ts @@ -0,0 +1,93 @@ +import { parser } from '@prometheus-io/lezer-promql'; + +type SyntaxNode = ReturnType['topNode']; + +/** + * Whether a matcher appended before `closingBrace` needs a leading comma: the + * list has a matcher and no trailing comma. Commas aren't nodes in the tree, so + * look for one after the last matcher, skipping comments that may contain one. + */ +function needsSeparator( + expression: string, + labelMatchers: SyntaxNode, + closingBrace: number, +): boolean { + const comments: SyntaxNode[] = []; + let lastMatcher: SyntaxNode | undefined; + for (let child = labelMatchers.firstChild; child; child = child.nextSibling) { + if (child.name === 'LineComment') comments.push(child); + else lastMatcher = child; + } + if (!lastMatcher) return false; + + let position = lastMatcher.to; + let gap = ''; + for (const comment of comments) { + if (comment.from < position) continue; + gap += expression.slice(position, comment.from); + position = comment.to; + } + gap += expression.slice(position, closingBrace); + return !gap.includes(','); +} + +/** + * Add label matchers to every vector selector in a PromQL expression. + * `matchers` is a comma-separated matcher list without braces, e.g. + * `job="api", env=~"a|b"`. + * + * Throws when the expression doesn't parse, since silently skipping the + * injection would show unfiltered data as if it were filtered. + */ +export function injectPromqlLabelMatchers( + expression: string, + matchers: string, +): string { + if (!matchers.trim()) return expression; + + const tree = parser.parse(expression); + const insertions: { at: number; text: string }[] = []; + let errorAt: number | undefined; + + tree.iterate({ + enter: node => { + if (node.type.isError) { + errorAt ??= node.from; + return; + } + if (node.name !== 'VectorSelector') return; + + const labelMatchers = node.node.getChild('LabelMatchers'); + if (!labelMatchers) { + insertions.push({ at: node.to, text: `{${matchers}}` }); + return; + } + + // Insert before the closing brace so a leading quoted metric name + // (`{"my.metric", job="x"}`) stays first. + const closingBrace = labelMatchers.to - 1; + const separator = needsSeparator(expression, labelMatchers, closingBrace) + ? ', ' + : ''; + insertions.push({ at: closingBrace, text: `${separator}${matchers}` }); + }, + }); + + if (errorAt != null) { + // Lezer error nodes carry no message, only where parsing went wrong. + console.warn( + `PromQL parse error at offset ${errorAt}: ${JSON.stringify(expression.slice(errorAt, errorAt + 20))}`, + expression, + ); + throw new Error( + 'Ad hoc filters could not be applied: the PromQL expression failed to parse.', + ); + } + + return insertions + .sort((a, b) => b.at - a.at) + .reduce( + (result, { at, text }) => result.slice(0, at) + text + result.slice(at), + expression, + ); +} diff --git a/packages/common-utils/src/filters.ts b/packages/common-utils/src/filters.ts index 9fa6b71c35..123033a762 100644 --- a/packages/common-utils/src/filters.ts +++ b/packages/common-utils/src/filters.ts @@ -4,17 +4,27 @@ import { escapeSqlString, replaceJsonExpressions } from '@/core/utils'; import { parse } from '@/queryParser'; import { AdhocDashboardFilter, + AdhocFilterCondition, AdhocFilterSourceType, AdhocFilterValue, + BroadcastFilter, ChartVariable, + ConditionFilter, DASHBOARD_VARIABLE_NAME_MAX_LENGTH, DASHBOARD_VARIABLE_NAME_PATTERN_ANCHORED, DashboardFilter, DashboardFilterValue, Filter, + PROMQL_ADHOC_FILTER_OPERATORS, + PromqlConditionFilter, SourceKind, + SQL_ADHOC_FILTER_OPERATORS, } from '@/types'; -import { getVariableReferences, substituteVariables } from '@/variables'; +import { + escapePromqlStringValue, + getVariableReferences, + substituteVariables, +} from '@/variables'; export type FilterState = { [key: string]: { @@ -59,7 +69,7 @@ export const filtersToQuery = ( /** Map of DateTime/Date column name → its ClickHouse type. */ dateTimeColumns?: ReadonlyMap; } = {}, -): Filter[] => { +): ConditionFilter[] => { return Object.entries(filters) .filter( ([_, values]) => @@ -68,7 +78,7 @@ export const filtersToQuery = ( values.range != null, ) .flatMap(([key, values]) => { - const conditions: Filter[] = []; + const conditions: ConditionFilter[] = []; const actualKey = stringifyKeys ? `toString(${key})` : key; // DateTime/DateTime64 columns can't be compared against a bare string @@ -943,6 +953,131 @@ export function isAdhocFilterValue( return value.type === 'adhoc'; } +const renderAdhocConditionSql = ({ + key, + operator, + value, +}: AdhocFilterCondition): string => { + // Stringified like broadcast filter selections, so Map/JSON/Dynamic values + // compare as strings. + const target = `toString(${key})`; + const literal = `'${escapeSqlString(value)}'`; + switch (operator) { + case '=': + case '!=': + case 'LIKE': + case 'NOT LIKE': + return `${target} ${operator} ${literal}`; + case '=~': + return `match(${target}, ${literal})`; + case '!~': + return `NOT match(${target}, ${literal})`; + default: + operator satisfies never; + throw new Error(`Unknown ad hoc filter operator '${operator}'`); + } +}; + +/** + * Render the conditions of a SQL ad hoc filter as filters, one per condition. + * Keys are SQL expressions and are used verbatim. + */ +export function adhocConditionsToFilters( + conditions: AdhocFilterCondition[], +): ConditionFilter[] { + return conditions.map( + (condition): ConditionFilter => ({ + type: 'sql', + condition: renderAdhocConditionSql(condition), + }), + ); +} + +const LEGACY_LABEL_NAME = /^[a-zA-Z_][a-zA-Z0-9_]*$/; + +const renderAdhocMatcherPromql = ({ + key, + operator, + value, +}: AdhocFilterCondition): string => { + // Label names outside the legacy charset must be quoted (Prometheus 3). + const label = LEGACY_LABEL_NAME.test(key) + ? key + : `"${escapePromqlStringValue(key)}"`; + switch (operator) { + case '=': + case '!=': + case '=~': + case '!~': + return `${label}${operator}"${escapePromqlStringValue(value)}"`; + case 'LIKE': + case 'NOT LIKE': + throw new Error( + `Ad hoc filter operator '${operator}' is not supported for PromQL`, + ); + default: + operator satisfies never; + throw new Error(`Unknown ad hoc filter operator '${operator}'`); + } +}; + +/** + * Whether an ad hoc filter of `sourceType` can apply `condition`. A filter + * switched from SQL to PromQL keeps its conditions, and PromQL has no LIKE. + */ +export function isAdhocConditionSupported( + sourceType: AdhocFilterSourceType, + { operator }: AdhocFilterCondition, +): boolean { + switch (sourceType) { + case 'sql': + return SQL_ADHOC_FILTER_OPERATORS.includes(operator); + case 'promql': + return PROMQL_ADHOC_FILTER_OPERATORS.includes(operator); + default: + sourceType satisfies never; + throw new Error(`Unknown ad hoc filter source type '${sourceType}'`); + } +} + +/** + * Render the conditions of a PromQL ad hoc filter as label matchers, one per + * condition. Conditions PromQL can't express are skipped; callers report them + * using `isAdhocConditionSupported`. + */ +export function adhocConditionsToPromqlFilters( + conditions: AdhocFilterCondition[], +): PromqlConditionFilter[] { + return conditions + .filter(condition => isAdhocConditionSupported('promql', condition)) + .map( + (condition): PromqlConditionFilter => ({ + type: 'promql', + condition: renderAdhocMatcherPromql(condition), + }), + ); +} + +function isPromqlFilter( + filter: BroadcastFilter, +): filter is PromqlConditionFilter { + return filter.type === 'promql'; +} + +/** The broadcast filters a SQL, Lucene or builder query can apply. */ +export function getSqlFilters(filters: BroadcastFilter[]): ConditionFilter[] { + return filters.filter( + (filter): filter is ConditionFilter => !isPromqlFilter(filter), + ); +} + +/** The broadcast filters a PromQL query can apply. */ +export function getPromqlFilters( + filters: BroadcastFilter[], +): PromqlConditionFilter[] { + return filters.filter(isPromqlFilter); +} + /** The SQL expression associated with the filter, if any. */ export function getFilterExpression( filter: DashboardFilter, diff --git a/packages/common-utils/src/macros.ts b/packages/common-utils/src/macros.ts index ada123557d..5c98c342e1 100644 --- a/packages/common-utils/src/macros.ts +++ b/packages/common-utils/src/macros.ts @@ -1,17 +1,23 @@ import { getPromqlMacroInputs, + getPromqlSeries, + getQueriedPromqlSeries, PROMQL_MACRO_NAMES, PROMQL_MACROS, PromqlMacroName, } from './core/promql'; +import { injectPromqlLabelMatchers } from './core/promqlLabelInjection'; import { splitAndTrimWithBracket } from './core/utils'; import { MacroExpansionError, MalformedMacroArgsError } from './macroErrors'; import { RawSqlQueryParam, renderQueryParam } from './rawSqlParams'; import { ChartVariable, + DisplayType, MetricsDataType, MetricsDataTypeSchema, + PromqlConditionFilter, PromqlExpressionList, + PromqlSeries, RawSqlChartConfig, } from './types'; import { @@ -512,18 +518,22 @@ export function replaceMacros( /** * Expand the macros (`$__interval`, `$__range`, `$__rate_interval`) and variable - * references in a PromQL config's expression(s), returning it with `variables` - * consumed. References to names that aren't variables are left as written, so - * `$1` in a `label_replace` replacement reaches Prometheus unchanged. + * references in a PromQL config's expression(s), then add its `filters` to + * every selector of the expressions its display queries, returning it with + * `variables` and `filters` consumed. + * References to names that aren't variables are left as written, so `$1` in a + * `label_replace` replacement reaches Prometheus unchanged. * * Macros and variables expand in one pass, so a selected value that looks like - * `$__range` stays inert. Dropping `variables` from the result ensures variables - * can't be substituted twice. + * `$__range` stays inert. Dropping `variables` and `filters` from the result + * ensures neither can be applied twice. */ export function substitutePromqlChartConfigTemplates< T extends { promqlExpression: PromqlExpressionList; variables?: ChartVariable[]; + filters?: PromqlConditionFilter[]; + displayType?: DisplayType; granularity?: string; dateRange: [Date, Date]; minGranularitySeconds?: number; @@ -532,6 +542,7 @@ export function substitutePromqlChartConfigTemplates< const { promqlExpression, variables = [], + filters = [], granularity, dateRange, minGranularitySeconds, @@ -561,8 +572,20 @@ export function substitutePromqlChartConfigTemplates< ]), ); - const substitute = (expression: string) => - expandTemplate(expression, { + const matchers = filters.map(filter => filter.condition).join(', '); + + // Matchers go only into expressions the display queries: the editor saves + // every row, and an unfinished one that's never queried mustn't fail to parse. + const allSeries = getPromqlSeries(config); + const queriedSeries = new Set( + getQueriedPromqlSeries({ + promqlExpression: allSeries, + displayType: config.displayType, + }), + ); + + const substitute = (series: PromqlSeries) => { + const expanded = expandTemplate(series.expression, { // `$__filter` / `$__conditionalAll` are scanned only so a reference in // their arguments isn't expanded; they have no meaning in PromQL. macroNames: [...VARIABLE_MACRO_NAMES, ...PROMQL_MACRO_NAMES], @@ -570,16 +593,21 @@ export function substitutePromqlChartConfigTemplates< macrosByName.get(token.name)?.replace(token.args) ?? token.raw, expandReference: token => expandVariableToken(token, variableContext), }); + return queriedSeries.has(series) + ? injectPromqlLabelMatchers(expanded, matchers) + : expanded; + }; return { ...config, promqlExpression: typeof promqlExpression === 'string' - ? substitute(promqlExpression) - : promqlExpression.map(series => ({ + ? substitute(allSeries[0]) + : allSeries.map(series => ({ ...series, - expression: substitute(series.expression), + expression: substitute(series), })), variables: undefined, + filters: undefined, }; } diff --git a/packages/common-utils/src/types.ts b/packages/common-utils/src/types.ts index 013869fa45..c0e963be44 100644 --- a/packages/common-utils/src/types.ts +++ b/packages/common-utils/src/types.ts @@ -1231,16 +1231,35 @@ export const SqlAstFilterSchema = z.object({ export type SqlAstFilter = z.infer; +export const ConditionFilterSchema = z.object({ + type: z.enum(['lucene', 'sql']), + condition: z.string(), +}); + +export type ConditionFilter = z.infer; + export const FilterSchema = z.union([ - z.object({ - type: z.enum(['lucene', 'sql']), - condition: z.string(), - }), + ConditionFilterSchema, SqlAstFilterSchema, ]); export type Filter = z.infer; +/** + * A PromQL label matcher (e.g. `job="api"`) added to every selector of a + * PromQL tile. Kept out of `FilterSchema` because saved searches, alerts and + * the APIs only take SQL and Lucene conditions. + */ +export const PromqlConditionFilterSchema = z.object({ + type: z.literal('promql'), + condition: z.string(), +}); + +export type PromqlConditionFilter = z.infer; + +/** A dashboard filter condition broadcast to tiles, in the tile's query language. */ +export type BroadcastFilter = ConditionFilter | PromqlConditionFilter; + export const VariableFilterValueSchema = z.object({ type: z.literal('variable'), name: z.string().min(1).max(1024), @@ -1249,9 +1268,15 @@ export const VariableFilterValueSchema = z.object({ export type VariableFilterValue = z.infer; +/** The kind of sources an ad hoc filter reads its keys from. */ +export const AdhocFilterSourceTypeSchema = z.enum(['sql', 'promql']); + +export type AdhocFilterSourceType = z.infer; + /** * Operators an ad hoc filter condition can use, spelled as in PromQL so one set - * serves both source types. `LIKE` / `NOT LIKE` are offered for SQL sources only. + * serves both source types. See `SQL_ADHOC_FILTER_OPERATORS` and + * `PROMQL_ADHOC_FILTER_OPERATORS` for which ones each source type supports. */ export const AdhocFilterOperatorSchema = z.enum([ '=', @@ -1264,6 +1289,16 @@ export const AdhocFilterOperatorSchema = z.enum([ export type AdhocFilterOperator = z.infer; +export const SQL_ADHOC_FILTER_OPERATORS: readonly AdhocFilterOperator[] = + AdhocFilterOperatorSchema.options; + +export const PROMQL_ADHOC_FILTER_OPERATORS: readonly AdhocFilterOperator[] = [ + '=', + '!=', + '=~', + '!~', +]; + /** One `key operator value` condition picked in an ad hoc filter. */ export const AdhocFilterConditionSchema = z.object({ /** A SQL expression (column or map access) or a PromQL label name. */ @@ -1928,7 +1963,7 @@ const PromqlChartConfigSchema = PromqlBaseChartConfigSchema.extend({ // Copied from the source at query time and never persisted, so it cannot go // stale if the source moves to another connection. connection: z.string(), - filters: z.array(FilterSchema).optional(), + filters: z.array(PromqlConditionFilterSchema).optional(), from: z .object({ databaseName: z.string(), tableName: z.string() }) .optional(), @@ -2259,11 +2294,6 @@ export const PromqlLabelDashboardFilterSchema = isVariableEnabled: z.literal(true), }); -/** The kind of sources an ad hoc filter reads its keys from. */ -export const AdhocFilterSourceTypeSchema = z.enum(['sql', 'promql']); - -export type AdhocFilterSourceType = z.infer; - /** * A filter whose conditions (`key operator value`) are picked freely from the * keys of its sources: columns and map keys for SQL sources, labels for PromQL diff --git a/packages/common-utils/src/variables.ts b/packages/common-utils/src/variables.ts index 0c6a54d28c..9abe461e7c 100644 --- a/packages/common-utils/src/variables.ts +++ b/packages/common-utils/src/variables.ts @@ -61,9 +61,12 @@ const escapeRegexValue = (value: string) => const escapeLuceneValue = (value: string) => value.replace(/\\/g, '\\\\').replace(/"/g, '\\"'); -/** Escape `\` and `"` so a value survives inside a double-quoted PromQL string. */ -const escapePromqlStringValue = (value: string) => - value.replace(/\\/g, '\\\\').replace(/"/g, '\\"'); +/** + * Escape `\`, `"` and newlines so a value survives inside a double-quoted + * PromQL string, which can't span lines. + */ +export const escapePromqlStringValue = (value: string) => + value.replace(/\\/g, '\\\\').replace(/"/g, '\\"').replace(/\n/g, '\\n'); /** * Backslash-escape every ASCII punctuation character (CommonMark renders each diff --git a/yarn.lock b/yarn.lock index fc99d7bb95..dd87de2efb 100644 --- a/yarn.lock +++ b/yarn.lock @@ -4977,6 +4977,9 @@ __metadata: "@clickhouse/client-common": "npm:1.23.0-head.fae5998.1" "@clickhouse/client-web": "npm:1.23.0-head.fae5998.1" "@hyperdx/lucene": "npm:^3.1.1" + "@lezer/highlight": "npm:^1.1.2" + "@lezer/lr": "npm:^1.2.3" + "@prometheus-io/lezer-promql": "npm:^0.311.3" "@stryker-mutator/core": "npm:^9.6.1" "@stryker-mutator/jest-runner": "npm:^9.6.1" "@stryker-mutator/typescript-checker": "npm:^9.6.1" @@ -6395,19 +6398,19 @@ __metadata: languageName: node linkType: hard -"@lezer/common@npm:^1.0.0, @lezer/common@npm:^1.1.0, @lezer/common@npm:^1.2.0": - version: 1.2.1 - resolution: "@lezer/common@npm:1.2.1" - checksum: 10c0/af61436dc026f8deebaded13d8e1beea2ae307cbbfb270116cdedadb8208f0674da9c3b5963128a2b1cd4072b4e90bc8128133f4feaf31b6e801e4568f1a15a6 +"@lezer/common@npm:^1.0.0, @lezer/common@npm:^1.1.0, @lezer/common@npm:^1.2.0, @lezer/common@npm:^1.3.0": + version: 1.5.3 + resolution: "@lezer/common@npm:1.5.3" + checksum: 10c0/f06d64cd72ad95295f6bc65187188081e61fb655c48923c9c283b514e3183450003795a1c0b58dc0844da5075c748068d879f90d7eaedeea48c99d73f9b6e214 languageName: node linkType: hard -"@lezer/highlight@npm:^1.0.0": - version: 1.2.0 - resolution: "@lezer/highlight@npm:1.2.0" +"@lezer/highlight@npm:^1.0.0, @lezer/highlight@npm:^1.1.2": + version: 1.2.5 + resolution: "@lezer/highlight@npm:1.2.5" dependencies: - "@lezer/common": "npm:^1.0.0" - checksum: 10c0/d4312f95b78e4b6f10833b1cf99601c6381c22b755bbf60fd61d6fe9b4cf7780650e2e2dadf75beb8d94824dcb5ec81da5cfc9ca54122688a482e488103105aa + "@lezer/common": "npm:^1.3.0" + checksum: 10c0/cb40f6efd8ba947b4a391368114ae394e36cbb2b7db07a7629278f9feed953d909868296f4acb936c7b70e4e7032458c5fcd6a2e836012ae6c02790b2de574fa languageName: node linkType: hard @@ -6422,12 +6425,12 @@ __metadata: languageName: node linkType: hard -"@lezer/lr@npm:^1.0.0": - version: 1.4.0 - resolution: "@lezer/lr@npm:1.4.0" +"@lezer/lr@npm:^1.0.0, @lezer/lr@npm:^1.2.3": + version: 1.4.11 + resolution: "@lezer/lr@npm:1.4.11" dependencies: "@lezer/common": "npm:^1.0.0" - checksum: 10c0/1e3af297cc142bb6676cb3c4e1bd310da2e93b53273cf745dc85d839a08e1d3cbbd67e0fc0322a480cf25ee215fefe967c53bc2af3ddf5d9b1bf267081dfa164 + checksum: 10c0/4dc3daacdebc68d5b21af278bdcbac17d77a25309ce15cb3fc0f4ebe0111b670cd6784d6951e8660850edc11dc1a8f4614075116b64be1d2396934690d03d3f9 languageName: node linkType: hard @@ -8742,7 +8745,7 @@ __metadata: languageName: node linkType: hard -"@prometheus-io/lezer-promql@npm:0.311.3": +"@prometheus-io/lezer-promql@npm:0.311.3, @prometheus-io/lezer-promql@npm:^0.311.3": version: 0.311.3 resolution: "@prometheus-io/lezer-promql@npm:0.311.3" peerDependencies: