Skip to content

Commit 8f11151

Browse files
Loori-RLoori-R
committed
feat: add log level settings
1 parent f65ac33 commit 8f11151

13 files changed

+426
-90
lines changed

‎CHANGELOG.md

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,8 @@
22

33
## tip
44

5+
* FEATURE: add support for configuring log level using custom rules. See [this issue](https://github.com/VictoriaMetrics/victorialogs-datasource/issues/294).
6+
57
## v0.16.3
68

79
* FEATURE: enabled [PDC](https://grafana.com/docs/grafana-cloud/connect-externally-hosted/private-data-source-connect/) support. See [VictoriaMetrics#8800](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/8800) for details.

‎pkg/plugin/query.go

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -53,7 +53,7 @@ type Query struct {
5353
IntervalMs int64 `json:"intervalMs"`
5454
MaxLines int `json:"maxLines"`
5555
Step string `json:"step"`
56-
Field string `json:"field"`
56+
Fields []string `json:"field"`
5757
QueryType QueryType `json:"queryType"`
5858
url *url.URL
5959
ForAlerting bool `json:"-"`
@@ -256,7 +256,9 @@ func (q *Query) histQueryURL(queryParams url.Values, minInterval time.Duration)
256256
values.Set("start", strconv.FormatInt(q.TimeRange.From.Unix(), 10))
257257
values.Set("end", strconv.FormatInt(q.TimeRange.To.Unix(), 10))
258258
values.Set("step", step)
259-
values.Set("field", q.Field)
259+
for _, f := range q.Fields {
260+
values.Add("field", f)
261+
}
260262

261263
q.url.RawQuery = values.Encode()
262264
return q.url.String()

‎pkg/plugin/query_test.go

Lines changed: 10 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -296,7 +296,7 @@ func TestQuery_getQueryURL(t *testing.T) {
296296
rawURL: "http://127.0.0.1:9429",
297297
queryParams: "",
298298
},
299-
want: "http://127.0.0.1:9429/select/logsql/hits?end=1609462800&field=&query=&start=1609459200&step=15s",
299+
want: "http://127.0.0.1:9429/select/logsql/hits?end=1609462800&query=&start=1609459200&step=15s",
300300
wantErr: false,
301301
},
302302
{
@@ -315,7 +315,7 @@ func TestQuery_getQueryURL(t *testing.T) {
315315
rawURL: "http://127.0.0.1:9429",
316316
queryParams: "",
317317
},
318-
want: "http://127.0.0.1:9429/select/logsql/hits?end=1609462800&field=&query=&start=1609459200&step=15s",
318+
want: "http://127.0.0.1:9429/select/logsql/hits?end=1609462800&query=&start=1609459200&step=15s",
319319
wantErr: false,
320320
},
321321
{
@@ -334,7 +334,7 @@ func TestQuery_getQueryURL(t *testing.T) {
334334
rawURL: "http://127.0.0.1:9429",
335335
queryParams: "",
336336
},
337-
want: "http://127.0.0.1:9429/select/logsql/hits?end=1609462800&field=&query=&start=1609459200&step=15s",
337+
want: "http://127.0.0.1:9429/select/logsql/hits?end=1609462800&query=&start=1609459200&step=15s",
338338
wantErr: false,
339339
},
340340
{
@@ -353,7 +353,7 @@ func TestQuery_getQueryURL(t *testing.T) {
353353
rawURL: "http://127.0.0.1:9429",
354354
queryParams: "",
355355
},
356-
want: "http://127.0.0.1:9429/select/logsql/hits?end=1609462800&field=&query=_time%3A1s&start=1609459200&step=15s",
356+
want: "http://127.0.0.1:9429/select/logsql/hits?end=1609462800&query=_time%3A1s&start=1609459200&step=15s",
357357
wantErr: false,
358358
},
359359
{
@@ -372,7 +372,7 @@ func TestQuery_getQueryURL(t *testing.T) {
372372
rawURL: "http://127.0.0.1:9429",
373373
queryParams: "",
374374
},
375-
want: "http://127.0.0.1:9429/select/logsql/hits?end=1609462800&field=&query=_time%3A1s+%7C+stats+by%28type%29+count%28%29&start=1609459200&step=15s",
375+
want: "http://127.0.0.1:9429/select/logsql/hits?end=1609462800&query=_time%3A1s+%7C+stats+by%28type%29+count%28%29&start=1609459200&step=15s",
376376
wantErr: false,
377377
},
378378
{
@@ -391,7 +391,7 @@ func TestQuery_getQueryURL(t *testing.T) {
391391
rawURL: "http://127.0.0.1:9429",
392392
queryParams: "",
393393
},
394-
want: "http://127.0.0.1:9429/select/logsql/hits?end=1609462800&field=&query=_time%3A1s+%7C+stats+by%28type%29+count%28%29&start=1609459200&step=15s",
394+
want: "http://127.0.0.1:9429/select/logsql/hits?end=1609462800&query=_time%3A1s+%7C+stats+by%28type%29+count%28%29&start=1609459200&step=15s",
395395
wantErr: false,
396396
},
397397
{
@@ -410,7 +410,7 @@ func TestQuery_getQueryURL(t *testing.T) {
410410
rawURL: "http://127.0.0.1:9429",
411411
queryParams: "",
412412
},
413-
want: "http://127.0.0.1:9429/select/logsql/hits?end=1609462800&field=&query=_time%3A1s+and+syslog&start=1609459200&step=15s",
413+
want: "http://127.0.0.1:9429/select/logsql/hits?end=1609462800&query=_time%3A1s+and+syslog&start=1609459200&step=15s",
414414
wantErr: false,
415415
},
416416
{
@@ -429,7 +429,7 @@ func TestQuery_getQueryURL(t *testing.T) {
429429
rawURL: "http://127.0.0.1:9429",
430430
queryParams: "",
431431
},
432-
want: "http://127.0.0.1:9429/select/logsql/hits?end=1609462800&field=&query=_time%3A1s+and+syslog+%7C+stats+by%28type%29+count%28%29&start=1609459200&step=15s",
432+
want: "http://127.0.0.1:9429/select/logsql/hits?end=1609462800&query=_time%3A1s+and+syslog+%7C+stats+by%28type%29+count%28%29&start=1609459200&step=15s",
433433
wantErr: false,
434434
},
435435
{
@@ -448,7 +448,7 @@ func TestQuery_getQueryURL(t *testing.T) {
448448
rawURL: "http://127.0.0.1:9429",
449449
queryParams: "",
450450
},
451-
want: "http://127.0.0.1:9429/select/logsql/hits?end=1609462800&field=&query=_time%3A1s+and+syslog+%7C+stats+by%28type%29+count%28%29&start=1609459200&step=15s",
451+
want: "http://127.0.0.1:9429/select/logsql/hits?end=1609462800&query=_time%3A1s+and+syslog+%7C+stats+by%28type%29+count%28%29&start=1609459200&step=15s",
452452
wantErr: false,
453453
},
454454
{
@@ -467,7 +467,7 @@ func TestQuery_getQueryURL(t *testing.T) {
467467
rawURL: "http://127.0.0.1:9429",
468468
queryParams: "",
469469
},
470-
want: "http://127.0.0.1:9429/select/logsql/hits?end=1609462800&field=&query=%2A+and+syslog+%7C+stats+by%28type%29+count%28%29&start=1609459200&step=15s",
470+
want: "http://127.0.0.1:9429/select/logsql/hits?end=1609462800&query=%2A+and+syslog+%7C+stats+by%28type%29+count%28%29&start=1609459200&step=15s",
471471
wantErr: false,
472472
},
473473
}

‎src/README.md

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -81,6 +81,31 @@ And apply `Transformations` by labels:
8181

8282
<img alt="Transformations" src="https://github.com/VictoriaMetrics/victorialogs-datasource/blob/main/src/img/panel_table_transformation.png?raw=true">
8383

84+
### Log level rules
85+
86+
The **Log level rules** section in the datasource configuration allows you to assign log levels based on custom field conditions. This helps classify logs dynamically (e.g., as `error`, `info`, `debug`, etc.) using rules you define.
87+
88+
#### How to use
89+
90+
1. Open the datasource settings.
91+
92+
2. Scroll to the **Log level rules** section.
93+
94+
3. Click **"Add rule"** to define a new rule.
95+
96+
4. For each rule, configure the following:
97+
98+
* **Enable switch** – enable or disable the rule.
99+
* **Field name** – the log field the condition will evaluate.
100+
* **Operator** – choose from: `Equals`, `Not equal`, `Matches regex`, `Less than`, `Greater than`
101+
* **Value** – the value to compare the field against.
102+
* **Log level** – level to assign if the condition matches: `critical`, `warning`, `error`, `info`, `debug`, `trace`, `unknown`
103+
* **Delete button** – remove the rule.
104+
105+
5. After adding or editing rules, click **"Save & test"** to apply the changes.
106+
107+
**Rule priority**: If multiple rules match a log entry, the **first matching rule** (top to bottom) takes precedence.
108+
84109
## License
85110

86111
This project is licensed under

‎src/backendResultTransformer.ts

Lines changed: 71 additions & 51 deletions
Original file line numberDiff line numberDiff line change
@@ -3,17 +3,28 @@ import {
33
DataQueryError,
44
DataQueryRequest,
55
DataQueryResponse,
6+
Field,
67
FieldType,
7-
isDataFrame,
8+
LogLevel,
89
QueryResultMeta,
10+
isDataFrame,
911
} from '@grafana/data';
1012

13+
import { LogLevelRule } from "./configuration/LogLevelRules/types";
14+
import { resolveLogLevel } from "./configuration/LogLevelRules/utils";
1115
import { getDerivedFields } from './getDerivedFields';
1216
import { makeTableFrames } from './makeTableFrames';
1317
import { getHighlighterExpressionsFromQuery } from './queryUtils';
1418
import { dataFrameHasError } from './responseUtils';
1519
import { DerivedFieldConfig, Query, QueryType } from './types';
1620

21+
const ANNOTATIONS_REF_ID = 'Anno';
22+
23+
enum FrameField {
24+
Labels = 'labels',
25+
Level = 'level'
26+
}
27+
1728
function isMetricFrame(frame: DataFrame): boolean {
1829
return frame.fields.every((field) => field.type === FieldType.time || field.type === FieldType.number);
1930
}
@@ -29,22 +40,67 @@ function setFrameMeta(frame: DataFrame, meta: QueryResultMeta): DataFrame {
2940
};
3041
}
3142

43+
function addLevelField(frame: DataFrame, rules: LogLevelRule[]): DataFrame {
44+
const rows = frame.length ?? frame.fields[0]?.values.length ?? 0;
45+
const labelsField = frame.fields.find(f => f.name === FrameField.Labels);
46+
47+
const levelValues: LogLevel[] = Array.from({ length: rows }, (_, idx) => {
48+
const labels = (labelsField?.values[idx] ?? {}) as Record<string, any>;
49+
return resolveLogLevel(labels, rules);
50+
});
51+
52+
const levelField: Field<LogLevel> = {
53+
name: FrameField.Level,
54+
type: FieldType.string,
55+
config: {},
56+
values: levelValues,
57+
};
58+
59+
return { ...frame, fields: [...frame.fields, levelField] };
60+
}
61+
62+
function transformDashboardLabelField(field: Field): Field {
63+
if (field.name !== FrameField.Labels) {
64+
return field;
65+
}
66+
67+
return {
68+
...field,
69+
values: field.values.map((value) => {
70+
return Object.entries(value).map(([key, val]) => {
71+
return `${key}: ${JSON.stringify(val)}`;
72+
});
73+
}),
74+
};
75+
}
76+
77+
function getStreamFields(fields: Field[], transformLabels: boolean): Field[] {
78+
if (!transformLabels) {
79+
return fields;
80+
}
81+
82+
return fields.map(transformDashboardLabelField);
83+
}
84+
3285
function processStreamsFrames(
3386
frames: DataFrame[],
3487
queryMap: Map<string, Query>,
3588
derivedFieldConfigs: DerivedFieldConfig[],
89+
logLevelRules: LogLevelRule[]
3690
): DataFrame[] {
3791
return frames.map((frame) => {
3892
const query = frame.refId !== undefined ? queryMap.get(frame.refId) : undefined;
39-
if (query?.refId === "Anno") {return processDashboardStreamFrame(frame, query, derivedFieldConfigs);}
40-
return processStreamFrame(frame, query, derivedFieldConfigs);
93+
const isAnnotations = query?.refId === ANNOTATIONS_REF_ID
94+
return processStreamFrame(frame, query, derivedFieldConfigs, logLevelRules, isAnnotations);
4195
});
4296
}
4397

4498
function processStreamFrame(
4599
frame: DataFrame,
46100
query: Query | undefined,
47-
derivedFieldConfigs: DerivedFieldConfig[]
101+
derivedFieldConfigs: DerivedFieldConfig[],
102+
logLevelRules: LogLevelRule[],
103+
transformLabels = false
48104
): DataFrame {
49105
const custom: Record<string, string> = {
50106
...frame.meta?.custom, // keep the original meta.custom
@@ -61,55 +117,18 @@ function processStreamFrame(
61117
custom,
62118
};
63119

64-
const newFrame = setFrameMeta(frame, meta);
65-
const derivedFields = getDerivedFields(newFrame, derivedFieldConfigs);
66-
return {
67-
...newFrame,
68-
fields: [...newFrame.fields, ...derivedFields],
69-
};
70-
}
120+
const frameWithMeta = setFrameMeta(frame, meta);
121+
const frameWithLevel = addLevelField(frameWithMeta, logLevelRules);
71122

72-
function processDashboardStreamFrame(
73-
frame: DataFrame,
74-
query: Query | undefined,
75-
derivedFieldConfigs: DerivedFieldConfig[]
76-
): DataFrame {
77-
const custom: Record<string, string> = {
78-
...frame.meta?.custom, // keep the original meta.custom
79-
};
80-
81-
if (dataFrameHasError(frame)) {
82-
custom.error = 'Error when parsing some of the logs';
83-
}
84-
85-
const meta: QueryResultMeta = {
86-
preferredVisualisationType: 'logs',
87-
limit: query?.maxLines,
88-
searchWords: query !== undefined ? getHighlighterExpressionsFromQuery(query.expr) : undefined,
89-
custom,
90-
};
91-
92-
const newFrame = setFrameMeta(frame, meta);
93-
const derivedFields = getDerivedFields(newFrame, derivedFieldConfigs);
123+
const derivedFields = getDerivedFields(frameWithLevel, derivedFieldConfigs);
124+
const baseFields = getStreamFields(frameWithLevel.fields, transformLabels)
94125

95126
return {
96-
...newFrame,
127+
...frameWithLevel,
97128
fields: [
98-
...newFrame.fields.map((field) => {
99-
if (field.name === 'labels') {
100-
return {
101-
...field,
102-
values: field.values.map((value) => {
103-
return Object.entries(value).map(([key, value]) => {
104-
return `${key}: ${JSON.stringify(value)}`;
105-
});
106-
}),
107-
};
108-
}
109-
return field;
110-
}),
111-
...derivedFields,
112-
],
129+
...baseFields,
130+
...derivedFields
131+
]
113132
};
114133
}
115134

@@ -180,7 +199,8 @@ function improveError(error: DataQueryError | undefined, queryMap: Map<string, Q
180199
export function transformBackendResult(
181200
response: DataQueryResponse,
182201
request: DataQueryRequest,
183-
derivedFieldConfigs: DerivedFieldConfig[]
202+
derivedFieldConfigs: DerivedFieldConfig[],
203+
logLevelRules: LogLevelRule[],
184204
): DataQueryResponse {
185205
const { data, errors, ...rest } = response;
186206
const queries = request.targets;
@@ -208,7 +228,7 @@ export function transformBackendResult(
208228
data: [
209229
...processMetricRangeFrames(metricRangeFrames),
210230
...processMetricInstantFrames(metricInstantFrames),
211-
...processStreamsFrames(streamsFrames, queryMap, derivedFieldConfigs),
231+
...processStreamsFrames(streamsFrames, queryMap, derivedFieldConfigs, logLevelRules),
212232
],
213233
};
214234
}

‎src/configuration/ConfigEditor.tsx

Lines changed: 16 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1,16 +1,17 @@
11
import React from 'react';
22
import { gte } from 'semver';
33

4-
import { DataSourcePluginOptionsEditorProps, DataSourceSettings } from '@grafana/data';
4+
import { DataSourcePluginOptionsEditorProps, DataSourceSettings, FeatureToggles } from '@grafana/data';
55
import { config } from '@grafana/runtime';
6-
import { InlineSwitch, InlineField, DataSourceHttpSettings, SecureSocksProxySettings } from "@grafana/ui";
6+
import { InlineSwitch, InlineField, DataSourceHttpSettings, Space } from "@grafana/ui";
77

88
import { Options } from '../types';
99

1010
import { AlertingSettings } from './AlertingSettings';
1111
import { DerivedFields } from "./DerivedFields";
1212
import { HelpfulLinks } from "./HelpfulLinks";
1313
import { LimitsSettings } from "./LimitSettings";
14+
import { LogLevelRulesEditor } from "./LogLevelRules/LogLevelRulesEditor";
1415
import { LogsSettings } from './LogsSettings';
1516
import { QuerySettings } from './QuerySettings';
1617

@@ -41,18 +42,24 @@ const ConfigEditor = (props: Props) => {
4142
sigV4AuthToggleEnabled={config.sigV4AuthEnabled}
4243
/>
4344
<AlertingSettings options={options} onOptionsChange={onOptionsChange}/>
44-
<QuerySettings
45-
maxLines={options.jsonData.maxLines || ''}
46-
onMaxLinedChange={(value) => onOptionsChange(setMaxLines(options, value))}
47-
/>
45+
46+
<Space v={5} />
47+
48+
<LimitsSettings {...props}>
49+
<QuerySettings
50+
maxLines={options.jsonData.maxLines || ''}
51+
onMaxLinedChange={(value) => onOptionsChange(setMaxLines(options, value))}
52+
/>
53+
</LimitsSettings>
54+
55+
<LogsSettings {...props}/>
56+
4857
<DerivedFields
4958
fields={options.jsonData.derivedFields}
5059
onChange={(value) => onOptionsChange(setDerivedFields(options, value))}
5160
/>
5261

53-
<LogsSettings {...props}/>
54-
55-
<LimitsSettings {...props}/>
62+
<LogLevelRulesEditor {...props}/>
5663

5764
{config.featureToggles['secureSocksDSProxyEnabled' as keyof FeatureToggles] && gte(config.buildInfo.version, '10.0.0') && (
5865
<>

0 commit comments

Comments
 (0)