Skip to content

add support for configuring log level using custom rules #301

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 4 commits into from
May 28, 2025
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,8 @@

## tip

* BUGFIX: fix an issue when Grafana decides that the response is not a wide series and shows the error "input data must be a wide series but got type not (input refid)" . See [this issue](https://github.com/VictoriaMetrics/victorialogs-datasource/issues/302).
* FEATURE: add support for configuring log level using custom rules. See [this issue](https://github.com/VictoriaMetrics/victorialogs-datasource/issues/294).
* BUGFIX: fix an issue when Grafana decides that the response is not a wide series and shows the error "input data must be a wide series but got type not (input refid)". See [this issue](https://github.com/VictoriaMetrics/victorialogs-datasource/issues/302).

## v0.16.3

Expand Down
6 changes: 4 additions & 2 deletions pkg/plugin/query.go
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,7 @@ type Query struct {
IntervalMs int64 `json:"intervalMs"`
MaxLines int `json:"maxLines"`
Step string `json:"step"`
Field string `json:"field"`
Fields []string `json:"fields"`
QueryType QueryType `json:"queryType"`
url *url.URL
ForAlerting bool `json:"-"`
Expand Down Expand Up @@ -256,7 +256,9 @@ func (q *Query) histQueryURL(queryParams url.Values, minInterval time.Duration)
values.Set("start", strconv.FormatInt(q.TimeRange.From.Unix(), 10))
values.Set("end", strconv.FormatInt(q.TimeRange.To.Unix(), 10))
values.Set("step", step)
values.Set("field", q.Field)
for _, f := range q.Fields {
values.Add("field", f)
}

q.url.RawQuery = values.Encode()
return q.url.String()
Expand Down
20 changes: 10 additions & 10 deletions pkg/plugin/query_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -296,7 +296,7 @@ func TestQuery_getQueryURL(t *testing.T) {
rawURL: "http://127.0.0.1:9429",
queryParams: "",
},
want: "http://127.0.0.1:9429/select/logsql/hits?end=1609462800&field=&query=&start=1609459200&step=15s",
want: "http://127.0.0.1:9429/select/logsql/hits?end=1609462800&query=&start=1609459200&step=15s",
wantErr: false,
},
{
Expand All @@ -315,7 +315,7 @@ func TestQuery_getQueryURL(t *testing.T) {
rawURL: "http://127.0.0.1:9429",
queryParams: "",
},
want: "http://127.0.0.1:9429/select/logsql/hits?end=1609462800&field=&query=&start=1609459200&step=15s",
want: "http://127.0.0.1:9429/select/logsql/hits?end=1609462800&query=&start=1609459200&step=15s",
wantErr: false,
},
{
Expand All @@ -334,7 +334,7 @@ func TestQuery_getQueryURL(t *testing.T) {
rawURL: "http://127.0.0.1:9429",
queryParams: "",
},
want: "http://127.0.0.1:9429/select/logsql/hits?end=1609462800&field=&query=&start=1609459200&step=15s",
want: "http://127.0.0.1:9429/select/logsql/hits?end=1609462800&query=&start=1609459200&step=15s",
wantErr: false,
},
{
Expand All @@ -353,7 +353,7 @@ func TestQuery_getQueryURL(t *testing.T) {
rawURL: "http://127.0.0.1:9429",
queryParams: "",
},
want: "http://127.0.0.1:9429/select/logsql/hits?end=1609462800&field=&query=_time%3A1s&start=1609459200&step=15s",
want: "http://127.0.0.1:9429/select/logsql/hits?end=1609462800&query=_time%3A1s&start=1609459200&step=15s",
wantErr: false,
},
{
Expand All @@ -372,7 +372,7 @@ func TestQuery_getQueryURL(t *testing.T) {
rawURL: "http://127.0.0.1:9429",
queryParams: "",
},
want: "http://127.0.0.1:9429/select/logsql/hits?end=1609462800&field=&query=_time%3A1s+%7C+stats+by%28type%29+count%28%29&start=1609459200&step=15s",
want: "http://127.0.0.1:9429/select/logsql/hits?end=1609462800&query=_time%3A1s+%7C+stats+by%28type%29+count%28%29&start=1609459200&step=15s",
wantErr: false,
},
{
Expand All @@ -391,7 +391,7 @@ func TestQuery_getQueryURL(t *testing.T) {
rawURL: "http://127.0.0.1:9429",
queryParams: "",
},
want: "http://127.0.0.1:9429/select/logsql/hits?end=1609462800&field=&query=_time%3A1s+%7C+stats+by%28type%29+count%28%29&start=1609459200&step=15s",
want: "http://127.0.0.1:9429/select/logsql/hits?end=1609462800&query=_time%3A1s+%7C+stats+by%28type%29+count%28%29&start=1609459200&step=15s",
wantErr: false,
},
{
Expand All @@ -410,7 +410,7 @@ func TestQuery_getQueryURL(t *testing.T) {
rawURL: "http://127.0.0.1:9429",
queryParams: "",
},
want: "http://127.0.0.1:9429/select/logsql/hits?end=1609462800&field=&query=_time%3A1s+and+syslog&start=1609459200&step=15s",
want: "http://127.0.0.1:9429/select/logsql/hits?end=1609462800&query=_time%3A1s+and+syslog&start=1609459200&step=15s",
wantErr: false,
},
{
Expand All @@ -429,7 +429,7 @@ func TestQuery_getQueryURL(t *testing.T) {
rawURL: "http://127.0.0.1:9429",
queryParams: "",
},
want: "http://127.0.0.1:9429/select/logsql/hits?end=1609462800&field=&query=_time%3A1s+and+syslog+%7C+stats+by%28type%29+count%28%29&start=1609459200&step=15s",
want: "http://127.0.0.1:9429/select/logsql/hits?end=1609462800&query=_time%3A1s+and+syslog+%7C+stats+by%28type%29+count%28%29&start=1609459200&step=15s",
wantErr: false,
},
{
Expand All @@ -448,7 +448,7 @@ func TestQuery_getQueryURL(t *testing.T) {
rawURL: "http://127.0.0.1:9429",
queryParams: "",
},
want: "http://127.0.0.1:9429/select/logsql/hits?end=1609462800&field=&query=_time%3A1s+and+syslog+%7C+stats+by%28type%29+count%28%29&start=1609459200&step=15s",
want: "http://127.0.0.1:9429/select/logsql/hits?end=1609462800&query=_time%3A1s+and+syslog+%7C+stats+by%28type%29+count%28%29&start=1609459200&step=15s",
wantErr: false,
},
{
Expand All @@ -467,7 +467,7 @@ func TestQuery_getQueryURL(t *testing.T) {
rawURL: "http://127.0.0.1:9429",
queryParams: "",
},
want: "http://127.0.0.1:9429/select/logsql/hits?end=1609462800&field=&query=%2A+and+syslog+%7C+stats+by%28type%29+count%28%29&start=1609459200&step=15s",
want: "http://127.0.0.1:9429/select/logsql/hits?end=1609462800&query=%2A+and+syslog+%7C+stats+by%28type%29+count%28%29&start=1609459200&step=15s",
wantErr: false,
},
}
Expand Down
25 changes: 25 additions & 0 deletions src/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -81,6 +81,31 @@ And apply `Transformations` by labels:

<img alt="Transformations" src="https://github.com/VictoriaMetrics/victorialogs-datasource/blob/main/src/img/panel_table_transformation.png?raw=true">

### Log level rules

The **Log level rules** section in the datasource configuration allows you to assign log levels based on custom field conditions. This helps classify logs dynamically (e.g., as `error`, `info`, `debug`, etc.) using rules you define.

#### How to use

1. Open the datasource settings.

2. Scroll to the **Log level rules** section.

3. Click **"Add rule"** to define a new rule.

4. For each rule, configure the following:

* **Enable switch** – enable or disable the rule.
* **Field name** – the log field the condition will evaluate.
* **Operator** – choose from: `Equals`, `Not equal`, `Matches regex`, `Less than`, `Greater than`
* **Value** – the value to compare the field against.
* **Log level** – level to assign if the condition matches: `critical`, `warning`, `error`, `info`, `debug`, `trace`, `unknown`
* **Delete button** – remove the rule.

5. After adding or editing rules, click **"Save & test"** to apply the changes.

**Rule priority**: If multiple rules match a log entry, the **first matching rule** (top to bottom) takes precedence.

## License

This project is licensed under
Expand Down
122 changes: 71 additions & 51 deletions src/backendResultTransformer.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,17 +3,28 @@ import {
DataQueryError,
DataQueryRequest,
DataQueryResponse,
Field,
FieldType,
isDataFrame,
LogLevel,
QueryResultMeta,
isDataFrame,
} from '@grafana/data';

import { LogLevelRule } from "./configuration/LogLevelRules/types";
import { resolveLogLevel } from "./configuration/LogLevelRules/utils";
import { getDerivedFields } from './getDerivedFields';
import { makeTableFrames } from './makeTableFrames';
import { getHighlighterExpressionsFromQuery } from './queryUtils';
import { dataFrameHasError } from './responseUtils';
import { DerivedFieldConfig, Query, QueryType } from './types';

const ANNOTATIONS_REF_ID = 'Anno';

enum FrameField {
Labels = 'labels',
Level = 'level'
}

function isMetricFrame(frame: DataFrame): boolean {
return frame.fields.every((field) => field.type === FieldType.time || field.type === FieldType.number);
}
Expand All @@ -29,22 +40,67 @@ function setFrameMeta(frame: DataFrame, meta: QueryResultMeta): DataFrame {
};
}

function addLevelField(frame: DataFrame, rules: LogLevelRule[]): DataFrame {
const rows = frame.length ?? frame.fields[0]?.values.length ?? 0;
const labelsField = frame.fields.find(f => f.name === FrameField.Labels);

const levelValues: LogLevel[] = Array.from({ length: rows }, (_, idx) => {
const labels = (labelsField?.values[idx] ?? {}) as Record<string, any>;
return resolveLogLevel(labels, rules);
});

const levelField: Field<LogLevel> = {
name: FrameField.Level,
type: FieldType.string,
config: {},
values: levelValues,
};

return { ...frame, fields: [...frame.fields, levelField] };
}

function transformDashboardLabelField(field: Field): Field {
if (field.name !== FrameField.Labels) {
return field;
}

return {
...field,
values: field.values.map((value) => {
return Object.entries(value).map(([key, val]) => {
return `${key}: ${JSON.stringify(val)}`;
});
}),
};
}

function getStreamFields(fields: Field[], transformLabels: boolean): Field[] {
if (!transformLabels) {
return fields;
}

return fields.map(transformDashboardLabelField);
}

function processStreamsFrames(
frames: DataFrame[],
queryMap: Map<string, Query>,
derivedFieldConfigs: DerivedFieldConfig[],
logLevelRules: LogLevelRule[]
): DataFrame[] {
return frames.map((frame) => {
const query = frame.refId !== undefined ? queryMap.get(frame.refId) : undefined;
if (query?.refId === "Anno") {return processDashboardStreamFrame(frame, query, derivedFieldConfigs);}
return processStreamFrame(frame, query, derivedFieldConfigs);
const isAnnotations = query?.refId === ANNOTATIONS_REF_ID
return processStreamFrame(frame, query, derivedFieldConfigs, logLevelRules, isAnnotations);
});
}

function processStreamFrame(
frame: DataFrame,
query: Query | undefined,
derivedFieldConfigs: DerivedFieldConfig[]
derivedFieldConfigs: DerivedFieldConfig[],
logLevelRules: LogLevelRule[],
transformLabels = false
): DataFrame {
const custom: Record<string, string> = {
...frame.meta?.custom, // keep the original meta.custom
Expand All @@ -61,55 +117,18 @@ function processStreamFrame(
custom,
};

const newFrame = setFrameMeta(frame, meta);
const derivedFields = getDerivedFields(newFrame, derivedFieldConfigs);
return {
...newFrame,
fields: [...newFrame.fields, ...derivedFields],
};
}
const frameWithMeta = setFrameMeta(frame, meta);
const frameWithLevel = addLevelField(frameWithMeta, logLevelRules);

function processDashboardStreamFrame(
frame: DataFrame,
query: Query | undefined,
derivedFieldConfigs: DerivedFieldConfig[]
): DataFrame {
const custom: Record<string, string> = {
...frame.meta?.custom, // keep the original meta.custom
};

if (dataFrameHasError(frame)) {
custom.error = 'Error when parsing some of the logs';
}

const meta: QueryResultMeta = {
preferredVisualisationType: 'logs',
limit: query?.maxLines,
searchWords: query !== undefined ? getHighlighterExpressionsFromQuery(query.expr) : undefined,
custom,
};

const newFrame = setFrameMeta(frame, meta);
const derivedFields = getDerivedFields(newFrame, derivedFieldConfigs);
const derivedFields = getDerivedFields(frameWithLevel, derivedFieldConfigs);
const baseFields = getStreamFields(frameWithLevel.fields, transformLabels)

return {
...newFrame,
...frameWithLevel,
fields: [
...newFrame.fields.map((field) => {
if (field.name === 'labels') {
return {
...field,
values: field.values.map((value) => {
return Object.entries(value).map(([key, value]) => {
return `${key}: ${JSON.stringify(value)}`;
});
}),
};
}
return field;
}),
...derivedFields,
],
...baseFields,
...derivedFields
]
};
}

Expand Down Expand Up @@ -180,7 +199,8 @@ function improveError(error: DataQueryError | undefined, queryMap: Map<string, Q
export function transformBackendResult(
response: DataQueryResponse,
request: DataQueryRequest,
derivedFieldConfigs: DerivedFieldConfig[]
derivedFieldConfigs: DerivedFieldConfig[],
logLevelRules: LogLevelRule[],
): DataQueryResponse {
const { data, errors, ...rest } = response;
const queries = request.targets;
Expand Down Expand Up @@ -208,7 +228,7 @@ export function transformBackendResult(
data: [
...processMetricRangeFrames(metricRangeFrames),
...processMetricInstantFrames(metricInstantFrames),
...processStreamsFrames(streamsFrames, queryMap, derivedFieldConfigs),
...processStreamsFrames(streamsFrames, queryMap, derivedFieldConfigs, logLevelRules),
],
};
}
25 changes: 16 additions & 9 deletions src/configuration/ConfigEditor.tsx
Original file line number Diff line number Diff line change
@@ -1,16 +1,17 @@
import React from 'react';
import { gte } from 'semver';

import { DataSourcePluginOptionsEditorProps, DataSourceSettings } from '@grafana/data';
import { DataSourcePluginOptionsEditorProps, DataSourceSettings, FeatureToggles } from '@grafana/data';
import { config } from '@grafana/runtime';
import { InlineSwitch, InlineField, DataSourceHttpSettings, SecureSocksProxySettings } from "@grafana/ui";
import { InlineSwitch, InlineField, DataSourceHttpSettings, Space } from "@grafana/ui";

import { Options } from '../types';

import { AlertingSettings } from './AlertingSettings';
import { DerivedFields } from "./DerivedFields";
import { HelpfulLinks } from "./HelpfulLinks";
import { LimitsSettings } from "./LimitSettings";
import { LogLevelRulesEditor } from "./LogLevelRules/LogLevelRulesEditor";
import { LogsSettings } from './LogsSettings';
import { QuerySettings } from './QuerySettings';

Expand Down Expand Up @@ -41,18 +42,24 @@ const ConfigEditor = (props: Props) => {
sigV4AuthToggleEnabled={config.sigV4AuthEnabled}
/>
<AlertingSettings options={options} onOptionsChange={onOptionsChange}/>
<QuerySettings
maxLines={options.jsonData.maxLines || ''}
onMaxLinedChange={(value) => onOptionsChange(setMaxLines(options, value))}
/>

<Space v={5} />

<LimitsSettings {...props}>
<QuerySettings
maxLines={options.jsonData.maxLines || ''}
onMaxLinedChange={(value) => onOptionsChange(setMaxLines(options, value))}
/>
</LimitsSettings>

<LogsSettings {...props}/>

<DerivedFields
fields={options.jsonData.derivedFields}
onChange={(value) => onOptionsChange(setDerivedFields(options, value))}
/>

<LogsSettings {...props}/>

<LimitsSettings {...props}/>
<LogLevelRulesEditor {...props}/>

{config.featureToggles['secureSocksDSProxyEnabled' as keyof FeatureToggles] && gte(config.buildInfo.version, '10.0.0') && (
<>
Expand Down
Loading