diff --git a/apps/mongodb.go b/apps/mongodb.go index cb705ea779..e88849b1d8 100644 --- a/apps/mongodb.go +++ b/apps/mongodb.go @@ -16,12 +16,9 @@ package apps import ( "context" - "fmt" "strings" "github.com/GoogleCloudPlatform/ops-agent/confgenerator" - "github.com/GoogleCloudPlatform/ops-agent/confgenerator/fluentbit" - "github.com/GoogleCloudPlatform/ops-agent/confgenerator/fluentbit/modify" "github.com/GoogleCloudPlatform/ops-agent/confgenerator/otel" "github.com/GoogleCloudPlatform/ops-agent/internal/secret" ) @@ -95,20 +92,19 @@ func init() { confgenerator.MetricsReceiverTypes.RegisterType(func() confgenerator.MetricsReceiver { return &MetricsReceiverMongoDB{} }) } -type LoggingProcessorMongodb struct { - confgenerator.ConfigComponent `yaml:",inline"` +type LoggingProcessorMacroMongodb struct { } -func (*LoggingProcessorMongodb) Type() string { +func (LoggingProcessorMacroMongodb) Type() string { return "mongodb" } -func (p *LoggingProcessorMongodb) Components(ctx context.Context, tag, uid string) []fluentbit.Component { - c := []fluentbit.Component{} +func (p LoggingProcessorMacroMongodb) Expand(ctx context.Context) []confgenerator.InternalLoggingProcessor { + c := []confgenerator.InternalLoggingProcessor{} - c = append(c, p.JsonLogComponents(ctx, tag, uid)...) - c = append(c, p.RegexLogComponents(tag, uid)...) - c = append(c, p.severityParser(ctx, tag, uid)...) + c = append(c, p.JsonLogComponents(ctx)...) + c = append(c, p.RegexLogComponents()...) + c = append(c, p.severityParser()...) return c } @@ -116,99 +112,61 @@ func (p *LoggingProcessorMongodb) Components(ctx context.Context, tag, uid strin // JsonLogComponents are the fluentbit components for parsing log messages that are json formatted. // these are generally messages from mongo with versions greater than or equal to 4.4 // documentation: https://docs.mongodb.com/v4.4/reference/log-messages/#log-message-format -func (p *LoggingProcessorMongodb) JsonLogComponents(ctx context.Context, tag, uid string) []fluentbit.Component { - c := p.jsonParserWithTimeKey(ctx, tag, uid) +func (p LoggingProcessorMacroMongodb) JsonLogComponents(ctx context.Context) []confgenerator.InternalLoggingProcessor { + c := p.jsonParserWithTimeKey() - c = append(c, p.promoteWiredTiger(tag, uid)...) - c = append(c, p.renames(tag, uid)...) + c = append(c, p.promoteWiredTiger()...) + c = append(c, p.renames()...) return c } // jsonParserWithTimeKey requires promotion of the nested timekey for the json parser so we must // first promote the $date field from the "t" field before declaring the parser -func (p *LoggingProcessorMongodb) jsonParserWithTimeKey(ctx context.Context, tag, uid string) []fluentbit.Component { - c := []fluentbit.Component{} +func (p LoggingProcessorMacroMongodb) jsonParserWithTimeKey() []confgenerator.InternalLoggingProcessor { + c := []confgenerator.InternalLoggingProcessor{} - jsonParser := &confgenerator.LoggingProcessorParseJson{ + c = append(c, &confgenerator.LoggingProcessorParseJson{ ParserShared: confgenerator.ParserShared{ - TimeKey: "time", - TimeFormat: "%Y-%m-%dT%H:%M:%S.%L%z", Types: map[string]string{ "id": "integer", "message": "string", }, }, - } - jpComponents := jsonParser.Components(ctx, tag, uid) - - // The parserFilterComponent is the actual filter component that configures and defines - // which parser to use. We need the component to determine which parser to use when - // re-parsing below. Each time a parser filter is used, there are 2 filter components right - // before it to account for the nest lua script (see confgenerator/fluentbit/parse_deduplication.go). - // Therefore, the parse filter component is actually the third component in the list. - parserFilterComponent := jpComponents[2] - c = append(c, jpComponents...) - - tempPrefix := "temp_ts_" - timeKey := "time" + }) + // have to bring $date to top level in order for it to be parsed as timeKey // see https://github.com/fluent/fluent-bit/issues/1013 - liftTs := fluentbit.Component{ - Kind: "FILTER", - Config: map[string]string{ - "Name": "nest", - "Match": tag, - "Operation": "lift", - "Nested_under": "t", - "Add_prefix": tempPrefix, + c = append(c, &confgenerator.LoggingProcessorModifyFields{ + Fields: map[string]*confgenerator.ModifyField{ + "jsonPayload.time": { + MoveFrom: "jsonPayload.t.$date", + }, }, - } - - renameTsOption := modify.NewHardRenameOptions(fmt.Sprintf("%s$date", tempPrefix), timeKey) - renameTs := renameTsOption.Component(tag) + }) - c = append(c, liftTs, renameTs) + c = append(c, &confgenerator.LoggingProcessorRemoveField{ + Field: "t", + }) // IMPORTANT: now that we have lifted the json to top level // we need to re-parse in order to properly set time at the // parser level - nestFilters := fluentbit.LuaFilterComponents(tag, fluentbit.ParserNestLuaFunction, fmt.Sprintf(fluentbit.ParserNestLuaScriptContents, "message")) - parserFilter := fluentbit.Component{ - Kind: "FILTER", - Config: map[string]string{ - "Name": "parser", - "Match": tag, - "Key_Name": "message", - "Reserve_Data": "True", - "Parser": parserFilterComponent.OrderedConfig[0][1], - }, - } - mergeFilters := fluentbit.LuaFilterComponents(tag, fluentbit.ParserMergeLuaFunction, fluentbit.ParserMergeLuaScriptContents) - c = append(c, nestFilters...) - c = append(c, parserFilter) - c = append(c, mergeFilters...) - - removeTimestamp := fluentbit.Component{ - Kind: "FILTER", - Config: map[string]string{ - "Name": "modify", - "Match": tag, - "Remove": timeKey, + c = append(c, &confgenerator.LoggingProcessorParseTimestamp{ + ParserShared: confgenerator.ParserShared{ + TimeKey: "time", + TimeFormat: "%Y-%m-%dT%H:%M:%S.%L%z", }, - } - c = append(c, removeTimestamp) + }) return c } // severityParser is used by both regex and json parser to ensure an "s" field on the entry gets translated // to a valid logging.googleapis.com/seveirty field -func (p *LoggingProcessorMongodb) severityParser(ctx context.Context, tag, uid string) []fluentbit.Component { - severityComponents := []fluentbit.Component{} - - severityComponents = append(severityComponents, - confgenerator.LoggingProcessorModifyFields{ +func (p LoggingProcessorMacroMongodb) severityParser() []confgenerator.InternalLoggingProcessor { + return []confgenerator.InternalLoggingProcessor{ + &confgenerator.LoggingProcessorModifyFields{ Fields: map[string]*confgenerator.ModifyField{ "jsonPayload.severity": { MoveFrom: "jsonPayload.s", @@ -231,115 +189,85 @@ func (p *LoggingProcessorMongodb) severityParser(ctx context.Context, tag, uid s }, InstrumentationSourceLabel: instrumentationSourceValue(p.Type()), }, - }.Components(ctx, tag, uid)..., - ) - - return severityComponents + }, + } } -func (p *LoggingProcessorMongodb) renames(tag, uid string) []fluentbit.Component { - r := []fluentbit.Component{} +func (p LoggingProcessorMacroMongodb) renames() []confgenerator.InternalLoggingProcessor { + r := []confgenerator.InternalLoggingProcessor{} renames := []struct { src string dest string }{ - {"c", "component"}, - {"ctx", "context"}, - {"msg", "message"}, + {"jsonPayload.c", "jsonPayload.component"}, + {"jsonPayload.ctx", "jsonPayload.context"}, + {"jsonPayload.msg", "jsonPayload.message"}, + {"jsonPayload.attr", "jsonPayload.attributes"}, } for _, rename := range renames { - rename := modify.NewRenameOptions(rename.src, rename.dest) - r = append(r, rename.Component(tag)) + r = append(r, &confgenerator.LoggingProcessorModifyFields{ + Fields: map[string]*confgenerator.ModifyField{ + rename.dest: { + MoveFrom: rename.src, + }, + }, + }) } return r } -func (p *LoggingProcessorMongodb) promoteWiredTiger(tag, uid string) []fluentbit.Component { +func (p LoggingProcessorMacroMongodb) promoteWiredTiger() []confgenerator.InternalLoggingProcessor { // promote messages that are WiredTiger messages and are nested in attr.message + c := []confgenerator.InternalLoggingProcessor{} + addPrefix := "temp_attributes_" - upNest := fluentbit.Component{ - Kind: "FILTER", - Config: map[string]string{ - "Name": "nest", - "Match": tag, - "Operation": "lift", - "Nested_under": "attr", - "Add_prefix": addPrefix, - }, - } - hardRenameMessage := modify.NewHardRenameOptions(fmt.Sprintf("%smessage", addPrefix), "msg") - wiredTigerRename := hardRenameMessage.Component(tag) - - renameRemainingAttributes := fluentbit.Component{ - Kind: "FILTER", - Config: map[string]string{ - "Name": "nest", - "Wildcard": fmt.Sprintf("%s*", addPrefix), - "Match": tag, - "Operation": "nest", - "Nest_under": "attributes", - "Remove_prefix": addPrefix, + c = append(c, &confgenerator.LoggingProcessorModifyFields{ + Fields: map[string]*confgenerator.ModifyField{ + "jsonPayload.temp_attributes_message": { + MoveFrom: "jsonPayload.attr.message", + }, }, - } + }) + + c = append(c, &confgenerator.LoggingProcessorRenameIfExists{ + Field: addPrefix + "message", + NewName: "msg", + }) - return []fluentbit.Component{upNest, wiredTigerRename, renameRemainingAttributes} + return c } -func (p *LoggingProcessorMongodb) RegexLogComponents(tag, uid string) []fluentbit.Component { - c := []fluentbit.Component{} - parseKey := "message" - parser, parserName := fluentbit.ParserComponentBase("%Y-%m-%dT%H:%M:%S.%L%z", "timestamp", map[string]string{ - "message": "string", - "id": "integer", - "s": "string", - "component": "string", - "context": "string", - }, fmt.Sprintf("%s_regex", tag), uid) - parser.Config["Format"] = "regex" - parser.Config["Regex"] = `^(?[^ ]*)\s+(?\w)\s+(?[^ ]+)\s+\[(?[^\]]+)]\s+(?.*?) *(?(\d+))?(:?ms)?$` - parser.Config["Key_Name"] = parseKey - - nestFilters := fluentbit.LuaFilterComponents(tag, fluentbit.ParserNestLuaFunction, fmt.Sprintf(fluentbit.ParserNestLuaScriptContents, parseKey)) - parserFilter := fluentbit.Component{ - Kind: "FILTER", - Config: map[string]string{ - "Match": tag, - "Name": "parser", - "Parser": parserName, - "Reserve_Data": "True", - "Key_Name": parseKey, - }, - } - mergeFilters := fluentbit.LuaFilterComponents(tag, fluentbit.ParserMergeLuaFunction, fluentbit.ParserMergeLuaScriptContents) +func (p LoggingProcessorMacroMongodb) RegexLogComponents() []confgenerator.InternalLoggingProcessor { + c := []confgenerator.InternalLoggingProcessor{} - c = append(c, parser) - c = append(c, nestFilters...) - c = append(c, parserFilter) - c = append(c, mergeFilters...) + c = append(c, &confgenerator.LoggingProcessorParseRegex{ + ParserShared: confgenerator.ParserShared{ + TimeKey: "timestamp", + TimeFormat: "%Y-%m-%dT%H:%M:%S.%L%z", + Types: map[string]string{ + "message": "string", + "id": "integer", + "s": "string", + "component": "string", + "context": "string", + }, + }, + Regex: `^(?[^ ]*)\s+(?\w)\s+(?[^ ]+)\s+\[(?[^\]]+)]\s+(?.*?) *(?(\d+))?(:?ms)?$`, + Field: "message", + }) return c } -type LoggingReceiverMongodb struct { - LoggingProcessorMongodb `yaml:",inline"` - ReceiverMixin confgenerator.LoggingReceiverFilesMixin `yaml:",inline" validate:"structonly"` -} - -func (r *LoggingReceiverMongodb) Components(ctx context.Context, tag string) []fluentbit.Component { - if len(r.ReceiverMixin.IncludePaths) == 0 { - r.ReceiverMixin.IncludePaths = []string{ - // default logging location - "/var/log/mongodb/mongod.log*", - } +func loggingReceiverFilesMixinMongodb() confgenerator.LoggingReceiverFilesMixin { + return confgenerator.LoggingReceiverFilesMixin{ + IncludePaths: []string{"/var/log/mongodb/mongod.log*"}, } - c := r.ReceiverMixin.Components(ctx, tag) - c = append(c, r.LoggingProcessorMongodb.Components(ctx, tag, "mongodb")...) - return c } func init() { - confgenerator.LoggingReceiverTypes.RegisterType(func() confgenerator.LoggingReceiver { return &LoggingReceiverMongodb{} }) + confgenerator.RegisterLoggingFilesProcessorMacro[LoggingProcessorMacroMongodb](loggingReceiverFilesMixinMongodb) } diff --git a/confgenerator/logging_processors.go b/confgenerator/logging_processors.go index 820119928f..c949f01877 100644 --- a/confgenerator/logging_processors.go +++ b/confgenerator/logging_processors.go @@ -534,6 +534,83 @@ func (p LoggingProcessorNestWildcard) Components(ctx context.Context, tag, uid s } } +type LoggingProcessorRemoveField struct { + Field string +} + +func (p LoggingProcessorRemoveField) Components(ctx context.Context, tag, uid string) []fluentbit.Component { + filter := fluentbit.Component{ + Kind: "FILTER", + Config: map[string]string{ + "Name": "modify", + "Match": tag, + "Remove": p.Field, + }, + } + return []fluentbit.Component{filter} +} + +type LoggingProcessorParseTimestamp struct { + ParserShared `yaml:",inline"` +} + +func (p LoggingProcessorParseTimestamp) Components(ctx context.Context, tag, uid string) []fluentbit.Component { + parser, parserName := p.ParserShared.Component(tag, uid) + + parser.Config["Format"] = "regex" + parser.Config["Regex"] = fmt.Sprintf("^(?<%s>.*)$", p.TimeKey) + + components := []fluentbit.Component{} + components = append(components, parser) + + components = append(components, + fluentbit.ParserFilterComponents(tag, p.TimeKey, []string{parserName}, false)..., + ) + + return components +} + +type LoggingProcessorRenameIfExists struct { + Field string + NewName string +} + +func (p LoggingProcessorRenameIfExists) Components(ctx context.Context, tag, uid string) []fluentbit.Component { + return []fluentbit.Component{modify.NewHardRenameOptions(p.Field, p.NewName).Component(tag)} +} + +func (p LoggingProcessorRenameIfExists) Processors(ctx context.Context) ([]otel.Component, error) { + fromMember, err := filter.NewMemberLegacy(p.Field) + if err != nil { + return nil, err + } + fromAccessor, err := fromMember.OTTLAccessor() + if err != nil { + return nil, err + } + + toMember, err := filter.NewMemberLegacy(p.NewName) + if err != nil { + return nil, err + } + toAccessor, err := toMember.OTTLAccessor() + if err != nil { + return nil, err + } + + // Rename only if the source field exists + statements := ottl.NewStatements( + toAccessor.SetIf(fromAccessor, fromAccessor.IsPresent()), + fromAccessor.DeleteIf(fromAccessor.IsPresent()), + ) + + return []otel.Component{otel.Transform( + "log", "log", + statements, + )}, nil + +} + var LegacyBuiltinProcessors = map[string]LoggingProcessor{ "lib:default_message_parser": &LoggingProcessorParseRegex{ Regex: `^(?.*)$`, diff --git a/confgenerator/testdata/feature/golden.csv b/confgenerator/testdata/feature/golden.csv index 7725da67ee..18f08d572f 100644 --- a/confgenerator/testdata/feature/golden.csv +++ b/confgenerator/testdata/feature/golden.csv @@ -132,6 +132,7 @@ App,Field,Override, *confgenerator.loggingProcessorMacroAdapter[github.com/GoogleCloudPlatform/ops-agent/apps.LoggingProcessorMacroHbaseSystem],confgenerator.ConfigComponent.Type, *confgenerator.loggingProcessorMacroAdapter[github.com/GoogleCloudPlatform/ops-agent/apps.LoggingProcessorMacroJettyAccess],confgenerator.ConfigComponent.Type, *confgenerator.loggingProcessorMacroAdapter[github.com/GoogleCloudPlatform/ops-agent/apps.LoggingProcessorMacroKafka],confgenerator.ConfigComponent.Type, +*confgenerator.loggingProcessorMacroAdapter[github.com/GoogleCloudPlatform/ops-agent/apps.LoggingProcessorMacroMongodb],confgenerator.ConfigComponent.Type, *confgenerator.loggingProcessorMacroAdapter[github.com/GoogleCloudPlatform/ops-agent/apps.LoggingProcessorMacroMysqlError],confgenerator.ConfigComponent.Type, *confgenerator.loggingProcessorMacroAdapter[github.com/GoogleCloudPlatform/ops-agent/apps.LoggingProcessorMacroMysqlGeneral],confgenerator.ConfigComponent.Type, *confgenerator.loggingProcessorMacroAdapter[github.com/GoogleCloudPlatform/ops-agent/apps.LoggingProcessorMacroMysqlSlow],confgenerator.ConfigComponent.Type, @@ -177,6 +178,8 @@ App,Field,Override, *confgenerator.loggingReceiverMacroAdapter[*github.com/GoogleCloudPlatform/ops-agent/confgenerator.loggingFilesProcessorMacroAdapter[github.com/GoogleCloudPlatform/ops-agent/apps.LoggingProcessorMacroJettyAccess]],confgenerator.ConfigComponent.Type, *confgenerator.loggingReceiverMacroAdapter[*github.com/GoogleCloudPlatform/ops-agent/confgenerator.loggingFilesProcessorMacroAdapter[github.com/GoogleCloudPlatform/ops-agent/apps.LoggingProcessorMacroKafka]],ReceiverMacro, *confgenerator.loggingReceiverMacroAdapter[*github.com/GoogleCloudPlatform/ops-agent/confgenerator.loggingFilesProcessorMacroAdapter[github.com/GoogleCloudPlatform/ops-agent/apps.LoggingProcessorMacroKafka]],confgenerator.ConfigComponent.Type, +*confgenerator.loggingReceiverMacroAdapter[*github.com/GoogleCloudPlatform/ops-agent/confgenerator.loggingFilesProcessorMacroAdapter[github.com/GoogleCloudPlatform/ops-agent/apps.LoggingProcessorMacroMongodb]],ReceiverMacro, +*confgenerator.loggingReceiverMacroAdapter[*github.com/GoogleCloudPlatform/ops-agent/confgenerator.loggingFilesProcessorMacroAdapter[github.com/GoogleCloudPlatform/ops-agent/apps.LoggingProcessorMacroMongodb]],confgenerator.ConfigComponent.Type, *confgenerator.loggingReceiverMacroAdapter[*github.com/GoogleCloudPlatform/ops-agent/confgenerator.loggingFilesProcessorMacroAdapter[github.com/GoogleCloudPlatform/ops-agent/apps.LoggingProcessorMacroMysqlError]],ReceiverMacro, *confgenerator.loggingReceiverMacroAdapter[*github.com/GoogleCloudPlatform/ops-agent/confgenerator.loggingFilesProcessorMacroAdapter[github.com/GoogleCloudPlatform/ops-agent/apps.LoggingProcessorMacroMysqlError]],confgenerator.ConfigComponent.Type, *confgenerator.loggingReceiverMacroAdapter[*github.com/GoogleCloudPlatform/ops-agent/confgenerator.loggingFilesProcessorMacroAdapter[github.com/GoogleCloudPlatform/ops-agent/apps.LoggingProcessorMacroMysqlGeneral]],ReceiverMacro, diff --git a/confgenerator/testdata/goldens/invalid-logging-otel-unsupported_processor/golden/linux-gpu/error b/confgenerator/testdata/goldens/invalid-logging-otel-unsupported_processor/golden/linux-gpu/error index b1d3076381..69215d9115 100644 --- a/confgenerator/testdata/goldens/invalid-logging-otel-unsupported_processor/golden/linux-gpu/error +++ b/confgenerator/testdata/goldens/invalid-logging-otel-unsupported_processor/golden/linux-gpu/error @@ -1 +1 @@ -logging processor with type "unsupported" is not supported. Supported logging processor types: [apache_access, apache_error, cassandra_debug, cassandra_gc, cassandra_system, couchdb, elasticsearch_gc, elasticsearch_json, exclude_logs, flink, hadoop, hbase_system, jetty_access, kafka, modify_fields, mysql_error, mysql_general, mysql_slow, nginx_access, nginx_error, oracledb_alert, oracledb_audit, parse_json, parse_multiline, parse_regex, postgresql_general, rabbitmq, redis, saphana, solr_system, tomcat_access, tomcat_system, varnish, vault_audit, wildfly_system, zookeeper_general]. \ No newline at end of file +logging processor with type "unsupported" is not supported. Supported logging processor types: [apache_access, apache_error, cassandra_debug, cassandra_gc, cassandra_system, couchdb, elasticsearch_gc, elasticsearch_json, exclude_logs, flink, hadoop, hbase_system, jetty_access, kafka, modify_fields, mysql_error, mysql_general, mysql_slow, nginx_access, nginx_error, oracledb_alert, oracledb_audit, parse_json, parse_multiline, parse_regex, postgresql_general, rabbitmq, redis, saphana, solr_system, tomcat_access, tomcat_system, varnish, vault_audit, wildfly_system, zookeeper_general]. diff --git a/confgenerator/testdata/goldens/invalid-logging-otel-unsupported_processor/golden/linux/error b/confgenerator/testdata/goldens/invalid-logging-otel-unsupported_processor/golden/linux/error index b1d3076381..69215d9115 100644 --- a/confgenerator/testdata/goldens/invalid-logging-otel-unsupported_processor/golden/linux/error +++ b/confgenerator/testdata/goldens/invalid-logging-otel-unsupported_processor/golden/linux/error @@ -1 +1 @@ -logging processor with type "unsupported" is not supported. Supported logging processor types: [apache_access, apache_error, cassandra_debug, cassandra_gc, cassandra_system, couchdb, elasticsearch_gc, elasticsearch_json, exclude_logs, flink, hadoop, hbase_system, jetty_access, kafka, modify_fields, mysql_error, mysql_general, mysql_slow, nginx_access, nginx_error, oracledb_alert, oracledb_audit, parse_json, parse_multiline, parse_regex, postgresql_general, rabbitmq, redis, saphana, solr_system, tomcat_access, tomcat_system, varnish, vault_audit, wildfly_system, zookeeper_general]. \ No newline at end of file +logging processor with type "unsupported" is not supported. Supported logging processor types: [apache_access, apache_error, cassandra_debug, cassandra_gc, cassandra_system, couchdb, elasticsearch_gc, elasticsearch_json, exclude_logs, flink, hadoop, hbase_system, jetty_access, kafka, modify_fields, mysql_error, mysql_general, mysql_slow, nginx_access, nginx_error, oracledb_alert, oracledb_audit, parse_json, parse_multiline, parse_regex, postgresql_general, rabbitmq, redis, saphana, solr_system, tomcat_access, tomcat_system, varnish, vault_audit, wildfly_system, zookeeper_general]. diff --git a/confgenerator/testdata/goldens/invalid-logging-otel-unsupported_processor/golden/windows-2012/error b/confgenerator/testdata/goldens/invalid-logging-otel-unsupported_processor/golden/windows-2012/error index 81aa4d2801..81d4cc4b86 100644 --- a/confgenerator/testdata/goldens/invalid-logging-otel-unsupported_processor/golden/windows-2012/error +++ b/confgenerator/testdata/goldens/invalid-logging-otel-unsupported_processor/golden/windows-2012/error @@ -1 +1 @@ -logging processor with type "unsupported" is not supported. Supported logging processor types: [apache_access, apache_error, cassandra_debug, cassandra_gc, cassandra_system, couchdb, elasticsearch_gc, elasticsearch_json, exclude_logs, flink, hadoop, hbase_system, iis_access, jetty_access, kafka, modify_fields, mysql_error, mysql_general, mysql_slow, nginx_access, nginx_error, oracledb_alert, oracledb_audit, parse_json, parse_multiline, parse_regex, postgresql_general, rabbitmq, redis, saphana, solr_system, tomcat_access, tomcat_system, varnish, vault_audit, wildfly_system, zookeeper_general]. \ No newline at end of file +logging processor with type "unsupported" is not supported. Supported logging processor types: [apache_access, apache_error, cassandra_debug, cassandra_gc, cassandra_system, couchdb, elasticsearch_gc, elasticsearch_json, exclude_logs, flink, hadoop, hbase_system, iis_access, jetty_access, kafka, modify_fields, mysql_error, mysql_general, mysql_slow, nginx_access, nginx_error, oracledb_alert, oracledb_audit, parse_json, parse_multiline, parse_regex, postgresql_general, rabbitmq, redis, saphana, solr_system, tomcat_access, tomcat_system, varnish, vault_audit, wildfly_system, zookeeper_general]. diff --git a/confgenerator/testdata/goldens/invalid-logging-otel-unsupported_processor/golden/windows/error b/confgenerator/testdata/goldens/invalid-logging-otel-unsupported_processor/golden/windows/error index 81aa4d2801..81d4cc4b86 100644 --- a/confgenerator/testdata/goldens/invalid-logging-otel-unsupported_processor/golden/windows/error +++ b/confgenerator/testdata/goldens/invalid-logging-otel-unsupported_processor/golden/windows/error @@ -1 +1 @@ -logging processor with type "unsupported" is not supported. Supported logging processor types: [apache_access, apache_error, cassandra_debug, cassandra_gc, cassandra_system, couchdb, elasticsearch_gc, elasticsearch_json, exclude_logs, flink, hadoop, hbase_system, iis_access, jetty_access, kafka, modify_fields, mysql_error, mysql_general, mysql_slow, nginx_access, nginx_error, oracledb_alert, oracledb_audit, parse_json, parse_multiline, parse_regex, postgresql_general, rabbitmq, redis, saphana, solr_system, tomcat_access, tomcat_system, varnish, vault_audit, wildfly_system, zookeeper_general]. \ No newline at end of file +logging processor with type "unsupported" is not supported. Supported logging processor types: [apache_access, apache_error, cassandra_debug, cassandra_gc, cassandra_system, couchdb, elasticsearch_gc, elasticsearch_json, exclude_logs, flink, hadoop, hbase_system, iis_access, jetty_access, kafka, modify_fields, mysql_error, mysql_general, mysql_slow, nginx_access, nginx_error, oracledb_alert, oracledb_audit, parse_json, parse_multiline, parse_regex, postgresql_general, rabbitmq, redis, saphana, solr_system, tomcat_access, tomcat_system, varnish, vault_audit, wildfly_system, zookeeper_general]. diff --git a/confgenerator/testdata/goldens/invalid-logging-processor_unsupported_type/golden/linux-gpu/error b/confgenerator/testdata/goldens/invalid-logging-processor_unsupported_type/golden/linux-gpu/error index 8ed152dbea..146d1b04d9 100644 --- a/confgenerator/testdata/goldens/invalid-logging-processor_unsupported_type/golden/linux-gpu/error +++ b/confgenerator/testdata/goldens/invalid-logging-processor_unsupported_type/golden/linux-gpu/error @@ -1 +1 @@ -logging processor with type "unsupported_type" is not supported. Supported logging processor types: [apache_access, apache_error, cassandra_debug, cassandra_gc, cassandra_system, couchdb, elasticsearch_gc, elasticsearch_json, exclude_logs, flink, hadoop, hbase_system, jetty_access, kafka, modify_fields, mysql_error, mysql_general, mysql_slow, nginx_access, nginx_error, oracledb_alert, oracledb_audit, parse_json, parse_multiline, parse_regex, postgresql_general, rabbitmq, redis, saphana, solr_system, tomcat_access, tomcat_system, varnish, vault_audit, wildfly_system, zookeeper_general]. \ No newline at end of file +logging processor with type "unsupported_type" is not supported. Supported logging processor types: [apache_access, apache_error, cassandra_debug, cassandra_gc, cassandra_system, couchdb, elasticsearch_gc, elasticsearch_json, exclude_logs, flink, hadoop, hbase_system, jetty_access, kafka, modify_fields, mysql_error, mysql_general, mysql_slow, nginx_access, nginx_error, oracledb_alert, oracledb_audit, parse_json, parse_multiline, parse_regex, postgresql_general, rabbitmq, redis, saphana, solr_system, tomcat_access, tomcat_system, varnish, vault_audit, wildfly_system, zookeeper_general]. diff --git a/confgenerator/testdata/goldens/invalid-logging-processor_unsupported_type/golden/linux/error b/confgenerator/testdata/goldens/invalid-logging-processor_unsupported_type/golden/linux/error index 8ed152dbea..146d1b04d9 100644 --- a/confgenerator/testdata/goldens/invalid-logging-processor_unsupported_type/golden/linux/error +++ b/confgenerator/testdata/goldens/invalid-logging-processor_unsupported_type/golden/linux/error @@ -1 +1 @@ -logging processor with type "unsupported_type" is not supported. Supported logging processor types: [apache_access, apache_error, cassandra_debug, cassandra_gc, cassandra_system, couchdb, elasticsearch_gc, elasticsearch_json, exclude_logs, flink, hadoop, hbase_system, jetty_access, kafka, modify_fields, mysql_error, mysql_general, mysql_slow, nginx_access, nginx_error, oracledb_alert, oracledb_audit, parse_json, parse_multiline, parse_regex, postgresql_general, rabbitmq, redis, saphana, solr_system, tomcat_access, tomcat_system, varnish, vault_audit, wildfly_system, zookeeper_general]. \ No newline at end of file +logging processor with type "unsupported_type" is not supported. Supported logging processor types: [apache_access, apache_error, cassandra_debug, cassandra_gc, cassandra_system, couchdb, elasticsearch_gc, elasticsearch_json, exclude_logs, flink, hadoop, hbase_system, jetty_access, kafka, modify_fields, mysql_error, mysql_general, mysql_slow, nginx_access, nginx_error, oracledb_alert, oracledb_audit, parse_json, parse_multiline, parse_regex, postgresql_general, rabbitmq, redis, saphana, solr_system, tomcat_access, tomcat_system, varnish, vault_audit, wildfly_system, zookeeper_general]. diff --git a/confgenerator/testdata/goldens/invalid-logging-processor_unsupported_type/golden/windows-2012/error b/confgenerator/testdata/goldens/invalid-logging-processor_unsupported_type/golden/windows-2012/error index fa19c4e4fd..3ecbce5fba 100644 --- a/confgenerator/testdata/goldens/invalid-logging-processor_unsupported_type/golden/windows-2012/error +++ b/confgenerator/testdata/goldens/invalid-logging-processor_unsupported_type/golden/windows-2012/error @@ -1 +1 @@ -logging processor with type "unsupported_type" is not supported. Supported logging processor types: [apache_access, apache_error, cassandra_debug, cassandra_gc, cassandra_system, couchdb, elasticsearch_gc, elasticsearch_json, exclude_logs, flink, hadoop, hbase_system, iis_access, jetty_access, kafka, modify_fields, mysql_error, mysql_general, mysql_slow, nginx_access, nginx_error, oracledb_alert, oracledb_audit, parse_json, parse_multiline, parse_regex, postgresql_general, rabbitmq, redis, saphana, solr_system, tomcat_access, tomcat_system, varnish, vault_audit, wildfly_system, zookeeper_general]. \ No newline at end of file +logging processor with type "unsupported_type" is not supported. Supported logging processor types: [apache_access, apache_error, cassandra_debug, cassandra_gc, cassandra_system, couchdb, elasticsearch_gc, elasticsearch_json, exclude_logs, flink, hadoop, hbase_system, iis_access, jetty_access, kafka, modify_fields, mysql_error, mysql_general, mysql_slow, nginx_access, nginx_error, oracledb_alert, oracledb_audit, parse_json, parse_multiline, parse_regex, postgresql_general, rabbitmq, redis, saphana, solr_system, tomcat_access, tomcat_system, varnish, vault_audit, wildfly_system, zookeeper_general]. diff --git a/confgenerator/testdata/goldens/invalid-logging-processor_unsupported_type/golden/windows/error b/confgenerator/testdata/goldens/invalid-logging-processor_unsupported_type/golden/windows/error index fa19c4e4fd..3ecbce5fba 100644 --- a/confgenerator/testdata/goldens/invalid-logging-processor_unsupported_type/golden/windows/error +++ b/confgenerator/testdata/goldens/invalid-logging-processor_unsupported_type/golden/windows/error @@ -1 +1 @@ -logging processor with type "unsupported_type" is not supported. Supported logging processor types: [apache_access, apache_error, cassandra_debug, cassandra_gc, cassandra_system, couchdb, elasticsearch_gc, elasticsearch_json, exclude_logs, flink, hadoop, hbase_system, iis_access, jetty_access, kafka, modify_fields, mysql_error, mysql_general, mysql_slow, nginx_access, nginx_error, oracledb_alert, oracledb_audit, parse_json, parse_multiline, parse_regex, postgresql_general, rabbitmq, redis, saphana, solr_system, tomcat_access, tomcat_system, varnish, vault_audit, wildfly_system, zookeeper_general]. \ No newline at end of file +logging processor with type "unsupported_type" is not supported. Supported logging processor types: [apache_access, apache_error, cassandra_debug, cassandra_gc, cassandra_system, couchdb, elasticsearch_gc, elasticsearch_json, exclude_logs, flink, hadoop, hbase_system, iis_access, jetty_access, kafka, modify_fields, mysql_error, mysql_general, mysql_slow, nginx_access, nginx_error, oracledb_alert, oracledb_audit, parse_json, parse_multiline, parse_regex, postgresql_general, rabbitmq, redis, saphana, solr_system, tomcat_access, tomcat_system, varnish, vault_audit, wildfly_system, zookeeper_general]. diff --git a/confgenerator/testdata/goldens/logging-receiver_mongodb/golden/linux-gpu/08a90af0796513028c1d99ecc6e59ef3.lua b/confgenerator/testdata/goldens/logging-receiver_mongodb/golden/linux-gpu/08a90af0796513028c1d99ecc6e59ef3.lua new file mode 100644 index 0000000000..208adf4ef0 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-receiver_mongodb/golden/linux-gpu/08a90af0796513028c1d99ecc6e59ef3.lua @@ -0,0 +1,14 @@ + +function process(tag, timestamp, record) +local __field_0 = (function() +return record["attr"] +end)(); +(function(value) +record["attr"] = value +end)(nil); +local v = __field_0; +(function(value) +record["attributes"] = value +end)(v) +return 2, timestamp, record +end diff --git a/confgenerator/testdata/goldens/logging-receiver_mongodb/golden/linux-gpu/0b7b2df85fd188c3e6583425ed73e5b3.lua b/confgenerator/testdata/goldens/logging-receiver_mongodb/golden/linux-gpu/0b7b2df85fd188c3e6583425ed73e5b3.lua new file mode 100644 index 0000000000..4a0b4a0fcd --- /dev/null +++ b/confgenerator/testdata/goldens/logging-receiver_mongodb/golden/linux-gpu/0b7b2df85fd188c3e6583425ed73e5b3.lua @@ -0,0 +1,17 @@ + +function parser_nest(tag, timestamp, record) + local nestedRecord = {} + local parseKey = "time" + for k, v in pairs(record) do + if k ~= parseKey then + nestedRecord[k] = v + end + end + + local result = {} + result[parseKey] = record[parseKey] + result["logging.googleapis.com/__tmp"] = nestedRecord + + return 2, timestamp, result +end + diff --git a/confgenerator/testdata/goldens/logging-receiver_mongodb/golden/linux-gpu/1dda8dfcf80ff2aac28fa0838a4d59b7.lua b/confgenerator/testdata/goldens/logging-receiver_mongodb/golden/linux-gpu/1dda8dfcf80ff2aac28fa0838a4d59b7.lua new file mode 100644 index 0000000000..dbad2430ef --- /dev/null +++ b/confgenerator/testdata/goldens/logging-receiver_mongodb/golden/linux-gpu/1dda8dfcf80ff2aac28fa0838a4d59b7.lua @@ -0,0 +1,14 @@ + +function process(tag, timestamp, record) +local __field_0 = (function() +return record["msg"] +end)(); +(function(value) +record["msg"] = value +end)(nil); +local v = __field_0; +(function(value) +record["message"] = value +end)(v) +return 2, timestamp, record +end diff --git a/confgenerator/testdata/goldens/logging-receiver_mongodb/golden/linux-gpu/6b37080b5cf99b41252dbda2dfb4bf7a.lua b/confgenerator/testdata/goldens/logging-receiver_mongodb/golden/linux-gpu/6b37080b5cf99b41252dbda2dfb4bf7a.lua new file mode 100644 index 0000000000..7f4e6b4300 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-receiver_mongodb/golden/linux-gpu/6b37080b5cf99b41252dbda2dfb4bf7a.lua @@ -0,0 +1,22 @@ + +function process(tag, timestamp, record) +local __field_0 = (function() +if record["attr"] == nil +then +return nil +end +return record["attr"]["message"] +end)(); +(function(value) +if record["attr"] == nil +then +record["attr"] = {} +end +record["attr"]["message"] = value +end)(nil); +local v = __field_0; +(function(value) +record["temp_attributes_message"] = value +end)(v) +return 2, timestamp, record +end diff --git a/confgenerator/testdata/goldens/logging-receiver_mongodb/golden/linux-gpu/e568b00a7a9bb9a1c872f142c49757e3.lua b/confgenerator/testdata/goldens/logging-receiver_mongodb/golden/linux-gpu/e568b00a7a9bb9a1c872f142c49757e3.lua new file mode 100644 index 0000000000..abd822c4ea --- /dev/null +++ b/confgenerator/testdata/goldens/logging-receiver_mongodb/golden/linux-gpu/e568b00a7a9bb9a1c872f142c49757e3.lua @@ -0,0 +1,14 @@ + +function process(tag, timestamp, record) +local __field_0 = (function() +return record["c"] +end)(); +(function(value) +record["c"] = value +end)(nil); +local v = __field_0; +(function(value) +record["component"] = value +end)(v) +return 2, timestamp, record +end diff --git a/confgenerator/testdata/goldens/logging-receiver_mongodb/golden/linux-gpu/e9e7adf2c9870f9e200c5f6638b391a6.lua b/confgenerator/testdata/goldens/logging-receiver_mongodb/golden/linux-gpu/e9e7adf2c9870f9e200c5f6638b391a6.lua new file mode 100644 index 0000000000..9c2613b727 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-receiver_mongodb/golden/linux-gpu/e9e7adf2c9870f9e200c5f6638b391a6.lua @@ -0,0 +1,14 @@ + +function process(tag, timestamp, record) +local __field_0 = (function() +return record["ctx"] +end)(); +(function(value) +record["ctx"] = value +end)(nil); +local v = __field_0; +(function(value) +record["context"] = value +end)(v) +return 2, timestamp, record +end diff --git a/confgenerator/testdata/goldens/logging-receiver_mongodb/golden/linux-gpu/eb8352830870355ccbd0d44ad0273fda.lua b/confgenerator/testdata/goldens/logging-receiver_mongodb/golden/linux-gpu/eb8352830870355ccbd0d44ad0273fda.lua new file mode 100644 index 0000000000..52221c0468 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-receiver_mongodb/golden/linux-gpu/eb8352830870355ccbd0d44ad0273fda.lua @@ -0,0 +1,22 @@ + +function process(tag, timestamp, record) +local __field_0 = (function() +if record["t"] == nil +then +return nil +end +return record["t"]["$date"] +end)(); +(function(value) +if record["t"] == nil +then +record["t"] = {} +end +record["t"]["$date"] = value +end)(nil); +local v = __field_0; +(function(value) +record["time"] = value +end)(v) +return 2, timestamp, record +end diff --git a/confgenerator/testdata/goldens/logging-receiver_mongodb/golden/linux-gpu/feature_tracking_otlp.json b/confgenerator/testdata/goldens/logging-receiver_mongodb/golden/linux-gpu/feature_tracking_otlp.json index 8fe864f885..61a43e8f1b 100644 --- a/confgenerator/testdata/goldens/logging-receiver_mongodb/golden/linux-gpu/feature_tracking_otlp.json +++ b/confgenerator/testdata/goldens/logging-receiver_mongodb/golden/linux-gpu/feature_tracking_otlp.json @@ -1 +1 @@ -{"resourceMetrics":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeMetrics":[{"scope":{},"metrics":[{"name":"agent.googleapis.com/agent/internal/ops/feature_tracking","gauge":{"dataPoints":[{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"service:pipelines"}},{"key":"key","value":{"stringValue":"default_pipeline_overridden"}},{"key":"value","value":{"stringValue":"false"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"metrics"}},{"key":"feature","value":{"stringValue":"service:pipelines"}},{"key":"key","value":{"stringValue":"default_pipeline_overridden"}},{"key":"value","value":{"stringValue":"false"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"global"}},{"key":"feature","value":{"stringValue":"default:self_log"}},{"key":"key","value":{"stringValue":"default_self_log_file_collection"}},{"key":"value","value":{"stringValue":"true"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"service:otel_logging"}},{"key":"key","value":{"stringValue":"otel_logging_supported_config"}},{"key":"value","value":{"stringValue":"false"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"receivers:mongodb"}},{"key":"key","value":{"stringValue":"[0].enabled"}},{"key":"value","value":{"stringValue":"true"}}],"asInt":"1"}]}}]}]}]} \ No newline at end of file +{"resourceMetrics":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeMetrics":[{"scope":{},"metrics":[{"name":"agent.googleapis.com/agent/internal/ops/feature_tracking","gauge":{"dataPoints":[{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"service:pipelines"}},{"key":"key","value":{"stringValue":"default_pipeline_overridden"}},{"key":"value","value":{"stringValue":"false"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"metrics"}},{"key":"feature","value":{"stringValue":"service:pipelines"}},{"key":"key","value":{"stringValue":"default_pipeline_overridden"}},{"key":"value","value":{"stringValue":"false"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"global"}},{"key":"feature","value":{"stringValue":"default:self_log"}},{"key":"key","value":{"stringValue":"default_self_log_file_collection"}},{"key":"value","value":{"stringValue":"true"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"service:otel_logging"}},{"key":"key","value":{"stringValue":"otel_logging_supported_config"}},{"key":"value","value":{"stringValue":"false"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"receivers:mongodb"}},{"key":"key","value":{"stringValue":"[0].enabled"}},{"key":"value","value":{"stringValue":"true"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"receivers:mongodb"}},{"key":"key","value":{"stringValue":"[0].include_paths.__length"}},{"key":"value","value":{"stringValue":"1"}}],"asInt":"1"}]}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-receiver_mongodb/golden/linux-gpu/features.yaml b/confgenerator/testdata/goldens/logging-receiver_mongodb/golden/linux-gpu/features.yaml index 8216fe84f1..24cb696bd0 100644 --- a/confgenerator/testdata/goldens/logging-receiver_mongodb/golden/linux-gpu/features.yaml +++ b/confgenerator/testdata/goldens/logging-receiver_mongodb/golden/linux-gpu/features.yaml @@ -18,3 +18,7 @@ feature: receivers:mongodb key: "[0].enabled" value: "true" +- module: logging + feature: receivers:mongodb + key: "[0].include_paths.__length" + value: "1" diff --git a/confgenerator/testdata/goldens/logging-receiver_mongodb/golden/linux-gpu/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-receiver_mongodb/golden/linux-gpu/fluent_bit_main.conf index 80bbf20465..9b4175b9ea 100644 --- a/confgenerator/testdata/goldens/logging-receiver_mongodb/golden/linux-gpu/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-receiver_mongodb/golden/linux-gpu/fluent_bit_main.conf @@ -101,7 +101,7 @@ Match mongodb.mongodb Name parser Reserve_Data True - Parser mongodb.mongodb.mongodb + Parser mongodb.mongodb.0 [FILTER] Match mongodb.mongodb @@ -110,29 +110,28 @@ script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua [FILTER] - Add_prefix temp_ts_ - Match mongodb.mongodb - Name nest - Nested_under t - Operation lift + Match mongodb.mongodb + Name lua + call process + script eb8352830870355ccbd0d44ad0273fda.lua [FILTER] - Hard_rename temp_ts_$date time - Match mongodb.mongodb - Name modify + Match mongodb.mongodb + Name modify + Remove t [FILTER] Match mongodb.mongodb Name lua call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua + script 0b7b2df85fd188c3e6583425ed73e5b3.lua [FILTER] - Key_Name message + Key_Name time Match mongodb.mongodb Name parser - Parser mongodb.mongodb.mongodb Reserve_Data True + Parser mongodb.mongodb.3 [FILTER] Match mongodb.mongodb @@ -142,15 +141,9 @@ [FILTER] Match mongodb.mongodb - Name modify - Remove time - -[FILTER] - Add_prefix temp_attributes_ - Match mongodb.mongodb - Name nest - Nested_under attr - Operation lift + Name lua + call process + script 6b37080b5cf99b41252dbda2dfb4bf7a.lua [FILTER] Hard_rename temp_attributes_message msg @@ -158,27 +151,28 @@ Name modify [FILTER] - Match mongodb.mongodb - Name nest - Nest_under attributes - Operation nest - Remove_prefix temp_attributes_ - Wildcard temp_attributes_* + Match mongodb.mongodb + Name lua + call process + script e568b00a7a9bb9a1c872f142c49757e3.lua [FILTER] Match mongodb.mongodb - Name modify - Rename c component + Name lua + call process + script e9e7adf2c9870f9e200c5f6638b391a6.lua [FILTER] Match mongodb.mongodb - Name modify - Rename ctx context + Name lua + call process + script 1dda8dfcf80ff2aac28fa0838a4d59b7.lua [FILTER] Match mongodb.mongodb - Name modify - Rename msg message + Name lua + call process + script 08a90af0796513028c1d99ecc6e59ef3.lua [FILTER] Match mongodb.mongodb @@ -190,8 +184,8 @@ Key_Name message Match mongodb.mongodb Name parser - Parser mongodb.mongodb_regex.mongodb Reserve_Data True + Parser mongodb.mongodb.10 [FILTER] Match mongodb.mongodb diff --git a/confgenerator/testdata/goldens/logging-receiver_mongodb/golden/linux-gpu/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-receiver_mongodb/golden/linux-gpu/fluent_bit_parser.conf index 83a65ce1bc..309943611d 100644 --- a/confgenerator/testdata/goldens/logging-receiver_mongodb/golden/linux-gpu/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-receiver_mongodb/golden/linux-gpu/fluent_bit_parser.conf @@ -1,14 +1,18 @@ [PARSER] - Format json - Name mongodb.mongodb.mongodb + Format json + Name mongodb.mongodb.0 + Types id:integer message:string + +[PARSER] + Format regex + Name mongodb.mongodb.3 + Regex ^(?