conf: Convert existing parser*.conf files to yaml, add converter, add tests, drop old

conf/parser_custom.conf

@@ -1,14 +1,14 @@
- [PARSER]
+[PARSER]
       # https://rubular.com/r/6ZCuwV4Xa7nfA3
       Name                rabbitmq
       Format              regex
       Regex               (?<date>[^ ]+)\s(?<time>[^ ]+)\s\[(?<log_level>[^ \]]*)\]\s(?<PID>[^ ]*)\s(?<msg>((([a-zA-Z]*\s+)+[^ ]*)+)+)
- [PARSER]
+[PARSER]
       # https://rubular.com/r/jWfJIOMKr2LgcO
       Name                neo4j
       Format              regex
       Regex               (?<date>[^ ]*) (?<time>[^ ]*) (?<log_level>[^ ]*)\s(?<msg>([^ ]*\s+[^ ]*)+)
- [PARSER]
+[PARSER]
       # https://rubular.com/r/U8VbByp0oRPLU6
       Name                external-dns
       Format              regex

conf/parser_custom.yaml

@@ -0,0 +1,15 @@
+parsers:
+  - name: rabbitmq
+    # https://rubular.com/r/6ZCuwV4Xa7nfA3
+    format: regex
+    regex: (?<date>[^ ]+)\s(?<time>[^ ]+)\s\[(?<log_level>[^ \]]*)\]\s(?<PID>[^ ]*)\s(?<msg>((([a-zA-Z]*\s+)+[^ ]*)+)+)
+
+  - name: neo4j
+    # https://rubular.com/r/jWfJIOMKr2LgcO
+    format: regex
+    regex: (?<date>[^ ]*) (?<time>[^ ]*) (?<log_level>[^ ]*)\s(?<msg>([^ ]*\s+[^ ]*)+)
+
+  - name: external-dns
+    # https://rubular.com/r/U8VbByp0oRPLU6
+    format: regex
+    regex: ([^ ])\"(?<time>[^ ]+)\"\s([^ ]+)\=(?<log_level>[.+a-zA-Z]+)\s([^ ]+)\"(?<msg>([^ ]*\s+[^ ]*\s[a-zA-Z0-9]*)+)

conf/parsers.conf

@@ -82,6 +82,7 @@
     Time_Keep   On
 
 [PARSER]
+    # https://rubular.com/r/y5tae3pzf6sOHW
     Name        syslog-rfc3164
     Format      regex
     Regex       /^\<(?<pri>[0-9]+)\>(?<time>[^ ]* {1,2}[^ ]* [^ ]*) (?<host>[^ ]*) (?<ident>[a-zA-Z0-9_\/\.\-]*)(?:\[(?<pid>[0-9]+)\])?(?:[^\:]*\:)? *(?<message>.*)$/
@@ -116,7 +117,7 @@
     Time_Key start_time
 
 [PARSER]
-    # http://rubular.com/r/tjUt3Awgg4
+    # https://rubular.com/r/tjUt3Awgg4
     Name cri
     Format regex
     Regex ^(?<time>[^ ]+) (?<stream>stdout|stderr) (?<logtag>[^ ]*) (?<message>.*)$

conf/parsers.yaml

@@ -0,0 +1,122 @@
+parsers:
+  - name: apache
+    format: regex
+    regex: '^(?<host>[^ ]*) [^ ]* (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^\"]*?)(?: +\S*)?)?" (?<code>[^ ]*) (?<size>[^ ]*)(?: "(?<referer>[^\"]*)" "(?<agent>[^\"]*)")?$'
+    time_key: time
+    time_format: '%d/%b/%Y:%H:%M:%S %z'
+
+  - name: apache2
+    format: regex
+    regex: '^(?<host>[^ ]*) [^ ]* (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^ ]*) +\S*)?" (?<code>[^ ]*) (?<size>[^ ]*)(?: "(?<referer>[^\"]*)" "(?<agent>.*)")?$'
+    time_key: time
+    time_format: '%d/%b/%Y:%H:%M:%S %z'
+
+  - name: apache_error
+    format: regex
+    regex: '^\[[^ ]* (?<time>[^\]]*)\] \[(?<level>[^\]]*)\](?: \[pid (?<pid>[^\]]*)\])?( \[client (?<client>[^\]]*)\])? (?<message>.*)$'
+
+  - name: nginx
+    format: regex
+    regex: '^(?<remote>[^ ]*) (?<host>[^ ]*) (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^\"]*?)(?: +\S*)?)?" (?<code>[^ ]*) (?<size>[^ ]*)(?: "(?<referer>[^\"]*)" "(?<agent>[^\"]*)")'
+    time_key: time
+    time_format: '%d/%b/%Y:%H:%M:%S %z'
+
+  - name: k8s-nginx-ingress
+    # https://rubular.com/r/IhIbCAIs7ImOkc
+    format: regex
+    regex: '^(?<host>[^ ]*) - (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^\"]*?)(?: +\S*)?)?" (?<code>[^ ]*) (?<size>[^ ]*) "(?<referer>[^\"]*)" "(?<agent>[^\"]*)" (?<request_length>[^ ]*) (?<request_time>[^ ]*) \[(?<proxy_upstream_name>[^ ]*)\] (\[(?<proxy_alternative_upstream_name>[^ ]*)\] )?(?<upstream_addr>[^ ]*) (?<upstream_response_length>[^ ]*) (?<upstream_response_time>[^ ]*) (?<upstream_status>[^ ]*) (?<reg_id>[^ ]*).*$'
+    time_key: time
+    time_format: '%d/%b/%Y:%H:%M:%S %z'
+
+  - name: json
+    format: json
+    time_key: time
+    time_format: '%d/%b/%Y:%H:%M:%S %z'
+
+  - name: logfmt
+    format: logfmt
+
+  - name: docker
+    format: json
+    time_key: time
+    time_format: '%Y-%m-%dT%H:%M:%S.%L'
+    time_keep: On
+    # --
+    # Since Fluent Bit v1.2, if you are parsing Docker logs and using
+    # the Kubernetes filter, it's not longer required to decode the
+    # 'log' key.
+    #
+    # Command      |  Decoder | Field | Optional Action
+    # =============|==================|=================
+    #Decode_Field_As    json     log
+    #
+
+  - name: docker-daemon
+    format: regex
+    regex: time="(?<time>[^ ]*)" level=(?<level>[^ ]*) msg="(?<msg>[^ ].*)"
+    time_key: time
+    time_format: '%Y-%m-%dT%H:%M:%S.%L'
+    time_keep: On
+
+  - name: syslog-rfc5424
+    format: regex
+    regex: ^\<(?<pri>[0-9]{1,5})\>1 (?<time>[^ ]+) (?<host>[^ ]+) (?<ident>[^ ]+) (?<pid>[-0-9]+) (?<msgid>[^ ]+) (?<extradata>(\[(.*?)\]|-)) (?<message>.+)$
+    time_key: time
+    time_format: '%Y-%m-%dT%H:%M:%S.%L%z'
+    time_keep: On
+
+  - name: syslog-rfc3164-local
+    format: regex
+    regex: '^\<(?<pri>[0-9]+)\>(?<time>[^ ]* {1,2}[^ ]* [^ ]*) (?<ident>[a-zA-Z0-9_\/\.\-]*)(?:\[(?<pid>[0-9]+)\])?(?:[^\:]*\:)? *(?<message>.*)$'
+    time_key: time
+    time_format: '%b %d %H:%M:%S'
+    time_keep: On
+
+  - name: syslog-rfc3164
+    format: regex
+    regex: '/^\<(?<pri>[0-9]+)\>(?<time>[^ ]* {1,2}[^ ]* [^ ]*) (?<host>[^ ]*) (?<ident>[a-zA-Z0-9_\/\.\-]*)(?:\[(?<pid>[0-9]+)\])?(?:[^\:]*\:)? *(?<message>.*)$/'
+    time_key: time
+    time_format: '%b %d %H:%M:%S'
+    time_keep: On
+
+  - name: mongodb
+    format: regex
+    regex: '^(?<time>[^ ]*)\s+(?<severity>\w)\s+(?<component>[^ ]+)\s+\[(?<context>[^\]]+)]\s+(?<message>.*?) *(?<ms>(\d+))?(:?ms)?$'
+    time_format: '%Y-%m-%dT%H:%M:%S.%L'
+    time_keep: On
+    time_key: time
+
+  - name: envoy
+    # https://rubular.com/r/0VZmcYcLWMGAp1
+    format: regex
+    regex: '^\[(?<start_time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^\"]*?)(?: +\S*)?)? (?<protocol>\S+)" (?<code>[^ ]*) (?<response_flags>[^ ]*) (?<bytes_received>[^ ]*) (?<bytes_sent>[^ ]*) (?<duration>[^ ]*) (?<x_envoy_upstream_service_time>[^ ]*) "(?<x_forwarded_for>[^ ]*)" "(?<user_agent>[^\"]*)" "(?<request_id>[^\"]*)" "(?<authority>[^ ]*)" "(?<upstream_host>[^ ]*)"'
+    time_format: '%Y-%m-%dT%H:%M:%S.%L%z'
+    time_keep: On
+    time_key: start_time
+
+  - name: istio-envoy-proxy
+    # https://rubular.com/r/hbsTIxFFMozLmh
+    format: regex
+    regex: '^\[(?<start_time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^\"]*?)(?: +\S*)?)? (?<protocol>\S+)" (?<response_code>[^ ]*) (?<response_flags>[^ ]*) (?<response_code_details>[^ ]*) (?<connection_termination_details>[^ ]*) "(?<upstream_transport_failure_reason>[^ ]*)" (?<bytes_received>[^ ]*) (?<bytes_sent>[^ ]*) (?<duration>[^ ]*) (?<x_envoy_upstream_service_time>[^ ]*) "(?<x_forwarded_for>[^ ]*)" "(?<user_agent>[^\"]*)" "(?<x_request_id>[^\"]*)" "(?<authority>[^ ]*)" "(?<upstream_host>[^ ]*)" (?<upstream_cluster>[^ ]*) (?<upstream_local_address>[^ ]*) (?<downstream_local_address>[^ ]*) (?<downstream_remote_address>[^ ]*) (?<requested_server_name>[^ ]*) (?<route_name>[^  ]*)$'
+    time_format: '%Y-%m-%dT%H:%M:%S.%L%z'
+    time_keep: On
+    time_key: start_time
+
+  - name: cri
+    # https://rubular.com/r/tjUt3Awgg4
+    format: regex
+    regex: ^(?<time>[^ ]+) (?<stream>stdout|stderr) (?<logtag>[^ ]*) (?<message>.*)$
+    time_key: time
+    time_format: '%Y-%m-%dT%H:%M:%S.%L%z'
+    time_keep: On
+
+  - name: kube-custom
+    format: regex
+    regex: '(?<tag>[^.]+)?\.?(?<pod_name>[a-z0-9](?:[-a-z0-9]*[a-z0-9])?(?:\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*)_(?<namespace_name>[^_]+)_(?<container_name>.+)-(?<docker_id>[a-z0-9]{64})\.log$'
+
+  - name: kmsg-netfilter-log
+    # Examples: TCP: https://rubular.com/r/Q8YY6fHqlqwGI0  UDP: https://rubular.com/r/B0ID69H9FvN0tp
+    format: regex
+    regex: '^\<(?<pri>[0-9]{1,5})\>1 (?<time>[^ ]+) (?<host>[^ ]+) kernel - - - \[[0-9\.]*\] (?<logprefix>[^ ]*)\s?IN=(?<in>[^ ]*) OUT=(?<out>[^ ]*) MAC=(?<macsrc>[0-9a-f]{2}:[0-9a-f]{2}:[0-9a-f]{2}:[0-9a-f]{2}:[0-9a-f]{2}:[0-9a-f]{2}):(?<macdst>[0-9a-f]{2}:[0-9a-f]{2}:[0-9a-f]{2}:[0-9a-f]{2}:[0-9a-f]{2}:[0-9a-f]{2}):(?<ethtype>[0-9a-f]{2}:[0-9a-f]{2}) SRC=(?<saddr>[^ ]*) DST=(?<daddr>[^ ]*) LEN=(?<len>[^ ]*) TOS=(?<tos>[^ ]*) PREC=(?<prec>[^ ]*) TTL=(?<ttl>[^ ]*) ID=(?<id>[^ ]*) (D*F*)\s*PROTO=(?<proto>[^ ]*)\s?((SPT=)?(?<sport>[0-9]*))\s?((DPT=)?(?<dport>[0-9]*))\s?((LEN=)?(?<protolen>[0-9]*))\s?((WINDOW=)?(?<window>[0-9]*))\s?((RES=)?(?<res>0?x?[0-9]*))\s?(?<flag>[^ ]*)\s?((URGP=)?(?<urgp>[0-9]*))'
+    time_key: time
+    time_format: '%Y-%m-%dT%H:%M:%S.%L%z'

conf/parsers_ambassador.yaml

@@ -0,0 +1,7 @@
+# Ambassador - open source Kubernetes-native API gateway for microservices built on the Envoy Proxy https://www.getambassador.io
+#
+
+parsers:
+  - name: ambassador
+    format: regex
+    regex: '^(?<type>\S+) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>(?:[^\"]|\\.)*?)(?: +\S*)?) (?<protocol>\S+)?" (?<response_code>\S+) (?<response_flags>\S+) (?<bytes_received>\S+) (?<bytes_sent>\S+) (?<duration>\S+) (?<x_envoy_upstream_service_time>\S+) "(?<x_forwarded_for>[^\"]*)" "(?<user_agent>[^\"]*)" "(?<x_request_id>[^\"]*)" "(?<authority>[^\"]*)" "(?<upstream_host>[^\"]*)"'

conf/parsers_cinder.conf

@@ -1,6 +1,6 @@
 
 [PARSER]
-    # http://rubular.com/r/IvZVElTgNl
+    # https://rubular.com/r/IvZVElTgNl
     Name ceph
     Format regex
     Regex ^(?<log_time>[^ ][-.\d\+:T]+[ ]*[.:\d]*)\s+(?<message>.*)$

conf/parsers_cinder.yaml

@@ -0,0 +1,8 @@
+parsers:
+  - name: ceph
+    # https://rubular.com/r/IvZVElTgNl
+    format: regex
+    regex: '^(?<log_time>[^ ][-.\d\+:T]+[ ]*[.:\d]*)\s+(?<message>.*)$'
+    time_format: '%Y-%m-%d %H:%M:%S.%L'
+    time_keep: Off
+    time_key: log_time

conf/parsers_extra.conf

@@ -1,7 +1,7 @@
 # Extra set of common parsers
 
 [PARSER]
-    # http://rubular.com/r/cCVd1HLCAO
+    # https://rubular.com/r/cCVd1HLCAO
     Name crowbar
     Format regex
     Regex ^.*\[(?<log_time>[^ ][-.\d\+:]+T[:\d]*)([^\]])*?\]\s+?(?<severity>[^ ]\w+)([\s-]*):?\s+(?<message>.*)
@@ -10,7 +10,7 @@
     Time_Key log_time
 
 [PARSER]
-    # http://rubular.com/r/frDgnElXW9 
+    # https://rubular.com/r/frDgnElXW9 
     Name chefclient
     Format regex
     Regex ^\[(?<log_time>[^ ][-.\d\+:]+T[:\d]*)([^\]])*?\]\s+(?<severity>[^ ]\w+):\s+(?<message>.*)$

conf/parsers_extra.yaml

@@ -0,0 +1,159 @@
+# Extra set of common parsers
+#
+
+parsers:
+  - name: crowbar
+    # https://rubular.com/r/cCVd1HLCAO
+    format: regex
+    regex: '^.*\[(?<log_time>[^ ][-.\d\+:]+T[:\d]*)([^\]])*?\]\s+?(?<severity>[^ ]\w+)([\s-]*):?\s+(?<message>.*)'
+    time_format: '%Y-%m-%dT%H:%M:%S'
+    time_keep: Off
+    time_key: log_time
+
+  - name: chefclient
+    # https://rubular.com/r/frDgnElXW9
+    format: regex
+    regex: '^\[(?<log_time>[^ ][-.\d\+:]+T[:\d]*)([^\]])*?\]\s+(?<severity>[^ ]\w+):\s+(?<message>.*)$'
+    time_format: '%Y-%m-%dT%H:%M:%S'
+    time_keep: Off
+    time_key: log_time
+
+  - name: mysql_error
+    format: regex
+    #regex: '^(?<log_time>[^ +][ -:0-9TZ]+|[[:upper:]][[:lower:]]{2})(\+\d+:\d+[TZ]*){0,1}\s*(?<myid>[^ ]\d+)\s+\[(?<severity>[^ ]\w+)\](\s+(?<subsystem>[^ ]\w+):){0,1}\s+(?<message>.*)$'
+    regex: '^(?<log_time>[^ +][-\d]+[\ T]*[:\dZ]+)\s*(?<myid>[^ ]\d+)\s+\[(?<severity>[^ ]\w+)\](\s+(?<subsystem>[^ ]\w+):){0,1}\s+(?<message>.*)$'
+    time_format: '%Y-%m-%d %H:%M:%S'
+    time_keep: Off
+    time_key: log_time
+
+  - name: mysql_slow
+    format: regex
+    regex: '^# User\@Host:\s+(?<user>[^\@][\w\[\]]+)[@\s]+(?<dbhost>[^ ][-.\w]+)\s+(\[(?<dbhost_address>[.\d]+)\]){0,1}\s+(?<message>.*)$'
+
+  - name: pacemaker
+    format: regex
+    regex: '^\s*(?<log_time>[^ ]* {1,2}[^ ]* [^ ]*) \[(?<pid>\d+)\] (?<node>[\-\w]*)\s*(?<component>\w*):\s+(?<severity>\w+):\s+(?<message>.*)$'
+    #time_format: '%Y-%m-%dT%H:%M:%S'
+    time_format: '%b %d %H:%M:%S'
+    time_keep: Off
+    time_key: log_time
+    #types: 'pid:integer'
+    #
+
+  - name: rabbitmq
+    format: regex
+    regex: '^=(?<severity>[^ ]\w+)\s+REPORT[=\s]*(?<log_time>[^ =][-:.\d\w]+)[\s=]+(?<message>.*)$'
+    time_format: '%d-%b-%Y::%H:%M:%S'
+    time_keep: Off
+    time_key: log_time
+
+  - name: http_statement
+    format: regex
+    regex: '^.*((?<req_method>GET|POST|PUT|DELETE|CONNECT|OPTIONS|HEAD[^ ]\w+)\s*(?<req_path>[^ ][-._?=%&\/[:alnum:]]*)\s*(?<req_protocol>[^ ][.\/\dHTFSP]+){0,1})([''"\s]*){0,1}((\s*status:\s*(?<req_status>[^ ]\d+)){0,1}(\s*len:\ (?<req_len>[^ ]\d+)){0,1}(\s*time:\s*(?<req_log_time>[^ ][.\d]+)){0,1}(\s*microversion:\s*(?<req_mver>[^ ][.\d]+)){0,1}){0,1}$'
+
+  - name: universal
+    format: regex
+    regex: ^(?<message>.*)$
+
+  - name: uuid
+    format: regex
+    regex: (?<uuid>[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-5][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12})
+    #UUID v1 :
+    #/^[0-9A-F]{8}-[0-9A-F]{4}-[1][0-9A-F]{3}-[89AB][0-9A-F]{3}-[0-9A-F]{12}$/i
+    #UUID v2 :
+    #/^[0-9A-F]{8}-[0-9A-F]{4}-[2][0-9A-F]{3}-[89AB][0-9A-F]{3}-[0-9A-F]{12}$/i
+    #UUID v3 :
+    #/^[0-9A-F]{8}-[0-9A-F]{4}-[3][0-9A-F]{3}-[89AB][0-9A-F]{3}-[0-9A-F]{12}$/i
+    #UUID v4 :
+    #/^[0-9A-F]{8}-[0-9A-F]{4}-[4][0-9A-F]{3}-[89AB][0-9A-F]{3}-[0-9A-F]{12}$/i
+    #UUID v5 :
+    #/^[0-9A-F]{8}-[0-9A-F]{4}-[5][0-9A-F]{3}-[89AB][0-9A-F]{3}-[0-9A-F]{12}$/i
+    #
+    # Parse IP Tables rules - this one regex should capture pretty much any IP Tables rule and split it into the various fields
+
+  - name: iptables
+    format: regex
+    regex: '\[(?<rule_chain>\w*)-(?<rule_name>\w*)-(?<accept_or_drop>\w*)\]IN=(?<in_interface>[\w.]+)? OUT=(?<out_interface>[\w.]+)? MAC=(?<mac_address>[\w:]+)? SRC=(?<source>(?:[0-9]{1,3}\.){3}[0-9]{1,3}) DST=(?<dest>(?:[0-9]{1,3}\.){3}[0-9]{1,3}) LEN=(?<pkt_len>\d+) TOS=(?<pkt_tos>[\w\d]+) PREC=(?<pkt_prec>[\w\d]+) TTL=(?<pkt_ttl>\d+) ID=(?<pkt_id>\d+)\s?(?<pkg_frg>[A-Z\s].?)\s?PROTO=(?<protocol>[\w\d]+) (SPT=(?<source_port>.*) DPT=(?<dest_port>.*) (LEN=(?<proto_pkt_len>\w+)?)?(WINDOW=(?<proto_window_size>\d+) RES=(?<pkt_res>\w+)? (?<pkt_type>\w+)\s((?<pkt_flag>\w+)?)\s?URGP=(?<pkg_urgency>\d))? )?(TYPE=(?<pkt_icmp_type>\d+) CODE=(?<pkt_icmp_code>\d+) ID=(?<pkt_icmp_id>\d+) SEQ=(?<pkt_icmp_seq>\d+) )?$'
+    types: 'source_port:integer,dest_port:integer,pkt_ttl:integer,pkt_tos:integer,pkt_len:integer'
+
+  - name: couchbase_json_log_nanoseconds
+    # Various parsers for Couchbase Server logs
+    #
+    format: json
+    time_key: timestamp
+    time_format: '%Y-%m-%dT%H:%M:%S.%L'
+    time_keep: On
+    # Do not remove the time field from the output we ship
+    #
+
+  - name: couchbase_rebalance_report
+    format: json
+    time_key: timestamp
+    time_format: '%Y-%m-%dT%H:%M:%SZ'
+    time_keep: On
+
+  - name: couchbase_simple_log
+    # The level may have optional brackets around it
+    format: regex
+    regex: '^(?<timestamp>\d+-\d+-\d+T\d+:\d+:\d+\.\d+(\+|-)\d+:\d+)\s+\[(?<level>\w+)\](?<message>.*)$'
+    time_key: timestamp
+    time_format: '%Y-%m-%dT%H:%M:%S.%L%z'
+    time_keep: On
+
+  - name: couchbase_simple_log_space_separated
+    format: regex
+    regex: '^(?<timestamp>\d+-\d+-\d+T\d+:\d+:\d+\.\d+(\+|-)\d+:\d+)\s+(?<level>\w+)\s+(?<message>.*)$'
+    time_key: timestamp
+    time_format: '%Y-%m-%dT%H:%M:%S.%L%z'
+    time_keep: On
+
+  - name: couchbase_simple_log_utc
+    # Slight change in time format to use Z at end instead of offset:
+    # 2021-03-09T17:32:02.136Z INFO ...
+    # https://rubular.com/r/EpG3M1dHb5AnTC
+    format: regex
+    regex: '^(?<timestamp>\d+-\d+-\d+T\d+:\d+:\d+\.\d+Z)\s+(?<level>\w+)(?<message>.*)$'
+    time_key: timestamp
+    time_format: '%Y-%m-%dT%H:%M:%S.%LZ'
+    time_keep: On
+
+  - name: couchbase_simple_log_mixed
+    # Cope with two different log formats, e.g.:
+    # 2021/03/09 17:32:15 cbauth: ...
+    # 2021-03-09T17:32:15.303+00:00 [INFO] ...
+    # https://rubular.com/r/XUt7xQqEJnrF2M
+    format: regex
+    regex: '^(?<timestamp>\d+(-|/)\d+(-|/)\d+(T|\s+)\d+:\d+:\d+(\.\d+(\+|-)\d+:\d+|))\s+((\[)?(?<level>\w+)(\]|:))(?<message>.*)$'
+    time_key: timestamp
+    time_keep: On
+    # We cannot parse the time as different formats directly, it could be done downstream and/or left as current time
+    #
+
+  - name: couchbase_erlang_multiline
+    format: regex
+    # For some reason this cannot parse an ending close bracket ] followed by a new line immediately
+    #regex: '\[(?<logger>\w+):(?<level>\w+),(?<timestamp>\d+-\d+-\d+T\d+:\d+:\d+.\d+Z),.*\](?<message>.*)$'
+    regex: '\[(?<logger>\w+):(?<level>\w+),(?<timestamp>\d+-\d+-\d+T\d+:\d+:\d+.\d+Z),(?<message>.*)$'
+    time_key: timestamp
+    time_format: '%Y-%m-%dT%H:%M:%S.%L'
+    time_keep: On
+
+  - name: couchbase_java_multiline
+    # 2021-03-09T17:32:25.339+00:00 INFO CBAS.bootstrap.AnalyticsNCApplication [main] ...
+    # https://rubular.com/r/9jh1oKtXBN5GEV
+    # Can include an exception stack trace or a thread dump as well but ignoring these for now
+    format: regex
+    regex: '^(?<timestamp>\d+-\d+-\d+T\d+:\d+:\d+\.\d+(\+|-)\d+:\d+)\s+(?<level>\w+)\s+(?<class>.*)\s+\[(?<thread>.*)\]\s+(?<message>.*)$'
+    time_key: timestamp
+    time_format: '%Y-%m-%dT%H:%M:%S.%L%z'
+    time_keep: On
+
+  - name: couchbase_http
+    # A slight modification of the usual Apache/Apache2 parsers
+    format: regex
+    regex: '^(?<host>[^ ]*) [^ ]* (?<user>[^ ]*) \[(?<timestamp>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^ ]*) +\S*)?" (?<code>[^ ]*) (?<size>[^ ]*) - (?<client>.*)$'
+    time_key: timestamp
+    time_format: '%d/%b/%Y:%H:%M:%S %z'
+    time_keep: On
+
+# End of Couchbase Server parsers

conf/parsers_java.yaml

@@ -0,0 +1,6 @@
+parsers:
+  - name: java_multiline
+    format: regex
+    regex: '/^(?<time>\d{4}-\d{1,2}-\d{1,2} \d{1,2}:\d{1,2}:\d{1,2}) \[(?<thread>.*)\] (?<level>[^\s]+)(?<message>.*)/'
+    time_key: time
+    time_format: '%Y-%m-%d %H:%M:%S'

conf/parsers_kafka.yaml

@@ -0,0 +1,14 @@
+parsers:
+  - name: confluent-schema-registry
+    # Confluent Schema Registry 7.1.1 default format
+    format: regex
+    time_key: time
+    time_format: '%Y-%m-%d %H:%M:%S,%L'
+    regex: '^\[(?<time>\d{4}-\d{1,2}-\d{1,2} \d{2}:\d{2}:\d{2},\d{1,4})] (?<level>[A-Z]{1,8}) (?<src>[\d.]+) - - \[(?<date>.*)] \"(?<method>[A-Z]+) (?<path>\/[\S]+|\/) (?<http_version>HTTP\/[\d]\.[\d])\" (?<code>[\d]{1,3}) (?<size>[\d]+) \"-\" \"(?<agent>.*)\" (?<extra_info>.*)'
+
+  - name: confluent-schema-registry-prefixed
+    # Confluent Schema Registry 7.1.1 default format - with _sr_ prefix to allow `nest`
+    format: regex
+    time_key: _sr_time
+    time_format: '%Y-%m-%d %H:%M:%S,%L'
+    regex: '^\[(?<_sr_time>\d{4}-\d{1,2}-\d{1,2} \d{2}:\d{2}:\d{2},\d{1,4})] (?<_sr_level>[A-Z]{1,8}) (?<_sr_src>[\d.]+) - - \[(?<_sr_date>.*)] \"(?<_sr_method>[A-Z]+) (?<_sr_path>\/[\S]+|\/) (?<_sr_http_version>HTTP\/[\d]\.[\d])\" (?<_sr_code>[\d]{1,3}) (?<_sr_size>[\d]+) \"-\" \"(?<_sr_agent>.*)\" (?<_sr_extra_info>.*)'