WIP on input filtering using tcp and custom parsers

fluentbit.conf

@@ -1,12 +1,13 @@
 [SERVICE]
     flush        1
     log_level    trace
+    parsers_file parsers.conf
     parsers_file /home/vcap/deps/0/apt/etc/fluent-bit/parsers.conf
-    Plugins_File plugins.conf
+    plugins_File plugins.conf
 
 [INPUT]
     name      dummy
-    dummy     {"message":"Using newrelic output plugin", "temp": "0.74", "extra": "false"}
+    dummy     {"message":"A simple test message", "temp": "0.74", "extra": "false"}
     samples   1
 
 
@@ -21,9 +22,49 @@
 #      total_file_size 50M
 #      upload_timeout 10m
 
+[INPUT]
+    name tcp
+    port ${PORT}
+    format none
+
+# These next two filter stanzas work, in that fluentbit approves and produces regex-modified output.
+[FILTER]
+    name multiline
+    match tcp.*
+    multiline.key_content log
+    multiline.parser combine-http-post
+
+[FILTER]
+    name parser
+    match tcp.*
+    key_name log
+    parser cg-http-post-parser
+
+# [FILTER]
+#     name parser
+#     match *
+#     key_name log
+#     parser post-with-syslog-rfc5424
+
+# [INPUT]
+#     name syslog
+#     port ${PORT}
+#     mode tcp
+#     parser syslog-rfc5424
+#     source_address_key log_source
+
+# This works but is commented out while testing other things.
+# [OUTPUT]
+#     Name newrelic
+#     Match *
+#     licenseKey ${NEW_RELIC_LICENSE_KEY}
+#     endpoint ${NEW_RELIC_LOGS_ENDPOINT}
+
+# [INPUT]
+#     Name http
+#     port ${PORT}
+
 
 [OUTPUT]
-    Name newrelic
-    Match *
-    licenseKey ${NEW_RELIC_LICENSE_KEY}
-    endpoint ${NEW_RELIC_LOGS_ENDPOINT}
+    name stdout
+    match *

parsers.conf

@@ -0,0 +1,36 @@
+[MULTILINE_PARSER]
+    # Combine an HTTP POST into a single message
+    name combine-http-post
+    type regex
+    #
+    # Regex rules for multiline parsing
+    # ---------------------------------
+    #
+    # configuration hints:
+    #
+    #  - first state always has the name: start_state
+    #  - every field in the rule must be inside double quotes
+    #
+    # rules |   state name  | regex pattern                  | next state
+    # ------|---------------|--------------------------------------------
+    rule      "start_state"   "/^POST \S+ HTTP\/1\.1\r/"       "cont"
+    rule      "cont"          "/.*/"                         "cont"
+
+
+[PARSER]
+    Name cg-http-post-parser
+    Format regex
+    # Regex /<\d+>\d+\s(?<time>\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d+\S+)\s.+\s-\s(?<message>.*)/m
+    Regex /\<(?<pri>[0-9]{1,5})\>1 (?<time>\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d+\S+)\s.+\s-\s(?<message>.*)/m
+    Time_Key time
+    Time_Format %Y-%m-%dT%H:%M:%S.%L%z
+    Time_Keep On
+
+[PARSER]
+    Name post-with-syslog-rfc5424
+    Format regex
+    # Regex       /.*\<(?<pri>[0-9]{1,5})\>1 (?<time>[^ ]+) (?<host>[^ ]+) (?<ident>[^ ]+) (?<pid>[-0-9]+) (?<msgid>[^ ]+) (?<extradata>(\[(.*?)\]|-)) (?<message>.+).*/m
+    Regex /\<(?<pri>[0-9]{1,5})\>1 (?<time>[^ ]+) (?<host>[^ ]+) (?<ident>[^ ]+) (?<pid>[-a0-9]+) (?<msgid>[^ ]+) (?<extradata>(\[(.*?)\]|-)) (?<message>.*)/m
+    # Time_Key time
+    # Time_Format %Y-%m-%dT%H:%M:%S.%L%z
+    # Time_Keep On

start.sh

@@ -1,3 +1,3 @@
 #!/bin/sh
 
-/home/vcap/deps/0/apt/opt/fluent-bit/bin/fluent-bit -P ${PORT} -c fluentbit.conf
+/home/vcap/deps/0/apt/opt/fluent-bit/bin/fluent-bit -c fluentbit.conf