Re-Activated all Logstash pipeline tests

Author: Chris Wiechmann

.github/workflows/logstash.yml

@@ -40,30 +40,30 @@ jobs:
       run: |
         wget -c https://github.com/magnusbaeck/logstash-filter-verifier/releases/download/1.6.3/logstash-filter-verifier_1.6.3_linux_amd64.tar.gz -O - | tar -xvz
     - name: Test Logstash pipeline
-      run: |
-        echo Using API_BUILDER_URL: $API_BUILDER_URL
-        echo Using MEMCACHED: $MEMCACHED
-        echo Using GEOIP_ENABLED: $GEOIP_ENABLED, GEOIP_CUSTOM_ATTRIBUTE: ${GEOIP_CUSTOM_ATTRIBUTE}, GEOIP_CACHE_SIZE: ${GEOIP_CACHE_SIZE}
-        ./logstash-filter-verifier --diff-command="diff -y" --logstash-output --keep-env=API_BUILDER_SSL_CERT --keep-env=API_BUILDER_URL --keep-env=MEMCACHED --keep-env=GEOIP_ENABLED --keep-env=GEOIP_CUSTOM_ATTRIBUTE --keep-env=GEOIP_CACHE_SIZE --keep-env=EVENTLOG_CUSTOM_ATTR ./logstash/test/http/test-events.json ./logstash/pipelines/EventsPipeline.conf
-        
       # run: |
       #   echo Using API_BUILDER_URL: $API_BUILDER_URL
       #   echo Using MEMCACHED: $MEMCACHED
-      #   ./logstash-filter-verifier --diff-command="diff -y" --keep-env=API_BUILDER_SSL_CERT --keep-env=API_BUILDER_URL --keep-env=MEMCACHED ./logstash/test/jms/test-opentrafficlog-jms.json ./logstash/pipelines/OpenTrafficPipeline.conf
-      #   ./logstash-filter-verifier --diff-command="diff -y" --keep-env=API_BUILDER_SSL_CERT --keep-env=API_BUILDER_URL --keep-env=MEMCACHED ./logstash/test/beatsInput/test-index-creation-lookup.json ./logstash/pipelines/BeatsInputPipeline.conf
-      #   ./logstash-filter-verifier --diff-command="diff -y" --keep-env=API_BUILDER_SSL_CERT --keep-env=API_BUILDER_URL --keep-env=MEMCACHED --keep-env=DROP_TRACE_MESSAGE_LEVELS ./logstash/test/http/test-tracemessages.json ./logstash/pipelines/TraceMessagesPipeline.conf
-      #   ./logstash-filter-verifier --diff-command="diff -y" --keep-env=API_BUILDER_SSL_CERT --keep-env=API_BUILDER_URL --keep-env=MEMCACHED ./logstash/test/http/test-tracemessages-gmt-4.json ./logstash/pipelines/TraceMessagesPipeline.conf
-      #   ./logstash-filter-verifier --diff-command="diff -y" --keep-env=API_BUILDER_SSL_CERT --keep-env=API_BUILDER_URL --keep-env=MEMCACHED --keep-env=GEOIP_ENABLED --keep-env=GEOIP_CUSTOM_ATTRIBUTE --keep-env=GEOIP_CACHE_SIZE ./logstash/test/http/test-events.json ./logstash/pipelines/EventsPipeline.conf
-      #   ./logstash-filter-verifier --diff-command="diff -y" --keep-env=API_BUILDER_SSL_CERT --keep-env=API_BUILDER_URL --keep-env=MEMCACHED ./logstash/test/http/test-opentrafficlog.json ./logstash/pipelines/OpenTrafficPipeline.conf
-      #   ./logstash-filter-verifier --diff-command="diff -y" --keep-env=API_BUILDER_SSL_CERT --keep-env=API_BUILDER_URL --keep-env=MEMCACHED ./logstash/test/filetransfer/test-opentrafficlog-filetransfer.json ./logstash/pipelines/OpenTrafficPipeline.conf
-      #   ./logstash-filter-verifier --diff-command="diff -y" --keep-env=API_BUILDER_SSL_CERT --keep-env=API_BUILDER_URL --keep-env=MEMCACHED ./logstash/test/domainAudit/test-domain-audit-events.json ./logstash/pipelines/DomainAuditPipeline.conf
+      #   echo Using GEOIP_ENABLED: $GEOIP_ENABLED, GEOIP_CUSTOM_ATTRIBUTE: ${GEOIP_CUSTOM_ATTRIBUTE}, GEOIP_CACHE_SIZE: ${GEOIP_CACHE_SIZE}
+      #   ./logstash-filter-verifier --diff-command="diff -y" --logstash-output --keep-env=API_BUILDER_SSL_CERT --keep-env=API_BUILDER_URL --keep-env=MEMCACHED --keep-env=GEOIP_ENABLED --keep-env=GEOIP_CUSTOM_ATTRIBUTE --keep-env=GEOIP_CACHE_SIZE --keep-env=EVENTLOG_CUSTOM_ATTR ./logstash/test/http/test-events.json ./logstash/pipelines/EventsPipeline.conf
+        
+      run: |
+        echo Using API_BUILDER_URL: $API_BUILDER_URL
+        echo Using MEMCACHED: $MEMCACHED
+        ./logstash-filter-verifier --diff-command="diff -y" --keep-env=API_BUILDER_SSL_CERT --keep-env=API_BUILDER_URL --keep-env=MEMCACHED ./logstash/test/jms/test-opentrafficlog-jms.json ./logstash/pipelines/OpenTrafficPipeline.conf
+        ./logstash-filter-verifier --diff-command="diff -y" --keep-env=API_BUILDER_SSL_CERT --keep-env=API_BUILDER_URL --keep-env=MEMCACHED ./logstash/test/beatsInput/test-index-creation-lookup.json ./logstash/pipelines/BeatsInputPipeline.conf
+        ./logstash-filter-verifier --diff-command="diff -y" --keep-env=API_BUILDER_SSL_CERT --keep-env=API_BUILDER_URL --keep-env=MEMCACHED --keep-env=DROP_TRACE_MESSAGE_LEVELS ./logstash/test/http/test-tracemessages.json ./logstash/pipelines/TraceMessagesPipeline.conf
+        ./logstash-filter-verifier --diff-command="diff -y" --keep-env=API_BUILDER_SSL_CERT --keep-env=API_BUILDER_URL --keep-env=MEMCACHED ./logstash/test/http/test-tracemessages-gmt-4.json ./logstash/pipelines/TraceMessagesPipeline.conf
+        ./logstash-filter-verifier --diff-command="diff -y" --keep-env=API_BUILDER_SSL_CERT --keep-env=API_BUILDER_URL --keep-env=MEMCACHED --keep-env=GEOIP_ENABLED --keep-env=GEOIP_CUSTOM_ATTRIBUTE --keep-env=GEOIP_CACHE_SIZE ./logstash/test/http/test-events.json ./logstash/pipelines/EventsPipeline.conf
+        ./logstash-filter-verifier --diff-command="diff -y" --keep-env=API_BUILDER_SSL_CERT --keep-env=API_BUILDER_URL --keep-env=MEMCACHED ./logstash/test/http/test-opentrafficlog.json ./logstash/pipelines/OpenTrafficPipeline.conf
+        ./logstash-filter-verifier --diff-command="diff -y" --keep-env=API_BUILDER_SSL_CERT --keep-env=API_BUILDER_URL --keep-env=MEMCACHED ./logstash/test/filetransfer/test-opentrafficlog-filetransfer.json ./logstash/pipelines/OpenTrafficPipeline.conf
+        ./logstash-filter-verifier --diff-command="diff -y" --keep-env=API_BUILDER_SSL_CERT --keep-env=API_BUILDER_URL --keep-env=MEMCACHED ./logstash/test/domainAudit/test-domain-audit-events.json ./logstash/pipelines/DomainAuditPipeline.conf
       env:
         API_BUILDER_URL: 'http://localhost:8080'
         MEMCACHED: 'localhost:11211'
         DROP_TRACE_MESSAGE_LEVELS: 'DEBUG'
         GEOIP_ENABLED: 'true'
         GEOIP_CUSTOM_ATTRIBUTE: 'xForwardedFor'
-        # The attribute at this point is only used to determine if a document should be generated and send to ES
+        # The attribute at this point is only used to determine if a document should be produced and send to ES
         # If not set - We avoid sending the event log to Elasticsearch (even if only relevant with GeoIP disabled)
         EVENTLOG_CUSTOM_ATTR: 'doesn-matter-for-the-pipeline-test'
         GEOIP_CACHE_SIZE: 1000