feat: allow *.xml (beside *.log) files as sample input files (#550)

Author: hsekowski-splunk

pytest_splunk_addon/standard_lib/event_ingestors/requirement_event_ingester.py

@@ -110,7 +110,7 @@ def get_events(self):
         if os.path.isdir(req_file_path):
             for file1 in os.listdir(req_file_path):
                 filename = os.path.join(req_file_path, file1)
-                if filename.endswith(".log"):
+                if filename.endswith(".log") or filename.endswith(".xml"):
                     if self.check_xml_format(filename):
                         root = self.get_root(filename)
                         for event_tag in root.iter("event"):
@@ -213,7 +213,7 @@ def get_events(self):
                         )
                 else:
                     LOGGER.error(
-                        "Requirement event ingestion failure: Invalid file format not .log {}".format(
+                        "Requirement event ingestion failure: Invalid file format not .log or .xml {}".format(
                             filename
                         )
                     )

pytest_splunk_addon/standard_lib/requirement_tests/test_generator.py

@@ -118,7 +118,7 @@ def generate_cim_req_params(self):
             for file1 in os.listdir(req_file_path):
                 filename = os.path.join(req_file_path, file1)
                 LOGGER.info(filename)
-                if filename.endswith(".log"):
+                if filename.endswith(".log") or filename.endswith(".xml"):
                     try:
                         self.check_xml_format(filename)
                     except Exception:

tests/requirement_test/sample_requiremen.xml

@@ -0,0 +1,52 @@
+<?xml version="1.0"?>
+<device>
+   <vendor>Juniper</vendor>
+   <product>JunOS</product>
+   <version id="16.2R1" />
+   <version id="17.1R1" />
+   <version id="17.2R1" />
+   <event code="" name="RT_FLOW_SESSION_CREATE" format="syslog">
+      <transport type="syslog" />
+      <source>
+         <jira id="ADDON-25170"/>
+         <comment>Got this event form Juniper document.</comment>
+      </source>
+      <raw>
+         <![CDATA[<111> 2020-02-12T03:27:09+10:00 sample.dvc RT_FLOW: RT_FLOW_SESSION_CREATE: session created 1.1.1.1/34667->10.0.0.1/5048 0x0 junos-http 1.1.1.2/34667->10.0.0.2/5048 0x0 sample_src_rule_type sample_src_rule_name sample_dst_rule_type sample_dest_rule_n**ame 6 1660(global) SAMPLE-SERVER-ZONE DUMMY_ZONE 113256 user2(admin) gg-0/0/0.1 SNMP DUMMY_APP UNKNOWN]]>
+      </raw>
+      <cim>
+         <models>
+            <model>Network Traffic</model>
+         </models>
+         <cim_fields>
+            <field name="action" value="allowed"/>
+            <field name="dest" value="10.0.0.1"/>
+            <field name="dest_ip" value="10.0.0.1"/>
+            <field name="dest_port" value="5048"/>
+            <field name="dest_zone" value="DUMMY_ZONE"/>
+            <field name="dvc" value="sample.dvc"/>
+            <field name="rule" value="sample_src_rule_name sample_dest_rule_n**ame 1660(global)"/>
+            <field name="session_id" value="113256"/>
+            <field name="src" value="1.1.1.1"/>
+            <field name="src_ip" value="1.1.1.1"/>
+            <field name="src_port" value="34667"/>
+            <field name="src_zone" value="SAMPLE-SERVER-ZONE"/>
+            <field name="src_interface" value="gg-0/0/0.1"/>
+            <field name="user" value="user2"/>
+            <field name="app" value="SNMP DUMMY_APP"/>
+            <field name="transport" value="tcp"/>
+            <field name="protocol" value="ip"/>
+            <field name="vendor_product" value="Incorrect vendor product"/>
+         </cim_fields>
+         <missing_recommended_fields>
+            <field>bytes</field>
+            <field>bytes_in</field>
+            <field>bytes_out</field>
+         </missing_recommended_fields>
+         <exceptions>
+            <field name="vendor_product" value="Incorrect vendor product" reason="testing exceptions"/>
+         </exceptions>
+      </cim>
+      <test></test>
+   </event>
+</device>

tests/test_splunk_addon.py

@@ -551,7 +551,8 @@ def empty_method():
     result.stdout.fnmatch_lines_random(
         constants.TA_REQUIREMENTS_PASSED + constants.TA_REQUIREMENTS_FAILED
     )
-    result.assert_outcomes(passed=len(constants.TA_REQUIREMENTS_PASSED), failed=1)
+    result.assert_outcomes(passed=2, failed=1)
+    #   passed=2 as the successful data comes from 2 sources (log & xml)
 
     # make sure that that we get a non '0' exit code for the testsuite as it contains failure
     assert result.ret != 0

tests/unit/tests_standard_lib/test_requirement_tests/test_test_generator.py

@@ -133,11 +133,27 @@ def test_extract_params():
             ["requirement.xml"],
             [True],
             ["syslog"],
-            {"event": ["event_1", "event_2"]},
-            [["model_1:dataset_1", "model_2:dataset_2"], ["model_3:dataset_3"]],
-            ["event_name_2"],
+            {"event": ["<34>Oct 11 22:14:15 machine1 pr1:event_1"]},
+            [["model_1:dataset_1", "model_2:dataset_2"]],
+            ["event_name_1"],
             [{"field1": "value1", "field2": "value2"}, {"field3": "value3"}],
-            [],
+            [
+                (
+                    {
+                        "model_list": [
+                            ("model_1", "dataset_1", ""),
+                            ("model_2", "dataset_2", ""),
+                        ],
+                        "escaped_event": "event_1",
+                        "exceptions_dict": {"field3": "value3"},
+                        "Key_value_dict": {"field1": "value1", "field2": "value2"},
+                        "modinput_params": None,
+                        "transport_type": "syslog",
+                    },
+                    "model_1:dataset_1 "
+                    "model_2:dataset_2::fake_path/requirement.xml::event_no::1::event_name::event_name_1",
+                ),
+            ],
         ),
         (
             ["req.log"],