feat: adding CLEAN_KEYS support (#869)

Added support for CLEAN_KEYS in Transforms.conf.
Added respective e2e test case for the same

Impact on Addons:

Now if Addons have have field names in transforms report which have
non-alphanumeric chars then those chars would be first converted to _
and then tests would be generated as by default in splunk CLEAN_KEYS is
set to true, so splunk also does the same thing.

Moreover if any report stanza has explicitly set CLEAN_KEYS=false then
for those reports field conversion would not happen and tests would be
generated as it is.

Author: harshilgajera-crest

pytest_splunk_addon/addon_parser/props_parser.py

@@ -16,7 +16,7 @@
 """
 Provides props.conf parsing mechanism
 """
-from typing import Dict
+from typing import Dict, List
 from typing import Generator
 from typing import Optional
 import logging
@@ -57,6 +57,14 @@ def props(self) -> Optional[Dict]:
         self._props = self._conf_parser.item_dict()
         return self._props if self._props else None
 
+    def update_field_names(self, field_list: List[str]) -> List[str]:
+        """
+        update field names to remove all the non-alphanumeric chars and replace them with _
+        """
+        for field in field_list:
+            field.name = re.sub(r"\W+", "_", field.name)
+        return field_list
+
     def get_props_fields(self):
         """
         Parse the props.conf and yield all supported fields
@@ -82,6 +90,13 @@ def get_props_fields(self):
                 else:
                     for transform_stanza, fields in self._get_report_fields(key, value):
                         field_list = list(fields)
+                        if (
+                            self.transforms_parser.transforms.get(
+                                transform_stanza, {}
+                            ).get("CLEAN_KEYS")
+                            != "false"
+                        ):
+                            field_list = self.update_field_names(field_list)
                         if field_list:
                             yield {
                                 "stanza": stanza_name,

pytest_splunk_addon/fields_tests/test_templates.py

@@ -20,12 +20,11 @@
 import pprint
 import logging
 import pytest
-from ..addon_parser import Field
 import json
 from itertools import chain
+from ..addon_parser import Field
 from ..utilities.log_helper import get_table_output
 from ..utilities.log_helper import format_search_query_log
-
 from .requirement_test_datamodel_tag_constants import dict_datamodel_tag
 
 TOP_FIVE_STRUCTURALLY_UNIQUE_EVENTS_QUERY_PART = " | dedup punct | head 5"

tests/e2e/addons/TA_broken/default/transforms.conf

@@ -48,4 +48,4 @@ case_sensitive_match = false
 # Expected result: FAIL
 [broken-NaN_lookup]
 filename = NaN.csv
-case_sensitive_match = false
+case_sensitive_match = false

tests/e2e/addons/TA_fiction/default/props.conf

@@ -81,6 +81,9 @@ EXTRACT-fiction-fourteen = (?<date_only_in_hotwarmpath_event>\d+-\d+-\d+).*in ho
 REPORT-fiction-tsc-delim-fields = fiction-tsc-delim-fields
 REPORT-fiction-tsc-sk-regex-format = fiction-tsc-sk-regex-format
 REPORT-fiction-tsc-sk-delim-format = fiction-tsc-sk-delim-format
+REPORT-fiction-tsc-sk-delim-format-with-clean-keys = fiction-tsc-sk-delim-format-with-clean-keys
+REPORT-fiction-tsc-non-alphanumeric = fiction-tsc-non-alphanumeric
+
 
 ## multiple transforms stanza associated with REPORT
 REPORT-fiction-tsc-regex-format = fiction-tsc-regex, fiction-tsc-regex-format

tests/e2e/addons/TA_fiction/default/transforms.conf

@@ -22,6 +22,23 @@ SOURCE_KEY = event_id
 DELIMS = "="
 FIELDS = server_contact_mode, dest
 
+# Component tested: REPORT, DELIM-FIELDS-SOURCE_KEY
+# Scenario:# Similar to the above scenario
+## Here as CLEAN_KEYS = false server-contact-mode will be searched as is instead of converting it.
+[fiction-tsc-sk-delim-format-with-clean-keys]
+CLEAN_KEYS = false
+SOURCE_KEY = event_id
+DELIMS = "="
+FIELDS = server-contact-mode, dest
+
+# Component tested: REPORT, DELIM
+# Scenario:
+## server-contact-mode should be searched as server_contact_mode as CLEAN_KEYS = true by default[fiction-tsc-non-alphanumeric]
+[fiction-tsc-non-alphanumeric]
+DELIMS = " "
+FIELDS = server-contact, dest_1
+
+
 # Component tested: REPORT, REGEX-FORMAT-SOURCE_KEY
 # Scenario: Source-key with regex and format 
 ## An individual search for SOURCE_KEY and each field extracted in FORMAT and a single search of all the fields with SOURCE_KEY.

tests/e2e/constants.py

@@ -64,6 +64,11 @@
     "*test_splunk_app_fiction.py::Test_App::test_props_fields*splunkd::field::dest* PASSED*",
     "*test_splunk_app_fiction.py::Test_App::test_props_fields*splunkd::REPORT-fiction-tsc-regex-format::fiction-tsc-regex* PASSED*",
     "*test_splunk_app_fiction.py::Test_App::test_props_fields*splunkd::REPORT-fiction-tsc-regex-format::fiction-tsc-regex-format* PASSED*",
+    "*test_splunk_app_fiction.py::Test_App::test_props_fields*splunkd::REPORT-fiction-tsc-sk-delim-format-with-clean-keys::fiction-tsc-sk-delim-format-with-clean-keys* PASSED*",
+    "*test_splunk_app_fiction.py::Test_App::test_props_fields*splunkd::field::server-contact-mode* PASSED*",
+    "*test_splunk_app_fiction.py::Test_App::test_props_fields*splunkd::field::server_contact* PASSED*",
+    "*test_splunk_app_fiction.py::Test_App::test_props_fields*splunkd::field::dest* PASSED*",
+    "*test_splunk_app_fiction.py::Test_App::test_props_fields*splunkd::field::dest_1* PASSED*",
     "*test_splunk_app_fiction.py::Test_App::test_props_fields*splunkd::field::size1* PASSED*",
     "*test_splunk_app_fiction.py::Test_App::test_props_fields*splunkd::field::size2* PASSED*",
     "*test_splunk_app_fiction.py::Test_App::test_props_fields*splunkd::field::myeval* PASSED*",
@@ -119,6 +124,11 @@
     "*test_splunk_app_fiction.py::Test_App::test_props_fields_no_dash_not_empty**/opt/splunk/var/log/splunk/splunkd.log*::field::splunk_server* PASSED*",
     "*test_splunk_app_fiction.py::Test_App::test_props_fields_no_dash_not_empty**/opt/splunk/var/log/splunk/splunkd_access.log*::field::splunk_server* PASSED*",
     "*test_splunk_app_fiction.py::Test_App::test_props_fields_no_dash_not_empty*splunkd::EXTRACT-fiction-one* PASSED*",
+    "*test_splunk_app_fiction.py::Test_App::test_props_fields_no_dash_not_empty*splunkd::REPORT-fiction-tsc-sk-delim-format-with-clean-keys::fiction-tsc-sk-delim-format-with-clean-keys* PASSED*",
+    "*test_splunk_app_fiction.py::Test_App::test_props_fields_no_dash_not_empty*splunkd::field::server-contact-mode* PASSED*",
+    "*test_splunk_app_fiction.py::Test_App::test_props_fields_no_dash_not_empty*splunkd::field::dest* PASSED*",
+    "*test_splunk_app_fiction.py::Test_App::test_props_fields_no_dash_not_empty*splunkd::field::dest_1* PASSED*",
+    "*test_splunk_app_fiction.py::Test_App::test_props_fields_no_dash_not_empty*splunkd::field::server_contact* PASSED*",
     "*test_splunk_app_fiction.py::Test_App::test_props_fields_no_dash_not_empty*splunkd::field::extractone* PASSED*",
     "*test_splunk_app_fiction.py::Test_App::test_props_fields_no_dash_not_empty*splunkd::EXTRACT-fiction-two* PASSED*",
     "*test_splunk_app_fiction.py::Test_App::test_props_fields_no_dash_not_empty*splunkd::field::extracttwoA* PASSED*",