feat: sample unification (#693)

BREAKING CHANGE: sample and test unification

Author: uoboda-splunk

.github/workflows/build-test-release.yml

@@ -8,7 +8,8 @@ on:
     tags:
       - "v[0-9]+.[0-9]+.[0-9]+"
   pull_request:
-    branches: [main, develop]
+    branches:
+      - "**"
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}
@@ -146,7 +147,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        splunk-version: ["8.1","8.2"]
+        splunk-version: ["8.1", "9.0"]
     steps:
       - uses: actions/checkout@v3
         with:
@@ -198,7 +199,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        splunk-version: [8.1, 8.2]
+        splunk-version: ["8.1", "9.0"]
         test-marker: [
           "splunk_connection_docker",
           "splunk_app_fiction",
@@ -208,10 +209,8 @@ jobs:
           "splunk_fiction_indextime",
           "splunk_fiction_indextime_broken",
           "splunk_setup_fixture",
-          "splunk_app_requirements",
-          "splunk_app_requirements_modinput",
-          "splunk_app_requirements_uf",
-          "splunk_app_requirements_scripted"
+          "splunk_app_req",
+          "splunk_app_req_broken",
         ]
     steps:
       - uses: actions/checkout@v3

.github/workflows/exclude-patterns.txt

@@ -2,4 +2,6 @@
 .circleci/
 deps/.*
 .*\.lock
-tests/
+tests/
+NOTICE
+pytest_splunk_addon/standard_lib/CIM_Models/5.0.0/Ticket_Management.json

.gitignore

@@ -30,4 +30,5 @@ docs/_build/
 !deps/build
 .vscode
 test_report.md
-.idea/
+.idea/
+.python-version

.licenserc.yaml

@@ -31,6 +31,7 @@ header:
     - "tests/**"
     - ".*"
     - "pytest_splunk_addon/standard_lib/**/*.json"
+    - "pytest_splunk_addon/standard_lib/**/*.xsd"
     - "MANIFEST.in"
     - "entrypoint.sh"
     - "pytest_splunk_addon/.ignore_splunk_internal_errors"

Dockerfile.uf

@@ -13,10 +13,11 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
-ARG SPLUNK_VERSION=latest
-FROM splunk/universalforwarder:$SPLUNK_VERSION
-ARG SPLUNK_VERSION=latest
+# TODO: go back to latest version when fixed
+ARG SPLUNK_VERSION=8.2.6
+FROM splunk/universalforwarder:8.2.6
+ARG SPLUNK_VERSION=8.2.6
 ARG SPLUNK_APP_ID=TA_UNKNOWN
 ARG SPLUNK_APP_PACKAGE=package
 RUN echo ${SPLUNK_VERSION} $SPLUNK_APP_PACKAGE
-COPY ${SPLUNK_APP_PACKAGE} /opt/splunkforwarder/etc/apps/${SPLUNK_APP_ID}
+COPY ${SPLUNK_APP_PACKAGE} /opt/splunkforwarder/etc/apps/${SPLUNK_APP_ID}

NOTICE

@@ -6,7 +6,8 @@ export PATH="~/.pyenv/bin:$PATH"
 eval "$(pyenv init -)"
 pyenv install 3.7.8
 pyenv local 3.7.8
-curl -sSL https://raw.githubusercontent.com/python-poetry/poetry/master/get-poetry.py | python
+curl -sSL https://install.python-poetry.org | python
+export PATH="/root/.local/bin:$PATH"
 source ~/.poetry/env
 sleep 15
 poetry install -E docker

deps/build/addonfactory_test_matrix_splunk

@@ -46,6 +46,8 @@ junitparser = "^2.2.0"
 addonfactory-splunk-conf-parser-lib = "^0.3.3"
 defusedxml = "^0.7.1"
 Faker = "^13.12.0"
+xmltodict = "^0.13.0"
+xmlschema = "^1.11.3"
 
 [tool.poetry.extras]
 docker = ['lovely-pytest-docker']

entrypoint.sh

@@ -18,4 +18,4 @@
 
 __author__ = """Splunk Inc."""
 __email__ = "[email protected]"
-__version__ = "3.0.8"
+__version__ = "4.0.0-beta.5"

poetry.lock

@@ -27,7 +27,6 @@
 
 
 def pytest_configure(config):
-    global test_generator
     """
     Setup configuration after command-line options are parsed
     """
@@ -41,13 +40,25 @@ def pytest_configure(config):
         "markers",
         "splunk_searchtime_fields_positive: Test search time fields positive scenarios only",
     )
+    config.addinivalue_line(
+        "markers",
+        "splunk_requirements: Tests that cover old requirement tests",
+    )
+    config.addinivalue_line(
+        "markers",
+        "splunk_searchtime_fields_requirements: Test checking fields from cim_fields",
+    )
     config.addinivalue_line(
         "markers",
         "splunk_searchtime_fields_negative: Test search time fields negative scenarios only",
     )
     config.addinivalue_line(
         "markers", "splunk_searchtime_fields_tags: Test search time tags only"
     )
+    config.addinivalue_line(
+        "markers",
+        "splunk_searchtime_fields_datamodels: Test checking datamodel defined in model",
+    )
     config.addinivalue_line(
         "markers",
         "splunk_searchtime_fields_eventtypes: Test search time eventtypes only",
@@ -78,8 +89,10 @@ def pytest_configure(config):
         "markers",
         "splunk_searchtime_requirements: Test an requirement test only  is mapped with only one data models",
     )
-    if config.getoption("splunk_app", None):
-        test_generator = AppTestGenerator(config)
+    config.addinivalue_line(
+        "markers",
+        "splunk_requirements_unit: Test checking if all fields for datamodel are defined in cim_fields and missing_recommended_fields",
+    )
 
     cim_report = config.getoption("cim_report")
     if cim_report and not hasattr(config, "slaveinput"):
@@ -96,7 +109,7 @@ def pytest_unconfigure(config):
 
 
 def pytest_sessionstart(session):
-
+    global test_generator
     SampleXdistGenerator.event_path = session.config.getoption("event_path")
     SampleXdistGenerator.event_stored = False
     SampleXdistGenerator.tokenized_event_source = session.config.getoption(
@@ -114,6 +127,8 @@ def pytest_sessionstart(session):
         store_events = session.config.getoption("store_events")
         sample_generator = SampleXdistGenerator(app_path, config_path)
         sample_generator.get_samples(store_events)
+    if session.config.getoption("splunk_app", None):
+        test_generator = AppTestGenerator(session.config)
 
 
 def pytest_generate_tests(metafunc):

pyproject.toml

@@ -23,6 +23,7 @@
 import logging
 import os
 import shutil
+from collections import defaultdict
 from time import sleep
 import json
 import pytest
@@ -32,6 +33,7 @@
 from .helmut.splunk.cloud import CloudSplunk
 from .helmut_lib.SearchUtil import SearchUtil
 from .standard_lib.event_ingestors import IngestorHelper
+from .standard_lib.CIM_Models.datamodel_definition import datamodels
 import configparser
 from filelock import FileLock
 
@@ -265,13 +267,6 @@ def pytest_addoption(parser):
         dest="ignore_errors_not_related_to_addon",
         help=("Path to file where list of errors not related to addon are suppressed."),
     )
-    group.addoption(
-        "--requirement-test",
-        action="store",
-        dest="requirement_test",
-        default="None",
-        help="Default None, path to --requirement-test files if requirement tests need to be run",
-    )
     group.addoption(
         "--splunk-uf-host",
         action="store",
@@ -733,7 +728,6 @@ def splunk_ingest_data(request, splunk_hec_uri, sc4s, uf, splunk_events_cleanup)
     ):
         addon_path = request.config.getoption("splunk_app")
         config_path = request.config.getoption("splunk_data_generator")
-        run_requirement_test = request.config.getoption("requirement_test")
         ingest_meta_data = {
             "uf_host": uf.get("uf_host"),
             "uf_port": uf.get("uf_port"),
@@ -752,7 +746,6 @@ def splunk_ingest_data(request, splunk_hec_uri, sc4s, uf, splunk_events_cleanup)
             config_path,
             thread_count,
             store_events,
-            run_requirement_test,
         )
         sleep(50)
         if "PYTEST_XDIST_WORKER" in os.environ:
@@ -800,6 +793,38 @@ def file_system_prerequisite():
         os.mkdir(monitor_dir)
 
 
+@pytest.fixture(scope="session")
+def splunk_dm_recommended_fields():
+    """
+    Returns function which gets recommended fields from Splunk for given datamodel
+
+    Note that data is being dynamically retrieved from Splunk. When CIM add-on version changes
+    retrieved data may differ
+    """
+    recommended_fields = defaultdict(list)
+
+    def update_recommended_fields(model, datasets, cim_version):
+        model_key = f"{cim_version}:{model}:{':'.join(datasets)}".strip(":")
+
+        if model_key not in recommended_fields:
+            LOGGER.info(f"Fetching {model_key} definition")
+            datamodel_per_cim = datamodels.get(cim_version) or datamodels["latest"]
+            datamodel = datamodel_per_cim[model]
+            for object_name, value in datamodel.items():
+                if (
+                    object_name == "BaseEvent"
+                    or object_name in datasets
+                    or object_name == model
+                ):
+                    recommended_fields[model_key] += value
+
+        if not recommended_fields.get(model_key) or []:
+            raise ValueError(f"Model {model_key} definition was not fetched")
+        return recommended_fields
+
+    return update_recommended_fields
+
+
 def is_responsive_uf(uf):
     """
     Verify if the management port of Universal Forwarder is responsive or not