fix: ADDON-66599 deprecate splunk-hec-token default value and break test execution on exception (#804)

- deprecate default value of splunk-hec-token - give a warning when it's
used
- add console handler to logger - warnings and errors beside being
logged to pytest_splunk_addon.log are also logged to console
- other minor refactors from PR-803

References:
https://splunk.atlassian.net/browse/ADDON-66599
https://github.com/splunk/pytest-splunk-addon/pull/803/files

---------

Co-authored-by: Artem Rys <[email protected]>

Author: mkolasinski-splunk

.github/workflows/build-test-release.yml

@@ -186,6 +186,7 @@ jobs:
           "splunk_app_cim_broken",
           "splunk_fiction_indextime",
           "splunk_fiction_indextime_broken",
+          "splunk_fiction_indextime_wrong_hec_token",
           "splunk_setup_fixture",
           "splunk_app_req",
           "splunk_app_req_broken",

.gitignore

@@ -35,3 +35,4 @@ test_report.md
 *.pickle
 *_events.lock
 *_events
+.venv

pytest_splunk_addon/plugin.py

@@ -25,6 +25,8 @@
 
 test_generator = None
 
+EXC_MAP = [Exception]
+
 
 def pytest_configure(config):
     """
@@ -93,6 +95,10 @@ def pytest_configure(config):
         "markers",
         "splunk_requirements_unit: Test checking if all fields for datamodel are defined in cim_fields and missing_recommended_fields",
     )
+    if config.getoption("splunk_hec_token") == "9b741d03-43e9-4164-908b-e09102327d22":
+        LOGGER.warning(
+            "Using the default value for --splunk-hec-token which is going to be deprecated. Please provide --splunk-hec-token argument when executing tests."
+        )
 
     cim_report = config.getoption("cim_report")
     if cim_report and not hasattr(config, "slaveinput"):
@@ -115,6 +121,7 @@ def pytest_sessionstart(session):
     SampleXdistGenerator.tokenized_event_source = session.config.getoption(
         "tokenized_event_source"
     ).lower()
+    session.__exc_limits = EXC_MAP
     if (
         SampleXdistGenerator.tokenized_event_source == "store_new"
         and session.config.getoption("ingest_events").lower()
@@ -187,16 +194,31 @@ def init_pytest_splunk_addon_logger():
     """
     fh = logging.FileHandler(LOG_FILE)
     fh.setLevel(logging.INFO)
+    ch = logging.StreamHandler()
+    ch.setLevel(logging.WARNING)
     formatter = logging.Formatter(
         "%(asctime)s - %(levelname)s - %(filename)s - %(funcName)s - %(message)s"
     )
     fh.setFormatter(formatter)
+    ch.setFormatter(formatter)
     logger = logging.getLogger("pytest-splunk-addon")
     logger.addHandler(fh)
+    logger.addHandler(ch)
     logging.root.propagate = False
     logger.setLevel(logging.INFO)
     return logger
 
 
 init_pytest_splunk_addon_logger()
 LOGGER = logging.getLogger("pytest-splunk-addon")
+
+
+def pytest_exception_interact(node, call, report):
+    """
+    Hook called when an exception is raised during a test.
+    If the number of occurrences for a specific exception exceeds the limit in session.__exc_limits, pytest exits
+    https://docs.pytest.org/en/stable/reference/reference.html#pytest.hookspec.pytest_exception_interact
+    """
+    if call.excinfo.type in node.session.__exc_limits:
+        # pytest exits only for exceptions defined in EXC_MAP
+        pytest.exit(f"Exiting pytest due to: {call.excinfo.type}")

pytest_splunk_addon/splunk.py

@@ -110,7 +110,7 @@ def pytest_addoption(parser):
         action="store",
         dest="splunk_hec_token",
         default="9b741d03-43e9-4164-908b-e09102327d22",
-        help='Splunk HTTP event collector token. default is "9b741d03-43e9-4164-908b-e09102327d22" If an external forwarder is used provide HEC token of forwarder.',
+        help="Splunk HTTP event collector token. If an external forwarder is used provide HEC token of forwarder. Please specify it as default value is going to be deprecated.",
     )
     group.addoption(
         "--splunk-port",
@@ -552,6 +552,7 @@ def splunk_docker(
         # get the temp directory shared by all workers
         root_tmp_dir = tmp_path_factory.getbasetemp().parent
         fn = root_tmp_dir / "pytest_docker"
+        # if you encounter docker-compose not found modify shell path in your IDE to use /bin/bash
         with FileLock(str(fn) + ".lock"):
             docker_services.start("splunk")
 
@@ -576,11 +577,12 @@ def splunk_docker(
         docker_services.port_for("splunk", 9997),
     )
 
-    docker_services.wait_until_responsive(
-        timeout=180.0,
-        pause=0.5,
-        check=lambda: is_responsive_splunk(splunk_info),
-    )
+    for _ in range(RESPONSIVE_SPLUNK_TIMEOUT):
+        if is_responsive_splunk(splunk_info) and is_responsive_hec(
+            request, splunk_info
+        ):
+            break
+        sleep(1)
 
     return splunk_info
 
@@ -610,9 +612,9 @@ def splunk_external(request):
         )
 
     for _ in range(RESPONSIVE_SPLUNK_TIMEOUT):
-        if is_responsive_splunk(splunk_info):
-            break
-        if is_responsive_hec(request, splunk_info):
+        if is_responsive_splunk(splunk_info) and is_responsive_hec(
+            request, splunk_info
+        ):
             break
         sleep(1)
 

tests/e2e/incorrect_hec_token_conftest.py

@@ -0,0 +1,44 @@
+import os
+import pytest
+import requests
+
+pytest_plugins = "pytester"
+
+
+def pytest_configure(config):
+    config.addinivalue_line("markers", "external: Test search time only")
+    config.addinivalue_line("markers", "docker: Test search time only")
+    config.addinivalue_line("markers", "doc: Test Sphinx docs")
+
+
+@pytest.fixture(scope="session")
+def docker_compose_files(request):
+    """
+    Get an absolute path to the  `docker-compose.yml` file. Override this
+    fixture in your tests if you need a custom location.
+
+    Returns:
+        string: the path of the `docker-compose.yml` file
+
+    """
+    docker_compose_path = os.path.join(
+        str(request.config.invocation_dir), "docker-compose.yml"
+    )
+    # LOGGER.info("docker-compose path: %s", docker_compose_path)
+
+    return [docker_compose_path]
+
+
+@pytest.fixture(scope="session")
+def docker_services_project_name(pytestconfig):
+    rootdir = str(pytestconfig.rootdir)
+    docker_compose_v2_rootdir = rootdir.lower().replace("/", "")
+    return f"pytest{docker_compose_v2_rootdir}"
+
+
+@pytest.fixture(scope="session")
+def splunk_hec_uri(splunk, request):
+    splunk_session = requests.Session()
+    splunk_session.headers = {"Authorization": f"Splunk test-token"}
+    uri = f'{request.config.getoption("splunk_hec_scheme")}://{splunk["forwarder_host"]}:{splunk["port_hec"]}/services/collector'
+    return splunk_session, uri

tests/e2e/test_splunk_addon.py

@@ -166,6 +166,57 @@ def empty_method():
     assert result.ret == 0
 
 
+@pytest.mark.docker
+@pytest.mark.splunk_fiction_indextime_wrong_hec_token
+def test_splunk_fiction_indextime_wrong_hec_token(testdir):
+    """Make sure that pytest accepts our fixture."""
+
+    testdir.makepyfile(
+        """
+        from pytest_splunk_addon.standard_lib.addon_basic import Basic
+        class Test_App(Basic):
+            def empty_method():
+                pass
+
+    """
+    )
+
+    shutil.copytree(
+        os.path.join(testdir.request.fspath.dirname, "addons/TA_fiction_indextime"),
+        os.path.join(testdir.tmpdir, "package"),
+    )
+
+    shutil.copytree(
+        os.path.join(testdir.request.fspath.dirname, "test_data_models"),
+        os.path.join(testdir.tmpdir, "tests/data_models"),
+    )
+
+    setup_test_dir(testdir)
+    SampleGenerator.clean_samples()
+    Rule.clean_rules()
+    with open(
+        os.path.join(testdir.request.fspath.dirname, "incorrect_hec_token_conftest.py")
+    ) as conf_test_file:
+        testdir.makeconftest(conf_test_file.read())
+
+    # run pytest with the following cmd args
+    result = testdir.runpytest(
+        "--splunk-type=docker",
+        "-v",
+        "--search-interval=0",
+        "--search-retry=0",
+        "--splunk-data-generator=tests/addons/TA_fiction_indextime/default",
+        "--search-index=*,_internal",
+    )
+
+    result.assert_outcomes(errors=1, passed=0, failed=0, xfailed=0)
+    result.stdout.fnmatch_lines(
+        "!!!!!! _pytest.outcomes.Exit: Exiting pytest due to: <class 'Exception'> !!!!!!!"
+    )
+
+    assert result.ret != 0
+
+
 @pytest.mark.docker
 @pytest.mark.splunk_app_broken
 def test_splunk_app_broken(testdir):