Merge pull request #414 from splunk/Release-1.6.18

Release 1.6.18

Author: vmalaviya-splunk

.github/workflows/test.yml

@@ -45,12 +45,6 @@ jobs:
           echo password=changed! >> .splunkrc
           echo scheme=https >> .splunkrc
           echo version=${{ matrix.splunk }} >> .splunkrc
-      - name: Create build dir for ExamplesTestCase::test_build_dir_exists test case
-        run: |
-          cd ~
-          cd /home/runner/work/splunk-sdk-python/splunk-sdk-python/
-          python setup.py build
-          python setup.py dist
       - name: Install tox
         run: pip install tox
       - name: Test Execution

.gitignore

@@ -16,15 +16,15 @@ MANIFEST
 coverage_report
 test.log
 examples/*/local
-examples/*/metadata/local.meta
+examples/**/local.meta
+examples/**/*.log
 tests/searchcommands_data/log/
 tests/searchcommands_data/output/
 examples/searchcommands_app/searchcommand_app.log
 Test Results*.html
 tests/searchcommands/data/app/app.log
 splunk_sdk.egg-info/
 dist/
-examples/searchcommands_app/package/default/commands.conf
 examples/searchcommands_app/package/lib/splunklib
 tests/searchcommands/apps/app_with_logging_configuration/*.log
 *.observed

CHANGELOG.md

@@ -1,5 +1,18 @@
 # Splunk Enterprise SDK for Python Changelog
 
+## Version 1.6.18
+
+### Bug fixes
+* [#405](https://github.com/splunk/splunk-sdk-python/pull/405) Fix searchcommands_app example
+* [#406](https://github.com/splunk/splunk-sdk-python/pull/406) Fix mod inputs examples
+* [#407](https://github.com/splunk/splunk-sdk-python/pull/407) Fixed issue with Streaming and Generating Custom Search Commands dropping fields that aren't present in the first row of results. More details on how to opt-in to this fix can be found here: 
+https://github.com/splunk/splunk-sdk-python/blob/develop/README.md#customization [ [issue#401](https://github.com/splunk/splunk-sdk-python/issues/401) ]
+
+### Minor changes
+* [#408](https://github.com/splunk/splunk-sdk-python/pull/408) Add search mode example
+* [#409](https://github.com/splunk/splunk-sdk-python/pull/409) Add Support for authorization tokens read from .splunkrc [ [issue#388](https://github.com/splunk/splunk-sdk-python/issues/388) ]
+* [#413](https://github.com/splunk/splunk-sdk-python/pull/413) Default kvstore owner to nobody [ [issue#231](https://github.com/splunk/splunk-sdk-python/issues/231) ]
+
 ## Version 1.6.17
 
 ### Bug fixes

Makefile

@@ -18,16 +18,11 @@ DATE := `date "+%FT%T%z"`
 CONTAINER_NAME := 'splunk'
 
 .PHONY: all
-all: build_app test
+all: test
 
 init:
 	@echo "$(ATTN_COLOR)==> init $(NO_COLOR)"
 
-.PHONY: build_app
-build_app:
-	@echo "$(ATTN_COLOR)==> build_app $(NO_COLOR)"
-	@python setup.py build dist
-
 .PHONY: docs
 docs:
 	@echo "$(ATTN_COLOR)==> docs $(NO_COLOR)"

README.md

@@ -3,7 +3,7 @@
 
 # The Splunk Enterprise Software Development Kit for Python
 
-#### Version 1.6.17
+#### Version 1.6.18
 
 The Splunk Enterprise Software Development Kit (SDK) for Python contains library code and examples designed to enable developers to build applications using the Splunk platform.
 
@@ -71,6 +71,26 @@ To run the examples and unit tests, you must put the root of the SDK on your PYT
 
 The SDK command-line examples require a common set of arguments that specify the host, port, and login credentials for Splunk Enterprise. For a full list of command-line arguments, include `--help` as an argument to any of the examples.
 
+### Following are the different ways to connect to Splunk Enterprise
+#### Using username/password
+```python
+import splunklib.client as client
+    service = client.connect(host=<host_url>, username=<username>, password=<password>, autoLogin=True)
+```
+
+#### Using bearer token
+```python
+import splunklib.client as client
+service = client.connect(host=<host_url>, splunkToken=<bearer_token>, autologin=True)
+```
+
+#### Using session key
+```python
+import splunklib.client as client
+service = client.connect(host=<host_url>, token=<session_key>, autologin=True)
+```
+
+###
 #### Create a .splunkrc convenience file
 
 To connect to Splunk Enterprise, many of the SDK examples and unit tests take command-line arguments that specify values for the host, port, and login credentials for Splunk Enterprise. For convenience during development, you can store these arguments as key-value pairs in a text file named **.splunkrc**. Then, the SDK examples and unit tests use the values from the **.splunkrc** file when you don't specify them.
@@ -112,8 +132,18 @@ Save the file as **.splunkrc** in the current user's home directory.
 
 Examples are located in the **/splunk-sdk-python/examples** directory. To run the examples at the command line, use the Python interpreter and include any arguments that are required by the example. In the commands below, replace "examplename" with the name of the specific example in the directory that you want to run:
 
+Using username and Password
+    
     python examplename.py --username="admin" --password="changeme"
 
+Using Bearer token
+    
+    python examplename.py --bearerToken=<value>
+
+Using Session key
+    
+    python examplename.py --sessionKey="<value>"
+
 If you saved your login credentials in the **.splunkrc** file, you can omit those arguments:
 
     python examplename.py
@@ -150,6 +180,53 @@ The test suite uses Python's standard library, the built-in `unittest` library,
 |/tests     | Source for unit tests                                      |
 |/utils     | Source for utilities shared by the examples and unit tests |
 
+### Customization
+* When working with custom search commands such as Custom Streaming Commands or Custom Generating Commands, We may need to add new fields to the records based on certain conditions.
+* Structural changes like this may not be preserved.
+* Make sure to use ``add_field(record, fieldname, value)`` method from SearchCommand to add a new field and value to the record.
+* ___Note:__ Usage of ``add_field`` method is completely optional, if you are not facing any issues with field retention._
+
+Do
+```python
+class CustomStreamingCommand(StreamingCommand):
+    def stream(self, records):
+        for index, record in enumerate(records):
+            if index % 1 == 0:
+                self.add_field(record, "odd_record", "true")
+            yield record
+```
+
+Don't
+```python
+class CustomStreamingCommand(StreamingCommand):
+    def stream(self, records):
+        for index, record in enumerate(records):
+            if index % 1 == 0:
+                record["odd_record"] = "true"
+            yield record
+```
+### Customization for Generating Custom Search Command
+* Generating Custom Search Command is used to generate events using SDK code.
+* Make sure to use ``gen_record()`` method from SearchCommand to add a new record and pass event data as a key=value pair separated by , (mentioned in below example).
+
+Do
+```python
+@Configuration()
+    class GeneratorTest(GeneratingCommand):
+        def generate(self):
+            yield self.gen_record(_time=time.time(), one=1)
+            yield self.gen_record(_time=time.time(), two=2)
+```
+
+Don't
+```python
+@Configuration()
+    class GeneratorTest(GeneratingCommand):
+        def generate(self):
+            yield {'_time': time.time(), 'one': 1}
+            yield {'_time': time.time(), 'two': 2}
+```
+
 ### Changelog
 
 The [CHANGELOG](CHANGELOG.md) contains a description of changes for each version of the SDK. For the latest version, see the [CHANGELOG.md](https://github.com/splunk/splunk-sdk-python/blob/master/CHANGELOG.md) on GitHub.

docker-compose.yml

@@ -9,6 +9,14 @@ services:
             - SPLUNK_HEC_TOKEN=11111111-1111-1111-1111-1111111111113
             - SPLUNK_PASSWORD=changed!
             - SPLUNK_APPS_URL=https://github.com/splunk/sdk-app-collection/releases/download/v1.1.0/sdkappcollection.tgz
+        volumes:
+            - ./examples/github_forks:/opt/splunk/etc/apps/github_forks
+            - ./splunklib:/opt/splunk/etc/apps/github_forks/lib/splunklib
+            - ./examples/random_numbers:/opt/splunk/etc/apps/random_numbers
+            - ./splunklib:/opt/splunk/etc/apps/random_numbers/lib/splunklib
+            - ./examples/searchcommands_app/package:/opt/splunk/etc/apps/searchcommands_app
+            - ./splunklib:/opt/splunk/etc/apps/searchcommands_app/lib/splunklib
+            - ./examples/twitted/twitted:/opt/splunk/etc/apps/twitted
         ports:
             - 8000:8000
             - 8088:8088

examples/github_forks/README.md

@@ -0,0 +1,12 @@
+splunk-sdk-python github_forks example
+========================================
+
+This app provides an example of a modular input that generates the number of repository forks according to the Github API based on the owner and repo_name provided by the user during setup of the input.
+
+To run this example locally run `SPLUNK_VERSION=latest docker compose up -d` from the root of this repository which will mount this example alongside the latest version of splunklib within `/opt/splunk/etc/apps/github_forks` and `/opt/splunk/etc/apps/github_forks/lib/splunklib` within the `splunk` container.
+
+Once the docker container is up and healthy log into the Splunk UI and setup a new `Github Repository Forks` input by visiting this page: http://localhost:8000/en-US/manager/github_forks/datainputstats and selecting the "Add new..." button next to the Local Inputs > Random Inputs. Enter values for a Github Repository owner and repo_name, for example owner = `splunk` repo_name = `splunk-sdk-python`.
+
+NOTE: If no Github Repository Forks input appears then the script is likely not running properly, see https://docs.splunk.com/Documentation/SplunkCloud/latest/AdvancedDev/ModInputsDevTools for more details on debugging the modular input using the command line and relevant logs.
+
+Once the input is created you should be able to see an event when running the following search: `source="github_forks://*"` the event should contain fields for `owner` and `repository` matching the values you input during setup and then a `fork_count` field corresponding to the number of forks the repo has according to the Github API.

examples/github_forks/github_forks.py renamed to examples/github_forks/bin/github_forks.py

@@ -15,10 +15,18 @@
 # under the License.
 
 from __future__ import absolute_import
-import sys, urllib2, json
+import os
+import sys
+import json
+# NOTE: splunklib must exist within github_forks/lib/splunklib for this
+# example to run! To run this locally use `SPLUNK_VERSION=latest docker compose up -d`
+# from the root of this repo which mounts this example and the latest splunklib
+# code together at /opt/splunk/etc/apps/github_forks
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "lib"))
 
 from splunklib.modularinput import *
 from splunklib import six
+from six.moves import http_client
 
 class MyScript(Script):
     """All modular inputs should inherit from the abstract base class Script
@@ -87,11 +95,9 @@ def validate_input(self, validation_definition):
         # Get the values of the parameters, and construct a URL for the Github API
         owner = validation_definition.parameters["owner"]
         repo_name = validation_definition.parameters["repo_name"]
-        repo_url = "https://api.github.com/repos/%s/%s" % (owner, repo_name)
 
-        # Read the response from the Github API, then parse the JSON data into an object
-        response = urllib2.urlopen(repo_url).read()
-        jsondata = json.loads(response)
+        # Call Github to retrieve repo information
+        jsondata = _get_github_repos(owner, repo_name)
 
         # If there is only 1 field in the jsondata object,some kind or error occurred
         # with the Github API.
@@ -125,9 +131,7 @@ def stream_events(self, inputs, ew):
             repo_name = input_item["repo_name"]
 
             # Get the fork count from the Github API
-            repo_url = "https://api.github.com/repos/%s/%s" % (owner, repo_name)
-            response = urllib2.urlopen(repo_url).read()
-            jsondata = json.loads(response)
+            jsondata = _get_github_repos(owner, repo_name)
             fork_count = jsondata["forks_count"]
 
             # Create an Event object, and set its fields
@@ -139,5 +143,20 @@ def stream_events(self, inputs, ew):
             # Tell the EventWriter to write this event
             ew.write_event(event)
 
+
+def _get_github_repos(owner, repo_name):
+    # Read the response from the Github API, then parse the JSON data into an object
+    repo_path = "/repos/%s/%s" % (owner, repo_name)
+    connection = http_client.HTTPSConnection('api.github.com')
+    headers = {
+        'Content-type': 'application/json',
+        'User-Agent': 'splunk-sdk-python',
+    }
+    connection.request('GET', repo_path, headers=headers)
+    response = connection.getresponse()
+    body = response.read().decode()
+    return json.loads(body)
+
+
 if __name__ == "__main__":
     sys.exit(MyScript().run(sys.argv))

examples/random_numbers/README.md

@@ -0,0 +1,12 @@
+splunk-sdk-python random_numbers example
+========================================
+
+This app provides an example of a modular input that generates a random number between the min and max values provided by the user during setup of the input.
+
+To run this example locally run `SPLUNK_VERSION=latest docker compose up -d` from the root of this repository which will mount this example alongside the latest version of splunklib within `/opt/splunk/etc/apps/random_numbers` and `/opt/splunk/etc/apps/random_numbers/lib/splunklib` within the `splunk` container.
+
+Once the docker container is up and healthy log into the Splunk UI and setup a new `Random Numbers` input by visiting this page: http://localhost:8000/en-US/manager/random_numbers/datainputstats and selecting the "Add new..." button next to the Local Inputs > Random Inputs. Enter values for the `min` and `max` values which the random number should be generated between.
+
+NOTE: If no Random Numbers input appears then the script is likely not running properly, see https://docs.splunk.com/Documentation/SplunkCloud/latest/AdvancedDev/ModInputsDevTools for more details on debugging the modular input using the command line and relevant logs.
+
+Once the input is created you should be able to see an event when running the following search: `source="random_numbers://*"` the event should contain a `number` field with a float between the min and max specified when the input was created.

examples/random_numbers/random_numbers.py renamed to examples/random_numbers/bin/random_numbers.py

@@ -17,6 +17,10 @@
 from __future__ import absolute_import
 import random, sys
 import os
+# NOTE: splunklib must exist within random_numbers/lib/splunklib for this
+# example to run! To run this locally use `SPLUNK_VERSION=latest docker compose up -d`
+# from the root of this repo which mounts this example and the latest splunklib
+# code together at /opt/splunk/etc/apps/random_numbers
 sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "lib"))
 
 from splunklib.modularinput import *

examples/search_modes.py

@@ -0,0 +1,41 @@
+import sys
+import os
+# import from utils/__init__.py
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), ".."))
+from utils import *
+import time
+from splunklib.client import connect
+from splunklib import results
+from splunklib import six
+
+def cmdline(argv, flags, **kwargs):
+    """A cmdopts wrapper that takes a list of flags and builds the
+       corresponding cmdopts rules to match those flags."""
+    rules = dict([(flag, {'flags': ["--%s" % flag]}) for flag in flags])
+    return parse(argv, rules, ".splunkrc", **kwargs)
+
+def modes(argv):
+    opts = cmdline(argv, [])
+    kwargs_splunk = dslice(opts.kwargs, FLAGS_SPLUNK)
+    service = connect(**kwargs_splunk)
+
+    # By default the job will run in 'smart' mode which will omit events for transforming commands
+    job = service.jobs.create('search index=_internal | head 10 | top host')
+    while not job.is_ready():
+        time.sleep(0.5)
+        pass    
+    reader = results.ResultsReader(job.events())
+    # Events found: 0
+    print('Events found with adhoc_search_level="smart": %s' % len([e for e in reader]))
+
+    # Now set the adhoc_search_level to 'verbose' to see the events
+    job = service.jobs.create('search index=_internal | head 10 | top host', adhoc_search_level='verbose')
+    while not job.is_ready():
+        time.sleep(0.5)
+        pass
+    reader = results.ResultsReader(job.events())
+    # Events found: 10
+    print('Events found with adhoc_search_level="verbose": %s' % len([e for e in reader]))
+
+if __name__ == "__main__":
+    modes(sys.argv[1:])