Merge pull request #250 from StackStorm/feature/tasks

Add tasks to the module

Author: nmaludy

.gitignore

@@ -22,8 +22,12 @@
 /convert_report.txt
 /update_report.txt
 .DS_Store
+.coverage
 .librarian
 .kitchen
 .tmp
 .bundle
+/cover/
+/ci/
 Puppetfile.lock
+*.pyc

.sync.yml

@@ -1,11 +1,15 @@
 ---
 .gitignore:
   paths:
+    - .coverage
     - .librarian
     - .kitchen
     - .tmp
     - .bundle
+    - /cover/
+    - /ci/
     - Puppetfile.lock
+    - '*.pyc'
 .gitlab-ci.yml:
   # we don't use GitLab
   unmanaged: true

.travis.yml

@@ -54,6 +54,22 @@ matrix:
         - UNIT_TEST="true"
         - PUPPET_GEM_VERSION="~> 6.0"
         - CHECK="syntax lint metadata_lint check:symlinks check:git_ignore check:dot_underscore check:test_file rubocop parallel_spec"
+    - name: "Unit Testing - Bolt Tasks Python 2.7"
+      language: python
+      python: 2.7
+      cache: pip
+      before_install:
+        - echo 'no bundler needed here'
+      script:
+        - make
+    - name: "Unit Testing - Bolt Tasks Python 3.6"
+      language: python
+      python: 3.6
+      cache: pip
+      before_install:
+        - echo 'no bundler needed here'
+      script:
+        - make
     - name: "Documentation Testing"
       rvm: 2.5
       # use default Gemfile in repo root (from PDK)

CHANGELOG.md

@@ -2,6 +2,16 @@
 
 ## Development
 
+- Added new tasks to communicate with the StackStorm CLI. The naming standard and parameters
+  are modeled after the `st2` CLI command and must be run on the StackStorm node:
+   - `st2::key_decrypt` - Decrypts an encrypted key/value pair
+   - `st2::key_get` - Retrieves the value for a key from the datastore
+   - `st2::key_load` - Loads a list of key/value pairs into the datastore
+   - `st2::pack_install` - Installs a list of packs
+   - `st2::pack_list` - Get a list of installed packs
+   - `st2::pack_remove` - Removes a list of packs
+  (Feature)
+
 - Fixed build for new release of `puppet/nginx` causing conflict with `puppetlabs/stdlib`.
   The new version `0.16.0` of `puppet/nginx` requires `puppetlabs/stdlib >= 5.0.0`.
   Several other modules we depend on require `puppetlabs/stdlib < 5.0.0` causing a conflict.
@@ -147,7 +157,7 @@
 - Fixed bug where the default nginx splash page was not being removed
   on RHEL/CentOS installs. (Bugfix)
   Contributed by @nmaludy
-  
+
 ## 1.1.0 (Sep 07, 2018)
 
 - DEPRECATION WARNING - Dropped support for Puppet 3. (Enhancement)
@@ -202,7 +212,7 @@
   Contributed by @nmaludy
 
 - Changed the behavior of `st2` packages. Previously they were automatically
-  updating due to the package resources having `ensure => latest` set. Going
+  updating due to the package resources having `ensure  latest` set. Going
   forward, packages will have `ensure => present` set by default  and it will be
   the responsibility of the end user to update the packages. (Change)
   Contributed by @nmaludy

Makefile

@@ -1,8 +1,46 @@
 THIS_FILE := $(lastword $(MAKEFILE_LIST))
 ROOT_DIR := $(shell dirname $(realpath $(lastword $(MAKEFILE_LIST))))
+CI_REPO_PATH ?= $(ROOT_DIR)/ci
+CI_REPO_BRANCH ?= master
+
+.PHONY: all
+all: .DEFAULT
 
 .PHONY: clean
-clean: clean-kitchen clean-puppet-librarian clean-bundler clean-pkg
+clean: clean-ci-repo clean-pyc clean-kitchen clean-puppet-librarian clean-bundler clean-pkg
+
+# Clone the ci-repo into the ci/ directory
+.PHONY: clone-ci-repo
+clone-ci-repo:
+	@echo
+	@echo "==================== clone-ci-repo ===================="
+	@echo
+	@if [ ! -d "$(CI_REPO_PATH)" ]; then \
+		git clone https://github.com/EncoreTechnologies/ci-puppet-python.git --depth 1 --single-branch --branch $(CI_REPO_BRANCH) $(CI_REPO_PATH); \
+	else \
+		cd $(CI_REPO_PATH); \
+		git pull; \
+	fi;
+
+# Clean the ci-repo (calling `make clean` in that directory), then remove the
+# ci-repo directory
+.PHONY: clean-ci-repo
+clean-ci-repo:
+	@echo
+	@echo "==================== clean-ci-repo ===================="
+	@echo
+	@if [ -d "$(CI_REPO_PATH)" ]; then \
+		make -f $(ROOT_DIR)/ci/Makefile clean; \
+	fi;
+	rm -rf $(CI_REPO_PATH)
+
+# Clean *.pyc files.
+.PHONY: clean-pyc
+clean-pyc:
+	@echo
+	@echo "==================== clean-pyc ===================="
+	@echo
+	find $(ROOT_DIR) -name 'ci' -prune -or -name '.git' -or -type f -name "*.pyc" -print | xargs -r rm
 
 # Clean kitchen build files
 .PHONY: clean-kitchen
@@ -40,3 +78,23 @@ clean-pkg:
 	@echo "== clean-pkg ======================================"
 	@echo
 	rm -rf ${ROOT_DIR}/pkg
+
+# list all makefile targets
+.PHONY: list
+list:
+	@if [ -d "$(CI_REPO_PATH)" ]; then \
+		$(MAKE) --no-print-directory -f $(ROOT_DIR)/ci/Makefile list; \
+	fi;
+	@$(MAKE) -pRrq -f $(lastword $(MAKEFILE_LIST)) : 2>/dev/null | awk -v RS= -F: '/^# File/,/^# Finished Make data base/ {if ($$1 !~ "^[#.]") {print $$1}}' | sort | egrep -v -e '^[^[:alnum:]]' -e '^$@$$' | sort | uniq | xargs
+
+# forward all make targets not found in this makefile to the ci makefile to do
+# the actual work (by calling the invoke-ci-makefile target)
+# http://stackoverflow.org/wiki/Last-Resort_Makefile_Targets
+# Unfortunately the .DEFAULT target doesn't allow for dependencies
+# so we have to manually specify all of the steps in this target.
+.DEFAULT: 
+	$(MAKE) clone-ci-repo
+	@echo
+	@echo "==================== invoke ci/Makefile (targets: $(MAKECMDGOALS)) ===================="
+	@echo
+	make -f $(ROOT_DIR)/ci/Makefile $(MAKECMDGOALS)

README.md

@@ -254,6 +254,99 @@ and [librarian-puppet](http://librarian-puppet.com/).
  * Ubuntu 16.06 - Puppet 4 - [build/ubuntu16-puppet4/Puppetfile](build/ubuntu16-puppet4/Puppetfile)
  * Ubuntu 16.06 - Puppet 5 - [build/ubuntu16-puppet5/Puppetfile](build/ubuntu16-puppet5/Puppetfile)
  * Ubuntu 16.06 - Puppet 6 - [build/ubuntu16-puppet6/Puppetfile](build/ubuntu16-puppet6/Puppetfile)
+ 
+ 
+## Tasks
+
+This module provides several tasks for interacting with StackStorm. These tasks
+are modeled after the `st2` CLI command, names of the tasks and parameters reflect this.
+Under the hood, the tasks invoke the `st2` CLI command so they must be executed on
+a node where StackStorm is installed.
+
+### Tasks List
+
+- `st2::key_decrypt` - Decrypts an encrypted key/value pair
+- `st2::key_get` - Retrieves the value for a key from the datastore
+- `st2::key_load` - Loads a list of key/value pairs into the datastore
+- `st2::pack_install` - Installs a list of packs
+- `st2::pack_list` - Get a list of installed packs
+- `st2::pack_remove` - Removes a list of packs
+  
+### Task Usage
+
+Tasks that interact with the `st2` CLI command require authentication with the StackStorm
+instance. There are three options for authentication:
+
+- API Key
+- Auth token
+- Username/password
+
+#### Task Usage - API Key
+
+API keys are the recommended way for systems to authenticate with StackStorm.
+To do this via a task, you would first create an API key in StackStorm:
+
+``` shell
+$ st2 apikey create -m '{"used_by": "bolt"}'
+```
+
+Copy the API `key` parameter in the output, and then use it when invoking one of
+the tasks in this module via the `api_key` parameter:
+
+Usage via command line:
+``` shell
+
+bolt task run st2::key_get key="testkey" api_key='xyz123'
+```
+
+Usage in a plan:
+``` puppet
+$res = run_task('st2::key_get', $stackstorm_target,
+                key        => 'testkey',
+                api_key    => $api_key)
+```
+
+#### Task Usage - Auth tokens
+
+Auth tokens can be used by `bolt` to communicate with StackStorm. First, the user
+needs to create an auth token, then pass it in via the `auth_token` parameter
+
+``` shell
+$ st2 auth myuser
+```
+
+Copy the auth token in the output, and then use it when invoking one of
+the tasks in this module:
+
+Usage via command line:
+``` shell
+bolt task run st2::key_get key="testkey" auth_token='xyz123'
+```
+
+Usage in a plan:
+``` puppet
+$res = run_task('st2::key_get', $stackstorm_target,
+                key        => 'testkey',
+                auth_token => $auth_token)
+```
+
+#### Task Usage - Username/Password
+
+Finally `bolt` can accept username/passwords to communicate with StackStorm.
+
+Usage via command line:
+``` shell
+bolt task run st2::key_get key="testkey" username="myuser" password="xyz123"
+```
+
+Usage in a plan:
+``` puppet
+$res = run_task('st2::key_get', $stackstorm_target,
+                key      => 'testkey',
+                username => $username,
+                password => $password)
+```
+
 
 ## Upgrading StackStorm
 

build/scripts/ci.sh

@@ -15,6 +15,9 @@ elif [ ! -z "$DOCS_TEST" ]; then
   export PUPPET_GEM_VERSION="$PUPPET_GEM_VERSION"
   echo $PUPPET_GEM_VERSION
   "$SCRIPT_DIR"/ci_docs_generate.sh
+elif [ ! -z "$PYTHON_TEST" ]; then
+  pushd "$SCRIPT_DIR"/../..
+  make
 else
   export TEST_NAME="$TEST_NAME"
   echo $TEST_NAME

files/st2_task_base.py

@@ -0,0 +1,82 @@
+#!/usr/bin/env python
+import json
+import os
+import subprocess
+import sys
+import traceback
+
+# import Bolt task helper
+sys.path.append(os.path.join(os.environ['PT__installdir'], 'python_task_helper', 'files'))
+from task_helper import TaskHelper, TaskError
+
+try:
+    # python 2
+    from urlparse import urlparse
+except ImportError:
+    # python 3
+    from urllib.parse import urlparse  # noqa
+
+
+class St2TaskBase(TaskHelper):
+
+    def login(self, args):
+        self.api_key = args.get('api_key')
+        self.auth_token = args.get('auth_token')
+        self.username = args.get('username')
+        self.password = args.get('password')
+        # inherit environment variables from the Bolt context to preserve things
+        # like locale... otherwise we get errors from the StackStorm client.
+        self.env = os.environ
+
+        # prefer API key over auth tokens
+        if self.api_key:
+            self.env['ST2_API_KEY'] = self.api_key
+        elif self.auth_token:
+            self.env['ST2_AUTH_TOKEN'] = self.auth_token
+        elif self.username and self.password:
+            # auth on the command line with username/password
+            cmd = ['st2', 'auth', '--only-token', '-p', self.password, self.username]
+            stdout = subprocess.check_output(cmd)
+            self.env['ST2_AUTH_TOKEN'] = stdout.rstrip()
+        # else
+        #    assume auth token is written in client config for this user.
+        #    don't worry, if there is no auth we'll get an error
+
+    def parse_output(self, stdout):
+        try:
+            # try to parse stdout as JSON and return the parse result
+            return {'result': json.loads(stdout)}
+        except ValueError:
+            # JSON parsing failed, return the raw stdout string
+            return {'result': stdout}
+
+    def exec_cmd(self, cmd, error_msg):
+        result = {}
+        try:
+            stdout = subprocess.check_output(cmd,
+                                             stderr=subprocess.STDOUT,
+                                             env=self.env)
+            result.update(self.parse_output(stdout))
+        except subprocess.CalledProcessError as e:
+            tb = traceback.format_exc()
+            raise TaskError(("Could not {}: {} \n {}\n {}".
+                             format(error_msg, str(e), e.output, tb)),
+                            'st2.task.base/subprocess_error')
+        except Exception as e:
+            tb = traceback.format_exc()
+            raise TaskError(("Could not {}: {}\n {}".
+                             format(error_msg, str(e), tb)),
+                            'st2.task.base/exec_exception')
+        return result
+
+    def task(self, args):
+        try:
+            self.login(args)
+            return self.task_impl(args)
+        except Exception as e:
+            tb = traceback.format_exc()
+            raise TaskError(str(e) + '\n' + tb,
+                            'st2.task.base/task_exception')
+
+    def task_impl(self, args):
+        raise NotImplementedError()

tasks/key_decrypt.json

@@ -0,0 +1,16 @@
+{
+  "description": "Decrypt a StackStorm key/value pair. TODO - Remove this when the following is closed: https://github.com/StackStorm/st2/issues/4545",
+  "parameters": {
+    "crypto_key_path": {
+      "type": "String",
+      "description": "Path to StackStorm crypto key"
+    },
+    "keys": {
+      "type": "Array[Hash]",
+      "description": "List of key value pairs"
+    }
+  },
+  "implementations": [
+    {"name": "key_decrypt.py"}
+  ]
+}