Merge pull request #333 from EncoreTechnologies/feature/service-scale-out

Process scale out and coordination

Author: nmaludy

CHANGELOG.md

@@ -2,6 +2,14 @@
 
 ## Development
 
+- Added the ability to scale out workflowengine, scheduler, rulesengine, and notifier services
+  so that they run in an active-active configuration
+  Contributed by @bishopbm1
+- Added the Redis configuration for a Coordination backend
+  Contributed by @bishopbm1
+- Added neccessary erlang package required for RHEL 8 rabbitmq
+  Contributed by @bishopbm1
+
 ## 2.1.0 (Mar 6, 2021)
   Note: This version only supports the 'integrated' LDAP plugin.
 

README.md

@@ -157,6 +157,7 @@ classes for use and configuration.
 * `st2::profile::nodejs` - st2 configured NodeJS installation
 * `st2::profile::python` - Python installed and configured for st2
 * `st2::profile::rabbitmq` - st2 configured RabbitMQ installation
+* `st2::profile::redis` - st2 configured Redis installation
 * `st2::proflle::server` - st2 server components
 * `st2::profile::web` - st2 web components
 * `st2::profile::chatops` - st2 chatops components
@@ -318,6 +319,33 @@ Configuration via Hiera:
     RESPOND_TO_DM: true
 ```
 
+#### Scaling out services
+
+This module supports scaling out workflowengine, scheduler, rulesengine, and notifier services
+per the [ST2 Documentation](https://docs.stackstorm.com/reference/ha.html).
+
+This would be something that you might consider doing if you have alot of rules running or if you
+have alot of workflows running in parrallel and/or you have alot of nested workflows and have a server
+that can be higher on CPU and Memory to allow more processes to run at the same time.
+
+Configuration all services:
+```ruby
+class { 'st2':
+  python_version     => '3.6',
+  workflowengine_num => 4,
+  scheduler_num      => 2,
+  rulesengine_num    => 1,
+  notifier_num       => 1,
+}
+```
+
+Or configure individual:
+```ruby
+class { 'st2::workflowengine':
+  workflowengine_num => 4,
+}
+```
+
 ### Tasks
 
 This module provides several tasks for interacting with StackStorm. These tasks
@@ -484,6 +512,8 @@ please submit a [Pull Request](https://github.com/StackStorm/puppet-st2/pulls).
 * StackStorm <[email protected]>
 * James Fryman
 * Patrick Hoolboom
+* Bradley Bishop
+  * GitHub - [@nmaludy](https://github.com/bishopbm1)
 
 ### Help
 

build/centos7-puppet6/Puppetfile

@@ -31,6 +31,7 @@
 #   │ └── puppetlabs-concat (v6.2.0)
 #   ├── puppet-nodejs (v7.0.1)
 #   ├── puppet-python (v4.1.1)
+#   ├── puppet-redis (v7.0.0)
 #   ├─┬ puppet-rabbitmq (v10.1.1)
 #   │ ├── camptocamp-systemd (v2.9.0)
 #   │ └── puppet-archive (v4.5.0)
@@ -64,3 +65,4 @@ mod 'puppetlabs-inifile'
 mod 'puppetlabs-stdlib'
 mod 'puppetlabs-yumrepo_core'
 mod 'saz-sudo'
+mod 'puppet/redis'

build/centos7-puppet7/Puppetfile

@@ -31,6 +31,7 @@
 #   │ └── puppetlabs-concat (v6.2.0)
 #   ├── puppet-nodejs (v7.0.1)
 #   ├── puppet-python (v4.1.1)
+#   ├── puppet-redis (v7.0.0)
 #   ├─┬ puppet-rabbitmq (v10.1.1)
 #   │ ├── camptocamp-systemd (v2.9.0)
 #   │ └── puppet-archive (v4.5.0)
@@ -64,3 +65,4 @@ mod 'puppetlabs-inifile'
 mod 'puppetlabs-stdlib'
 mod 'puppetlabs-yumrepo_core'
 mod 'saz-sudo'
+mod 'puppet/redis'

build/ubuntu16-puppet6/Puppetfile

@@ -31,6 +31,7 @@
 #   │ └── puppetlabs-concat (v6.2.0)
 #   ├── puppet-nodejs (v7.0.1)
 #   ├── puppet-python (v4.1.1)
+#   ├── puppet-redis (v7.0.0)
 #   ├─┬ puppet-rabbitmq (v10.1.1)
 #   │ ├── camptocamp-systemd (v2.9.0)
 #   │ └── puppet-archive (v4.5.0)
@@ -64,3 +65,4 @@ mod 'puppetlabs-inifile'
 mod 'puppetlabs-stdlib'
 mod 'puppetlabs-yumrepo_core'
 mod 'saz-sudo'
+mod 'puppet/redis'

build/ubuntu16-puppet7/Puppetfile

@@ -31,6 +31,7 @@
 #   │ └── puppetlabs-concat (v6.2.0)
 #   ├── puppet-nodejs (v7.0.1)
 #   ├── puppet-python (v4.1.1)
+#   ├── puppet-redis (v7.0.0)
 #   ├─┬ puppet-rabbitmq (v10.1.1)
 #   │ ├── camptocamp-systemd (v2.9.0)
 #   │ └── puppet-archive (v4.5.0)
@@ -64,3 +65,4 @@ mod 'puppetlabs-inifile'
 mod 'puppetlabs-stdlib'
 mod 'puppetlabs-yumrepo_core'
 mod 'saz-sudo'
+mod 'puppet/redis'

build/ubuntu18-puppet6/Puppetfile

@@ -31,6 +31,7 @@
 #   │ └── puppetlabs-concat (v6.2.0)
 #   ├── puppet-nodejs (v7.0.1)
 #   ├── puppet-python (v4.1.1)
+#   ├── puppet-redis (v7.0.0)
 #   ├─┬ puppet-rabbitmq (v10.1.1)
 #   │ ├── camptocamp-systemd (v2.9.0)
 #   │ └── puppet-archive (v4.5.0)
@@ -64,3 +65,4 @@ mod 'puppetlabs-inifile'
 mod 'puppetlabs-stdlib'
 mod 'puppetlabs-yumrepo_core'
 mod 'saz-sudo'
+mod 'puppet/redis'

build/ubuntu18-puppet7/Puppetfile

@@ -31,6 +31,7 @@
 #   │ └── puppetlabs-concat (v6.2.0)
 #   ├── puppet-nodejs (v7.0.1)
 #   ├── puppet-python (v4.1.1)
+#   ├── puppet-redis (v7.0.0)
 #   ├─┬ puppet-rabbitmq (v10.1.1)
 #   │ ├── camptocamp-systemd (v2.9.0)
 #   │ └── puppet-archive (v4.5.0)
@@ -64,3 +65,4 @@ mod 'puppetlabs-inifile'
 mod 'puppetlabs-stdlib'
 mod 'puppetlabs-yumrepo_core'
 mod 'saz-sudo'
+mod 'puppet/redis'

files/st2_task_base.py

@@ -39,14 +39,21 @@ def login(self, args):
         elif self.username and self.password:
             # auth on the command line with username/password
             cmd = ['st2', 'auth', '--only-token', '-p', self.password, self.username]
-            stdout = subprocess.check_output(cmd)
+            stdout = self.bytes_to_string(subprocess.check_output(cmd))
             self.env['ST2_AUTH_TOKEN'] = stdout.rstrip()
         # else
         #    assume auth token is written in client config for this user.
         #    don't worry, if there is no auth we'll get an error
 
+    def bytes_to_string(self, string):
+        if isinstance(string, bytes):
+            string = string.decode("utf-8")
+
+        return string
+
     def parse_output(self, stdout):
         try:
+            stdout = self.bytes_to_string(stdout)
             # try to parse stdout as JSON and return the parse result
             return {'result': json.loads(stdout)}
         except ValueError:

manifests/init.pp

@@ -212,6 +212,20 @@
 # @param nodejs_manage_repo
 #   Set this to false when you have your own repositories
 #   for NodeJS (default: true)
+# @param redis_bind_ip
+#   Bind IP of the Redis server. Default is 127.0.0.1
+# @param workflowengine_num
+#   The number of workflowengines to have in an active active state (default: 1)
+# @param scheduler_num
+#   The number of schedulers to have in an active active state (default: 1)
+# @param rulesengine_num
+#   The number of rulesengines to have in an active active state (default: 1)
+# @param notifier_num
+#   The number of notifiers to have in an active active state (default: 1)
+# @param erlang_url
+#   The url for the erlang repositiory to be used for rabbitmq
+# @param erlang_key
+#   The gpg key for the erlang repositiory to be used for rabbitmq
 #
 #
 # @example Basic Usage
@@ -314,6 +328,9 @@
   $rabbitmq_port            = $::st2::params::rabbitmq_port,
   $rabbitmq_bind_ip         = $::st2::params::rabbitmq_bind_ip,
   $rabbitmq_vhost           = $::st2::params::rabbitmq_vhost,
+  $erlang_url               = $::st2::params::erlang_url,
+  $erlang_key               = $::st2::params::erlang_key,
+  $redis_bind_ip            = $::st2::params::redis_bind_ip,
   $timersengine_enabled     = $::st2::params::timersengine_enabled,
   $timersengine_timezone    = $::st2::params::timersengine_timezone,
   $scheduler_sleep_interval = $::st2::params::scheduler_sleep_interval,
@@ -333,6 +350,10 @@
   $chatops_web_url                      = undef,
   $nodejs_version           = undef,
   $nodejs_manage_repo       = true,
+  $workflowengine_num       = $::st2::params::workflowengine_num,
+  $scheduler_num            = $::st2::params::scheduler_num,
+  $rulesengine_num          = $::st2::params::rulesengine_num,
+  $notifier_num             = $::st2::params::notifier_num,
 ) inherits st2::params {
 
   ########################################

manifests/notifier.pp

@@ -0,0 +1,44 @@
+# @summary Manages the <code>st2notifier</code> service (Orquesta)
+#
+# Normally this class is instantiated by +st2::profile::fullinstall+.
+# However, advanced users can instantiate this class directly to configure
+# and manage just the <code>st2notifier</code> service on a single node.
+# Parameters for this class mirror the parameters in the st2 config.
+#
+# @see https://github.com/StackStorm/st2/blob/master/conf/st2.conf.sample
+#
+# @example Basic usage
+#   include st2::notifier
+#
+# @param notifier_num
+#   The number of notifiers to have in an active active state
+# @param notifier_services
+#   Name of all the notifier services
+#
+class st2::notifier (
+  $notifier_num      = $st2::notifier_num,
+  $notifier_services = $st2::params::notifier_services,
+) inherits st2 {
+
+  $_logger_config = $::st2::syslog ? {
+    true    => 'syslog',
+    default => 'logging',
+  }
+
+  ########################################
+  ## Config
+  ini_setting { 'notifier_logging':
+    ensure  => present,
+    path    => '/etc/st2/st2.conf',
+    section => 'notifier',
+    setting => 'logging',
+    value   => "/etc/st2/${_logger_config}.notifier.conf",
+    tag     => 'st2::config',
+  }
+
+  st2::service { 'st2notifier':
+    service_name      => 'st2notifier',
+    service_num       => $notifier_num,
+    existing_services => $notifier_services,
+  }
+}

manifests/params.pp

@@ -87,12 +87,20 @@
     'st2api',
     'st2auth',
     'st2garbagecollector',
-    'st2notifier',
-    'st2rulesengine',
     'st2sensorcontainer',
     'st2stream',
   ]
 
+  ## StackStorm Workflow Engine (Orchestra)
+  $rulesengine_services = [
+    'st2rulesengine',
+  ]
+
+  ## StackStorm Workflow Engine (Orchestra)
+  $notifier_services = [
+    'st2notifier',
+  ]
+
   ## StackStorm ChatOps services
   $st2_chatops_services = [
     'st2chatops',
@@ -144,6 +152,18 @@
   # no max on the body size for large workflow support
   $nginx_client_max_body_size = '0'
 
+  # Number of workflow engines to run
+  $workflowengine_num = 1
+
+  # Number of schedulers to run
+  $scheduler_num = 1
+
+  # Number of rules engines to run
+  $rulesengine_num = 1
+
+  # Number of notifiers to run
+  $notifier_num = 1
+
   # st2web
   $web_root = '/opt/stackstorm/static/webui/'
 
@@ -164,6 +184,9 @@
   $rabbitmq_port = 5672
   $rabbitmq_bind_ip = '127.0.0.1'
   $rabbitmq_vhost = '/'
+  $erlang_url = "https://packagecloud.io/rabbitmq/erlang/el/${facts['os'][release][major]}/\$basearch"
+  $erlang_key = 'https://packagecloud.io/rabbitmq/rabbitmq-server/gpgkey'
+  $redis_bind_ip = '127.0.0.1'
 
   ## actionrunner config
   $actionrunner_workers = 10

manifests/profile/fullinstall.pp

@@ -32,6 +32,7 @@
   -> class { 'st2::repo': }
   -> class { 'st2::profile::selinux': }
   -> Anchor['st2::pre_reqs']
+  -> class { 'st2::profile::redis': }
   -> class { 'st2::profile::python': }
   -> class { 'st2::profile::nodejs': }
   -> class { 'st2::profile::rabbitmq': }

manifests/profile/python.pp

@@ -49,10 +49,9 @@
 
     # intall python and python-devel / python-dev
     class { 'python':
-      version                   => $version,
-      dev                       => present,
-      manage_pip_package        => false,
-      manage_virtualenv_package => false,
+      version            => $version,
+      dev                => present,
+      manage_pip_package => false,
     }
   }
 }

manifests/profile/rabbitmq.pp

@@ -22,16 +22,30 @@
 #   include st2::profile::rabbitmq
 #
 class st2::profile::rabbitmq (
-  $username = $::st2::rabbitmq_username,
-  $password = $::st2::rabbitmq_password,
-  $port     = $::st2::rabbitmq_port,
-  $bind_ip  = $::st2::rabbitmq_bind_ip,
-  $vhost    = $::st2::rabbitmq_vhost,
+  $username   = $::st2::rabbitmq_username,
+  $password   = $::st2::rabbitmq_password,
+  $port       = $::st2::rabbitmq_port,
+  $bind_ip    = $::st2::rabbitmq_bind_ip,
+  $vhost      = $::st2::rabbitmq_vhost,
+  $erlang_url = $::st2::erlang_url,
+  $erlang_key = $::st2::erlang_key
 ) inherits st2 {
 
   # RHEL 8 Requires another repo in addition to epel to be installed
   if ($::osfamily == 'RedHat') and ($facts['os']['release']['major'] == '8') {
     $repos_ensure = true
+
+    # This is required because when using the latest version of rabbitmq because the latest version in EPEL
+    # for Erlang is 22.0.7 which is not compatible: https://www.rabbitmq.com/which-erlang.html
+    yumrepo { 'erlang':
+      ensure   => present,
+      name     => 'rabbitmq_erlang',
+      baseurl  => $erlang_url,
+      gpgkey   => $erlang_key,
+      enabled  => 1,
+      gpgcheck => 1,
+      before   => Class['rabbitmq::repo::rhel'],
+    }
   }
   else {
     $repos_ensure = false

manifests/profile/redis.pp

@@ -0,0 +1,23 @@
+# @summary StackStorm compatable installation of Redis.
+#
+# @param [String] String
+#   Bind IP of the Redis server. Default is 127.0.0.1
+#
+# @example Basic Usage
+#  include st2::profile::redis
+#
+# @example Install with redis
+#   class { '::redis':
+#      bind => '127.0.0.1',
+#    }
+#
+class st2::profile::redis (
+  String  $bind_ip = $st2::redis_bind_ip,
+) inherits st2 {
+
+  class { 'redis':
+      bind => $bind_ip,
+  }
+
+  contain redis
+}