Merge pull request #135 from onaio/monit-scripts

monit scripts for services monitoring

script/monit/README.md

@@ -0,0 +1,9 @@
+## Required
+
+- pip install -r requirements.txt
+- Edit email, aws_email, and replace AWS_*, SITE_DOMAIN, ALERT_RECIPIENTS with correct details for your server setup.
+
+# Deploy
+
+    $ fab deploy:prod,scripts='system email aws_email, nginx postgres rabbitmq'
+

script/monit/apache

@@ -0,0 +1,8 @@
+check process apache
+    with pidfile /var/run/apache2.pid
+    start program = "/usr/sbin/apache2ctl start" as uid root and gid root
+    stop program = "/usr/sbin/apache2ctl stop" as uid root and gid root
+    if totalmem is greater than 1960.0 MB for 2 cycles then restart
+    if loadavg(5min) greater thab 10 for 8 cycles then restart
+    if 20 restarts within 20 cycles then timeout
+    group apache

script/monit/aws_email

@@ -0,0 +1,3 @@
+set mailserver email-smtp.us-east-1.amazonaws.com port 465
+    username "AWS_ACCESS_KEY_ID" password "AWS_SECRETE_ACCESS_KEY"
+    using sslv3 with timeout 15 seconds using hostname SITE_DOMAIN

script/monit/celerybeat

@@ -0,0 +1,6 @@
+check process celerybeat
+    with pidfile /var/run/celery/beat.pid
+    start program = "/etc/init.d/celerybeat start" as uid root and gid root
+    stop program = "/etc/init.d/celerybeat stop" as uid root and gid root
+    if 5 restarts within 10 cycles then timeout
+    group celery

script/monit/celeryd

@@ -0,0 +1,6 @@
+check process celeryd
+    with pidfile /var/run/celery/w1-ona.pid
+    start program = "/etc/init.d/celeryd-ona start" as uid root and gid root
+    stop program = "/etc/init.d/celeryd-ona stop" as uid root and gid root
+    if 5 restarts within 10 cycles then timeout
+    group celery

script/monit/email

@@ -0,0 +1,13 @@
+set mail-format {
+    from: noreply@SITE_DOMAIN
+    subject: $SERVICE $EVENT at $DATE on $HOST
+    message:
+Alert from $HOST
+Date: $DATE
+For $SERVICE ($EVENT) and take $ACTION
+--------------------------------------
+$DESCRIPTION
+}
+
+# alert recepients
+set alert ALERT_RECIPIENTS

script/monit/fabfile.py

@@ -0,0 +1,41 @@
+import os
+import sys
+
+from fabric.api import env, sudo, put
+
+
+DEPLOYMENTS = {
+    'stage': {
+        'home': '/home/ubuntu/src/',
+        'host_string': '[email protected]',
+        'key_filename': os.path.expanduser('~/.ssh/ona.pem')
+    },
+    'prod': {
+        'home': '/home/ubuntu/src/',
+        'host_string': '[email protected]',
+        'key_filename': os.path.expanduser('~/.ssh/ona.pem')
+    },
+}
+
+
+def check_key_filename(deployment_name):
+    if 'key_filename' in DEPLOYMENTS[deployment_name] and \
+       not os.path.exists(DEPLOYMENTS[deployment_name]['key_filename']):
+        print ("Cannot find required permissions file: %s" %
+               DEPLOYMENTS[deployment_name]['key_filename'])
+        return False
+    return True
+
+
+def setup_env(deployment_name):
+    env.update(DEPLOYMENTS[deployment_name])
+    if not check_key_filename(deployment_name):
+        sys.exit(1)
+
+
+def deploy(deployment_name, scripts=''):
+    setup_env(deployment_name)
+    sudo('which monit || apt-get install -y monit')
+    for script in scripts.split(' '):
+        put(script, '/etc/monit/conf.d/%s' % script, use_sudo=True)
+    sudo('/etc/init.d/monit restart')

script/monit/mongodb

@@ -0,0 +1,9 @@
+check process mongodb
+    with pidfile "/var/lib/mongodb/mongod.lock"
+    start program "/etc/init.d/mongodb start"
+    stop program "/etc/init.d/mongodb stop"
+    if failed port 28017 protocol http
+        and request "/" with timeout 10 seconds then restart
+    if 5 restarts within 5 cycles then timeout
+    group database
+

script/monit/monit

@@ -0,0 +1,3 @@
+set httpd port 2812 and
+use address localhost
+allow localhost

script/monit/mysql

@@ -0,0 +1,7 @@
+check process mysql
+    with pidfile /var/run/mysqld/mysqld.pid
+    start "/etc/init.d/mysql start"
+    stop "/etc/init.d/mysql stop"
+    if failed host 127.0.0.1 port 3306 then restart
+    if 5 restarts within 5 cycles then timeout
+    group database

script/monit/nginx

@@ -0,0 +1,8 @@
+check process nginx
+    with pidfile /var/run/nginx.pid
+    start program = "/etc/init.d/nginx start" as uid root and gid root
+    stop program = "/etc/init.d/nginx stop" as uid root and gid root
+    if totalmem is greater than 1960.0 MB for 2 cycles then restart
+    if loadavg(5min) greater than 10 for 8 cycles then restart
+    if 20 restarts within 20 cycles then timeout
+    group www-data #(for ubuntu, debian)

script/monit/postgres

@@ -0,0 +1,7 @@
+check process postgresql
+    with pidfile /var/run/postgresql/9.3-main.pid
+    start "/etc/init.d/postgresql start"
+    stop "/etc/init.d/postgresql stop"
+    if failed host 127.0.0.1 port 3306 then restart
+    if 5 restarts within 5 cycles then timeout
+    group database

script/monit/rabbitmq

@@ -0,0 +1,5 @@
+check process rabbitmq
+    with pidfile /var/run/rabbitmq/pid
+    start program = "/etc/init.d/rabbitmq-server start"
+    stop program = "/etc/init.d/rabbitmq-server stop"
+    group rabbitmq-server

script/monit/requirements.txt

@@ -0,0 +1 @@
+fabric

script/monit/system

@@ -0,0 +1,7 @@
+check system localhost
+    if memory usage > 85% then alert
+    if cpu usage (user) > 80% for 3 cycles then alert
+    if cpu usage (system) > 80% for 3 cycles then alert
+
+check filesystem rootfs with path /
+    if space usage > 80% then alert

script/monit/uwsgi

@@ -0,0 +1,9 @@
+check process uwsgi
+    with pidfile /var/run/ona.pid
+    start program = "/sbin/start ona" as uid root and gid root
+    stop program = "/sbin/stop ona" as uid root and gid root
+    if totalmem is greater than 1960.0 MB for 3 cycles then restart
+    if cpu > 50% for 2 cycles then alert
+    if loadavg(5min) greater than 10 for 8 cycles then restart
+    if 20 restarts within 20 cycles then timeout
+    group uwsgi