Update appliance-setup

.gitmodules

@@ -1,36 +1,51 @@
 [submodule "puppet/modules/java"]
 	path = puppet/modules/java
-	url = https://github.com/cirg/puppetlabs-java.git
+	url = https://github.com/puppetlabs/puppetlabs-java.git
+	branch = 1.6.0
 [submodule "puppet/modules/stdlib"]
 	path = puppet/modules/stdlib
-	url = https://github.com/cirg/puppetlabs-stdlib.git
+	url = https://github.com/puppetlabs/puppetlabs-stdlib.git
+	branch = 4.17.1
 [submodule "puppet/modules/mysql"]
 	path = puppet/modules/mysql
-	url = https://github.com/cirg/puppetlabs-mysql.git
+	url = https://github.com/puppetlabs/puppetlabs-mysql.git
+	branch = 3.11.0
 [submodule "puppet/modules/apt"]
 	path = puppet/modules/apt
-	url = https://github.com/cirg/puppetlabs-apt.git
+	url = https://github.com/puppetlabs/puppetlabs-apt.git
+	branch = 2.4.0
 [submodule "puppet/modules/openmrs"]
 	path = puppet/modules/openmrs
 	url = https://github.com/cirg/puppet-openmrs.git
 [submodule "puppet/modules/apache"]
 	path = puppet/modules/apache
-	url = https://github.com/cirg/puppetlabs-apache.git
+	url = https://github.com/puppetlabs/puppetlabs-apache.git
+	branch = 1.11.0
 [submodule "puppet/modules/postgresql"]
 	path = puppet/modules/postgresql
-	url = https://github.com/cirg/puppet-postgresql.git
+	url = https://github.com/puppetlabs/puppetlabs-postgresql.git
+	branch = 4.9.0
 [submodule "puppet/modules/tomcat"]
 	path = puppet/modules/tomcat
-	url = https://github.com/cirg/cirg-tomcat.git
+	url = https://github.com/puppetlabs/puppetlabs-tomcat.git
+	branch = 1.7.0
 [submodule "puppet/modules/locales"]
 	path = puppet/modules/locales
 	url = https://github.com/cirg/puppet-module-locales.git
 [submodule "puppet/modules/vcsrepo"]
 	path = puppet/modules/vcsrepo
-	url = https://github.com/cirg/puppetlabs-vcsrepo.git
+	url = https://github.com/puppetlabs/puppetlabs-vcsrepo.git
+	branch = 1.5.0
 [submodule "puppet/modules/pump"]
 	path = puppet/modules/pump
 	url = http://github.com/cirg/puppet-pump
 [submodule "puppet/modules/exim"]
 	path = puppet/modules/exim
 	url = http://github.com/cirg/puppet-exim
+[submodule "puppet/modules/concat"]
+	path = puppet/modules/concat
+	url = https://github.com/puppetlabs/puppetlabs-concat.git
+	branch = 2.2.1
+[submodule "puppet/modules/staging"]
+	path = puppet/modules/staging
+	url = https://github.com/voxpupuli/puppet-staging.git

I-TECH developer notes.txt

@@ -0,0 +1,17 @@
+Notes for cirg's appliance-setup tool
+
+These are notes I have made when working on these scripts to update for Ubuntu 16.04 Xenial. They are not any sort of official structured document, just things I discovered while learning puppet and the organization of this project.
+
+appliance-setup is a python 2 script that installs puppet and various puppet modules from git, then executes customized puppet scripts, some of which are written by cirg, some are official modules from puppetlabs
+
+The only modifications that should be made by developers to update puppet files are the files contained in appliance-setup/puppet/modules/appliance_components, unless there is no module provided by puppetlabs nor cirg. All of the other modules are imported from puppetlabs or cirg. In some cases the cirg modules should be replaced or updated from puppetlabs official sources. In other words, for example, do not hand-edit files in appliance-setup/puppet/modules/apache to make them work, they're from puppetlabs. Instead, place the customizations you need for apache in the appliance-setup/puppet/modules/appliance_components/manifests/apache.pp puppet file.
+
+xenial does not have python 2 installed by default
+
+xenial's default puppet version is version 3.x - puppetlabs has released puppetlabs 4.x in 2015, and puppet 5.x was released in June 2017.
+
+Several of the puppetlabs modules in git on github at (https://github.com/puppetlabs) have been updated to puppet 4 (and even 5), which can be incompatible with the puppet 3 scripts used by cirg's tool. 
+Finding the appropriate tag or branch involves going through the puppetlabs modules' changelogs for notes on what drops support for puppet 3 and setting .gitmodules branch attribute to the version before puppet 3 support is dropped. 
+This has been done for all cirg modules that were sourced from puppetlabs, and the sources have been updated to puppetlabs' git repositories instead of cirg's forks - this may introduce errors in situations where cirg has modified puppetlabs' modules, but should grant better compatibility across linux versions
+
+The tomcat module from cirg has been replaced with the puppetlabs tomcat module.

bin/appliance-setup

@@ -8,6 +8,8 @@ import shlex
 import stat
 import subprocess
 import sys
+import shutil
+import tomcat_config
 
 BOOTSTRAP_DIR = '/var/lib/appliance-setup'
 BOOTSTRAP_STAMP_FILE = os.path.join(BOOTSTRAP_DIR, 'bootstrap-stamp')
@@ -19,8 +21,11 @@ usage = """
 Usage: %s [OPTION...] <command>
 
 Options:
--h --help     display usage information
--v --verbose  provide verbose output
+-h --help       display usage information
+-v --verbose    provide verbose output
+--webapps=      where to place the webapps directory
+--clientcerts=  true or false to check for clientcerts
+--keystorepass= password for the keystore
 
 Commands:
 apply         configure the appliance
@@ -59,25 +64,30 @@ def bootstrap(force=False):
         os.makedirs(BOOTSTRAP_DIR)
 
     if not os.path.exists(BOOTSTRAP_STAMP_FILE):
-        with open('/etc/apt/sources.list.d/puppetlabs.list', 'w') as \
-            apt_sources:
-            apt_sources.write("""deb http://apt.puppetlabs.com/ squeeze main
-deb-src http://apt.puppetlabs.com/ squeeze main\n""")
-            apt_sources.write("""deb http://apt.puppetlabs.com precise dependencies\n""")
-        subprocess.check_call(shlex.split(
-            'apt-key adv --keyserver keyserver.ubuntu.com --recv 4BD6EC30'))
-        subprocess.check_call(shlex.split('apt-get update'))
-
-        # Install the latest version of Puppet
-        subprocess.check_call(shlex.split(
-            'apt-get -y install facter puppet pwgen python-yaml'))
+        import platform
+        if platform.system() == "Linux":
+            if platform.linux_distribution()[2] != "xenial":
+                with open('/etc/apt/sources.list.d/puppetlabs.list', 'w') as \
+                    apt_sources:
+                    apt_sources.write("""deb http://apt.puppetlabs.com/ squeeze main
+    deb-src http://apt.puppetlabs.com/ squeeze main\n""")
+                    apt_sources.write("""deb http://apt.puppetlabs.com precise dependencies\n""")
+                subprocess.check_call(shlex.split(
+                    'apt-key adv --keyserver keyserver.ubuntu.com --recv 4BD6EC30'))
+
+            subprocess.check_call(shlex.split('apt-get update'))
+
+            # Install the latest version of Puppet and some required python libraries
+            subprocess.check_call(shlex.split(
+                'apt-get -y --force-yes install facter puppet pwgen python-yaml python-lxml'))
 
         with open(BOOTSTRAP_STAMP_FILE, 'w') as stamp_file:
             stamp_file.write(datetime.datetime.now().isoformat() +
             '\n')
 
-def local_config():
+def local_config(options):
     """manages `local.yaml`, the local config file fed to puppet
+
 
     Generates the local configuration file if necessary, merging any
     APPLIANCE_COMPONENTS values with the existing set found.
@@ -86,12 +96,13 @@ def local_config():
     Delete from local.yaml to force a change.
 
     """
+
     # (python-yaml, brought in by bootstrap, should now be available)
     import yaml
 
     # The local and global configuration files
     lcf = os.path.join(script_base_dir,
-                       'puppet/etc/hieradb/local.yaml')
+                               'puppet/etc/hieradb/local.yaml')
     gcf = os.path.join(script_base_dir,
                                'puppet/etc/hieradb/global.yaml')
 
@@ -114,7 +125,32 @@ def local_config():
     # Require at least one component to go on
     if not len(config['classes']):
         available_components(require_one=True)
-
+
+    # Configure webapp directory and whether to use client certs
+    for option,option_value in options:
+        if option == '--webapps':
+            if option_value == 'default':
+                if 'appliance_components::tomcat::web_parent_dir' in config:
+                    config.pop('appliance_components::tomcat::web_parent_dir')
+            else:
+                if os.path.exists(option_value):
+                    config['appliance_components::tomcat::web_parent_dir'] = option_value
+                else:
+                    print("'" + option_value + "' is not a directory.")
+                    print("continuing to use previously defined directory (or default if none exists)")
+        elif option == '--clientcerts':
+            if option_value == 'default':
+                if 'appliance_components::tomcat::use_client_certs' in config:
+                    config.pop('appliance_components::tomcat::use_client_certs')
+            else:
+                if str.lower(option_value) == 'false' or str.lower(option_value) == 'true':
+                    config['appliance_components::tomcat::use_client_certs'] = option_value
+                else:
+                    print("clientcerts value must be true or false. Using previously defined value (false by default)")
+        elif option == '--keystorepass':
+            config['appliance_components::tomcat::keystore_pass'] = option_value
+
+
     # See if a password exists before overwritting
     if 'mysql::server::config_hash' not in config:
         pwgen = subprocess.Popen(shlex.split('pwgen -s 10 -n1'),
@@ -227,6 +263,10 @@ def print_version(output=sys.stdout):
     else:
         print >> output, version
 
+#additional configurations not supported by puppet        
+def config_programs():
+    tomcat_config.config()
+
 def update():
     cmds = ('git fetch',
             'git reset --hard @{u}',
@@ -266,7 +306,10 @@ script_base_dir = os.path.join(os.path.dirname(__file__), '..')
 if __name__ == "__main__":
     try:
         options, args = getopt.getopt(sys.argv[1:], 'hv', ['help',
-                                                           'verbose'])
+                                                           'verbose',
+                                                           'webapps=',
+                                                           'clientcerts=',
+                                                           'keystorepass='])
     except getopt.GetoptError, err:
         exit(error=str(err), status=2)
 
@@ -283,12 +326,13 @@ if __name__ == "__main__":
     arg = args.pop(0)
     if arg == 'apply':
         bootstrap()
-        local_config()
+        local_config(options)
         run_puppet_apply()
+        config_programs()
         print_version(VERSION_FILE)
     elif arg == 'bootstrap':
         bootstrap(force=True)
-        local_config()
+        local_config(options)
     elif arg == 'hiera':
         run_hiera(args)
     elif arg == 'list':
@@ -300,4 +344,4 @@ if __name__ == "__main__":
     elif arg == 'version':
         print_version()
     else:
-        exit(error="Invalid command: %s" % arg, status=2)
+        exit(error="Invalid command: %s" % arg, status=2)

bin/tomcat_config.py

@@ -0,0 +1,136 @@
+import os
+import subprocess
+import shutil
+import time
+
+#This script configs tomcat in additional ways that are
+#missed by puppet's tomcat config
+
+CATALINA_HOME='/usr/share/tomcat8.5'
+
+def config():
+    print('Configuring tomcat...')
+    remove_tomcat_extra_apps()
+    configure_web_xml()
+    configure_catalina_properties()
+    restrict_tomcat_files()
+    enable_tomcat_service()
+    print("Done configuring tomcat")
+
+def remove_tomcat_extra_apps():
+    docs = os.path.join(CATALINA_HOME, 'webapps/docs')
+    examples = os.path.join(CATALINA_HOME, 'webapps/examples')
+    host_manager = os.path.join(CATALINA_HOME, 'webapps/host-manager')
+    manager = os.path.join(CATALINA_HOME, 'webapps/manager')
+    managerXml = os.path.join(CATALINA_HOME, 'conf/Catalina/localhost/manager.xml')
+    print('Deleting unneccessary tomcat files...')
+    if os.path.exists(docs):
+        shutil.rmtree(docs)
+    if os.path.exists(examples):
+        shutil.rmtree(examples)
+    if os.path.exists(host_manager):
+        shutil.rmtree(host_manager)
+    if os.path.exists(manager):
+        shutil.rmtree(manager)
+    if os.path.exists(managerXml):
+        shutil.rmtree(managerXml)
+
+def configure_web_xml():
+    from lxml import etree
+    print('Configuring tomcat web.xml...')
+
+    '''
+  <security-constraint>
+    <web-resource-collection>
+      <web-resource-name>HTTPS ONLY</web-resource-name>
+      <url-pattern>/*</url-pattern>
+    </web-resource-collection>
+    <user-data-constraint>
+      <transport-guarantee>CONFIDENTIAL</transport-guarantee>
+    </user-data-constraint>
+  </security-constraint>
+  <error-page>
+    <exception-type>java.lang.Throwable</exception-type>
+    <location>/error.jsp</location>
+  </error-page>
+    '''
+    parser = etree.XMLParser(remove_blank_text=True)
+    elem_tree = etree.parse(os.path.join(CATALINA_HOME, 'conf/web.xml'), parser)
+
+    for element in elem_tree.xpath("//*[local-name() = 'security-constraint']"):
+        element.getparent().remove(element)
+    for element in elem_tree.xpath("//*[local-name() = 'error-page']"):
+        element.getparent().remove(element)
+
+    #setting security constraint to https only in web.xml
+    sec_con = etree.SubElement(elem_tree.getroot(), 'security-constraint')
+    wrc = etree.SubElement(sec_con, 'web-resource-collection')
+    etree.SubElement(wrc, 'web-resource-name').text = 'HTTPS ONLY'
+    etree.SubElement(wrc, 'url-pattern').text = '/*'
+    udc = etree.SubElement(sec_con, 'user-data-constraint')
+    etree.SubElement(udc, 'transport-guarantee').text = 'CONFIDENTIAL'
+
+    #setting to custom error-page to control stack trace
+    err_page = etree.SubElement(elem_tree.getroot(), 'error-page')
+    etree.SubElement(err_page, 'exception-type').text = 'java.lang.Throwable'
+    etree.SubElement(err_page, 'location').text = '/error.jsp'
+
+    elem_tree.write(os.path.join(CATALINA_HOME, 'conf/web.xml'), pretty_print=True)
+
+def configure_catalina_properties():
+    print('Configuring tomcat catalina.properties...')
+    filename = os.path.join(CATALINA_HOME, 'conf/catalina.properties')
+    f = open(filename, "a+")
+    if not in_file('org.apache.catalina.STRICT_SERVLET_COMPLIANCE=true', filename):
+        f.write('\norg.apache.catalina.STRICT_SERVLET_COMPLIANCE=true')
+    if not in_file('org.apache.catalina.connector.RECYCLE_FACADES=true', filename):
+        f.write('\norg.apache.catalina.connector.RECYCLE_FACADES=true')
+    if not in_file('org.apache.catalina.connector.CoyoteAdapter.ALLOW_BACKSLASH=false', filename):
+        f.write('\norg.apache.catalina.connector.CoyoteAdapter.ALLOW_BACKSLASH=false')
+    if not in_file('org.apache.tomcat.util.buf.UDecoder.ALLOW_ENCODED_SLASH=false', filename):
+        f.write('\norg.apache.tomcat.util.buf.UDecoder.ALLOW_ENCODED_SLASH=false')
+    if not in_file('org.apache.coyote.USE_CUSTOM_STATUS_MSG_IN_HEADER=false', filename):
+        f.write('\norg.apache.coyote.USE_CUSTOM_STATUS_MSG_IN_HEADER=false\n')
+
+def restrict_tomcat_files():
+    print('Restricting tomcat file permissions...')
+    cmd= "chown -RL tomcat:tomcat " + CATALINA_HOME
+    protect_files = subprocess.Popen(cmd.split(),
+                              stdout=None,
+                              stderr=None)
+    status = protect_files.wait()
+
+    cmd= "chmod -R g-w,o-rwx " + CATALINA_HOME
+    protect_files = subprocess.Popen(cmd.split(),
+                              stdout=None,
+                              stderr=None)
+    status = protect_files.wait() 
+
+def enable_tomcat_service():
+    print('Enabling Tomcat as a service...')
+    cmd= "sudo systemctl daemon-reload"
+    reload_daemons = subprocess.Popen(cmd.split(),
+                              stdout=None,
+                              stderr=None)
+    status = reload_daemons.wait()   
+
+    cmd= "sudo systemctl start tomcat.service"
+    start_tomcat = subprocess.Popen(cmd.split(),
+                              stdout=None,
+                              stderr=None)
+    status = start_tomcat.wait()
+
+    cmd= "sudo systemctl enable tomcat.service"
+    enablet_tomcat = subprocess.Popen(cmd.split(),
+                              stdout=None,
+                              stderr=None)
+    status = enablet_tomcat.wait()
+
+def in_file(search_string, filename):
+    f = open(filename)
+    for line in f:
+        if search_string in line:
+            f.close()
+            return True
+    return False
+