Aci 4.1 soc7 multivmm

.travis.yml

@@ -5,6 +5,9 @@ dist: trusty
 
 rvm: 2.1.9
 
+before_install:
+  - rvm @global do gem install bundler -v '< 2.0.0'
+
 matrix:
   include:
     - env: SYNTAXCHECK
@@ -13,3 +16,5 @@ matrix:
     - env: SPEC_TESTS
       script:
       - bundle exec rake spec
+    - name: "Databag testing"
+      script: bundle exec crowbar-validate-databags chef/data_bags/crowbar

Gemfile

@@ -18,6 +18,7 @@
 source "https://rubygems.org"
 
 group :development do
+  gem "crowbar-validate-databags"
   gem "rake", "< 12.0.0"
   gem "uglifier", "~> 2.7.2"
   gem "sass", "~> 3.2.19"

bin/prepare-mariadb

@@ -0,0 +1,143 @@
+#!/usr/bin/env ruby
+#
+# Copyright 2018, SUSE
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+$LOAD_PATH.unshift(File.expand_path("../../crowbar_framework/lib", __FILE__))
+$LOAD_PATH.unshift(File.expand_path("../../chef/cookbooks/database/libraries", __FILE__))
+
+require "chef"
+require "erb"
+require "crowbar"
+
+RECIPE = "recipe[database::pg2mariadb_preparation]"
+LOGDIR = "/var/log/crowbar"
+
+def chef_init
+  Chef::Config.node_name "crowbar"
+  Chef::Config.client_key "/opt/dell/crowbar_framework/config/client.pem"
+  Chef::Config.chef_server_url "http://localhost:4000"
+end
+
+def node_for_role(role)
+  nodes = []
+  Chef::Search::Query.new.search "node", "roles:#{role}" do |n|
+    nodes << n
+  end
+  nodes.sort_by! { |n| n.name }
+  nodes.empty? ? nil : nodes.first
+end
+
+def mysql_node
+    node_for_role "mysql-server"
+end
+
+# Select nodes which represent all services which use database
+# they might be standalone nodes or cluster members. In most simple
+# case there will be only one node used for all services.
+def selected_nodes(first_mysql_node)
+  nodes_and_roles = {
+    first_mysql_node.name => {
+      node: first_mysql_node, roles: ["mysql-server"]
+    }
+  }
+  CrowbarDatabaseHelper.roles_using_database.each do |role|
+    node = node_for_role(role)
+    next if node.nil?
+    if nodes_and_roles.include? node.name
+      nodes_and_roles[node.name][:roles] << role
+    else
+      nodes_and_roles[node.name] = { node: node, roles: [role] }
+    end
+  end
+  nodes_and_roles
+end
+
+def node_role(node)
+    Chef::Role.load("crowbar-"+node.name.gsub(".", "_"))
+end
+
+def add_recipe(node)
+  role = node_role(node)
+  role.run_list << RECIPE
+  role.save
+end
+
+def remove_recipe(node)
+  role = node_role(node)
+  role.run_list.remove(RECIPE)
+  role.save
+end
+
+# based on code from crowbar_framework/app/models/node.rb
+def run_ssh_cmd(node, cmd, log_suffix = nil)
+  log_file = "/var/log/crowbar/db-prepare.#{log_suffix}.log" if log_suffix
+  log_redirect = "> #{log_file} 2>&1" if log_file
+  start_time = Time.now
+  args = ["sudo", "-i", "-u", "root", "--",
+          "ssh", "-o", "ConnectTimeout=10",
+          "root@#{node.name}",
+          %("#{cmd.gsub('"', '\\"')} #{log_redirect}")
+  ].join(" ")
+  log "Log: #{log_file} on #{node.name}"
+  Open3.popen2e(args) do |stdin, stdout_and_stderr, wait_thr|
+    {
+      stdout_and_stderr: stdout_and_stderr.gets(nil),
+      exit_code: wait_thr.value.exitstatus,
+      run_time: Time.now - start_time
+    }
+  end
+end
+
+def log(msg)
+  print "#{msg}\n"
+end
+
+def prepare_node(node, roles)
+    log "Preparing node #{node.name}"
+    log "Adding #{RECIPE} to run_list"
+    add_recipe node
+    log "Running chef-client on #{node.name}..."
+    res = run_ssh_cmd(node, "chef-client", "chef-client")
+    log "Run time: #{res[:run_time]}s"
+    log "Removing #{RECIPE} from run_list"
+    remove_recipe node
+    unless res[:exit_code].zero?
+      log "ERROR: Chef-client failed with code: #{res[:exit_code]}"
+      return -2
+    end
+    log "Prepare completed for #{node.name}"
+    0
+end
+
+def main
+  ret = 0
+  chef_init
+  first_mysql_node = mysql_node
+  if first_mysql_node.nil?
+    log "ERROR: MySQL server not found. Please assign mysql-server role " \
+      "to some node or cluster and re-apply database proposal."
+    return -1
+  end
+  selected_nodes(first_mysql_node).values.each do |node_data|
+    node_ret = prepare_node(node_data[:node], node_data[:roles])
+    ret = node_ret if ret.zero? && !node_ret.zero?
+    log "Summary of used databases: /etc/pg2mysql/databases.yaml on " \
+      "#{first_mysql_node.name}" if node_data[:node] == first_mysql_node && ret.zero?
+  end
+  return ret
+end
+
+exit(main)

chef/cookbooks/aodh/attributes/default.rb

@@ -55,6 +55,7 @@
 default[:aodh][:notifier][:service_name]  = notifier_service_name
 default[:aodh][:listener][:service_name]  = listener_service_name
 default[:aodh][:evaluation_interval] = 600
+default[:aodh][:alarm_history_ttl] = -1
 
 default[:aodh][:debug] = false
 default[:aodh][:verbose] = false

chef/cookbooks/aodh/recipes/aodh.rb

@@ -145,9 +145,6 @@
   bind_port = node[:aodh][:api][:port]
 end
 
-node.normal[:apache][:listen_ports_crowbar] ||= {}
-node.normal[:apache][:listen_ports_crowbar][:aodh] = { plain: [bind_port] }
-
 template node[:aodh][:config_file] do
   source "aodh.conf.erb"
   owner "root"
@@ -163,7 +160,8 @@
     database_connection: db_connection,
     node_hostname: node["hostname"],
     aodh_ssl: node[:aodh][:ssl],
-    evaluation_interval: node[:aodh][:evaluation_interval]
+    evaluation_interval: node[:aodh][:evaluation_interval],
+    alarm_history_ttl: node[:aodh][:alarm_history_ttl]
   )
   notifies :reload, resources(service: "apache2")
 end

chef/cookbooks/aodh/templates/default/aodh.conf.erb

@@ -9,6 +9,7 @@ transport_url = <%= @rabbit_settings[:url] %>
 workers = <%= [node["cpu"]["total"], 2, 4].sort[1] %>
 
 [database]
+alarm_history_time_to_live = <%= @alarm_history_ttl %>
 connection = <%= @database_connection %>
 
 [keystone_authtoken]

chef/cookbooks/barbican/recipes/api.rb

@@ -112,7 +112,7 @@
   action :add_user
 end
 
-keystone_register "give barbican user access" do
+keystone_register "give barbican user access as admin" do
   protocol keystone_settings["protocol"]
   insecure keystone_settings["insecure"]
   host keystone_settings["internal_url_host"]
@@ -124,6 +124,94 @@
   action :add_access
 end
 
+keystone_register "add key-manager:service-admin role for barbican" do
+  protocol keystone_settings["protocol"]
+  insecure keystone_settings["insecure"]
+  host keystone_settings["internal_url_host"]
+  port keystone_settings["admin_port"]
+  auth register_auth_hash
+  role_name "key-manager:service-admin"
+  action :add_role
+end
+
+keystone_register "give barbican user access as key-manager:service-admin" do
+  protocol keystone_settings["protocol"]
+  insecure keystone_settings["insecure"]
+  host keystone_settings["internal_url_host"]
+  port keystone_settings["admin_port"]
+  auth register_auth_hash
+  user_name keystone_settings["service_user"]
+  tenant_name keystone_settings["service_tenant"]
+  role_name "key-manager:service-admin"
+  action :add_access
+end
+
+keystone_register "add creator role for barbican" do
+  protocol keystone_settings["protocol"]
+  insecure keystone_settings["insecure"]
+  host keystone_settings["internal_url_host"]
+  port keystone_settings["admin_port"]
+  auth register_auth_hash
+  role_name "creator"
+  action :add_role
+end
+
+keystone_register "give barbican user access as creator" do
+  protocol keystone_settings["protocol"]
+  insecure keystone_settings["insecure"]
+  host keystone_settings["internal_url_host"]
+  port keystone_settings["admin_port"]
+  auth register_auth_hash
+  user_name keystone_settings["service_user"]
+  tenant_name keystone_settings["service_tenant"]
+  role_name "creator"
+  action :add_access
+end
+
+keystone_register "add observer role for barbican" do
+  protocol keystone_settings["protocol"]
+  insecure keystone_settings["insecure"]
+  host keystone_settings["internal_url_host"]
+  port keystone_settings["admin_port"]
+  auth register_auth_hash
+  role_name "observer"
+  action :add_role
+end
+
+keystone_register "give barbican user access as observer" do
+  protocol keystone_settings["protocol"]
+  insecure keystone_settings["insecure"]
+  host keystone_settings["internal_url_host"]
+  port keystone_settings["admin_port"]
+  auth register_auth_hash
+  user_name keystone_settings["service_user"]
+  tenant_name keystone_settings["service_tenant"]
+  role_name "observer"
+  action :add_access
+end
+
+keystone_register "add audit role for barbican" do
+  protocol keystone_settings["protocol"]
+  insecure keystone_settings["insecure"]
+  host keystone_settings["internal_url_host"]
+  port keystone_settings["admin_port"]
+  auth register_auth_hash
+  role_name "audit"
+  action :add_role
+end
+
+keystone_register "give barbican user access as audit" do
+  protocol keystone_settings["protocol"]
+  insecure keystone_settings["insecure"]
+  host keystone_settings["internal_url_host"]
+  port keystone_settings["admin_port"]
+  auth register_auth_hash
+  user_name keystone_settings["service_user"]
+  tenant_name keystone_settings["service_tenant"]
+  role_name "audit"
+  action :add_access
+end
+
 crowbar_pacemaker_sync_mark "create-barbican_register" if ha_enabled
 
 if node[:barbican][:ha][:enabled]

chef/cookbooks/ceilometer/attributes/default.rb

@@ -78,6 +78,7 @@
 default[:ceilometer][:ha][:agent_notification][:agent] = "systemd:#{agent_notification_service_name}"
 default[:ceilometer][:ha][:agent_notification][:op][:monitor][:interval] = "10s"
 
+
 default[:ceilometer][:ha][:central][:enabled] = false
 default[:ceilometer][:ha][:central][:agent] = "systemd:#{central_service_name}"
 default[:ceilometer][:ha][:central][:op][:monitor][:interval] = "10s"
@@ -90,3 +91,7 @@
 # this establishes which node is used for mongo client connections that
 # we use to initialize the replica set
 default[:ceilometer][:ha][:mongodb][:replica_set][:controller] = false
+
+# Pacemaker ceilometer expirer cronjob link
+default[:ceilometer][:ha][:expirer][:cronjob][:agent] = "ocf:heartbeat:symlink"
+default[:ceilometer][:ha][:expirer][:cronjob][:op][:monitor][:interval] = "10s"

chef/cookbooks/ceilometer/recipes/server.rb

@@ -207,9 +207,6 @@
   bind_port = node[:ceilometer][:api][:port]
 end
 
-node.normal[:apache][:listen_ports_crowbar] ||= {}
-node.normal[:apache][:listen_ports_crowbar][:ceilometer] = { plain: [bind_port] }
-
 if ceilometer_protocol == "https"
   ssl_setup "setting up ssl for ceilometer" do
     generate_certs node[:ceilometer][:ssl][:generate_certs]

chef/cookbooks/ceilometer/recipes/server_ha.rb

@@ -23,6 +23,43 @@
   action :nothing
 end.run_action(:create)
 
+# install openstack-ceilometer-collector - the package contains the cron file
+# /usr/share/ceilometer/openstack-ceilometer-expirer.cron
+package "openstack-ceilometer-collector"
+
+# setup the expirer cronjob only on a single node to not
+# run into DB deadlocks (bsc#1113107)
+crowbar_pacemaker_sync_mark "wait-ceilometer_expirer_cron"
+
+expirer_transaction_objects = []
+
+ceilometer_expirer_cron_primitive = "ceilometer-expirer-cron"
+pacemaker_primitive ceilometer_expirer_cron_primitive do
+  agent node[:ceilometer][:ha][:expirer][:cronjob][:agent]
+  params(
+    # target is from the RPM package openstack-ceilometer
+    "target" => "/usr/share/ceilometer/openstack-ceilometer-expirer.cron",
+    "link" => "/etc/cron.daily/openstack-ceilometer-expirer.cron",
+    "backup_suffix" => ".orig"
+  )
+  op node[:ceilometer][:ha][:expirer][:cronjob][:op]
+  action :update
+  only_if { CrowbarPacemakerHelper.is_cluster_founder?(node) }
+end
+expirer_transaction_objects << "pacemaker_primitive[#{ceilometer_expirer_cron_primitive}]"
+
+ceilometer_expirer_cron_loc = openstack_pacemaker_controller_only_location_for ceilometer_expirer_cron_primitive
+expirer_transaction_objects << "pacemaker_location[#{ceilometer_expirer_cron_loc}]"
+
+pacemaker_transaction "ceilometer-expirer cron" do
+  cib_objects expirer_transaction_objects
+  # note that this will also automatically start the resources
+  action :commit_new
+  only_if { CrowbarPacemakerHelper.is_cluster_founder?(node) }
+end
+
+crowbar_pacemaker_sync_mark "create-ceilometer_expirer_cron"
+
 if node[:pacemaker][:clone_stateless_services]
   # Wait for all nodes to reach this point so we know that they will have
   # all the required packages installed and configuration files updated

chef/cookbooks/cinder/recipes/api.rb

@@ -31,6 +31,18 @@
 my_admin_host = CrowbarHelper.get_host_for_admin_url(node, ha_enabled)
 my_public_host = CrowbarHelper.get_host_for_public_url(node, node[:cinder][:api][:protocol] == "https", ha_enabled)
 
+if node[:cinder][:api][:protocol] == "https"
+  ssl_setup "setting up ssl for cinder" do
+    generate_certs node[:cinder][:ssl][:generate_certs]
+    certfile node[:cinder][:ssl][:certfile]
+    keyfile node[:cinder][:ssl][:keyfile]
+    group node[:cinder][:group]
+    fqdn node[:fqdn]
+    cert_required node[:cinder][:ssl][:cert_required]
+    ca_certs node[:cinder][:ssl][:ca_certs]
+  end
+end
+
 crowbar_pacemaker_sync_mark "wait-cinder_register"
 
 register_auth_hash = { user: keystone_settings["admin_user"],