Start moving away from search queries to static config items for cookbooks

chef/cookbooks/barclamp/libraries/barclamp_library.rb

@@ -500,5 +500,60 @@ def self.size_to_bytes(s)
         end
       end
     end
+
+    class Config
+      class << self
+        attr_accessor :node
+
+        def load(group, barclamp, instance = nil)
+          # If no instance is specified, see if this node uses an instance of
+          # this barclamp and use it
+          if instance.nil? && @node[barclamp] && @node[barclamp][:config]
+            instance = @node[barclamp][:config][:environment]
+          end
+
+          # Accept environments passed as instances
+          if instance =~ /^#{barclamp}-config-(.*)/
+            instance = $1
+          end
+
+          # Cache the config we load from data bag items.
+          # This cache needs to be invalidated for each chef-client run from
+          # chef-client daemon (which are all in the same process); so use the
+          # ohai time as a marker for that.
+          @cache ||= {}
+
+          if @cache["cache_time"] != @node[:ohai_time]
+            unless @cache["groups"].nil?
+              Chef::Log.info("Invalidating cached config loaded from data bag items")
+            end
+            @cache["groups"] = {}
+            @cache["cache_time"] = @node[:ohai_time]
+          end
+
+          @cache["groups"][group] ||= begin
+            Chef::DataBagItem.load("crowbar-config", group)
+          rescue Net::HTTPServerException
+            {}
+          end
+
+          if instance.nil?
+            # try the "default" instance, and fallback on any existing instance
+            instance = "default"
+            unless @cache["groups"][group].fetch("default", {}).key?(barclamp)
+              # sort to guarantee a consistent order
+              @cache["groups"][group].keys.sort.each do |key|
+                if @cache["groups"][group][key].key?(barclamp)
+                  instance = key
+                  break
+                end
+              end
+            end
+          end
+
+          @cache["groups"][group].fetch(instance, {}).fetch(barclamp, {})
+        end
+      end
+    end
   end
 end

chef/cookbooks/barclamp/recipes/default.rb

@@ -17,3 +17,4 @@ class Chef::Recipe
   include BarclampLibrary
 end
 
+Barclamp::Config.node = node

chef/cookbooks/logging/recipes/client.rb

@@ -18,14 +18,8 @@
 
 include_recipe "logging::common"
 
-env_filter = " AND environment:#{node[:logging][:config][:environment]}"
-servers = search(:node, "roles:logging\\-server#{env_filter}")
-
-if servers.nil?
-  servers = []
-else
-  servers = servers.map { |x| Chef::Recipe::Barclamp::Inventory.get_network_by_type(x, "admin").address }
-end
+logging_config = Barclamp::Config.load("core", "logging")
+servers = logging_config["servers"] || []
 
 # We can't be server and client, so remove server file if we were server before
 # No restart notification, as this file can only exist if the node moves from

chef/cookbooks/network/recipes/default.rb

@@ -90,7 +90,6 @@
   subscribes :run, resources(cookbook_file: "modprobe-bridge.conf"), :delayed
 end
 
-provisioner = search(:node, "roles:provisioner-server")[0]
 conduit_map = Barclamp::Inventory.build_node_map(node)
 Chef::Log.debug("Conduit mapping for this node:  #{conduit_map.inspect}")
 route_pref = 10000
@@ -425,12 +424,17 @@ def kill_nic(nic)
 end
 
 # Wait for the administrative network to come back up.
-Chef::Log.info("Checking we can ping #{provisioner.address.addr}; " +
-               "will wait up to 60 seconds") if provisioner
-60.times do
-  break if ::Kernel.system("ping -c 1 -w 1 -q #{provisioner.address.addr} > /dev/null")
-  sleep 1
-end if provisioner
+provisioner_config = Barclamp::Config.load("core", "provisioner")
+provisioner_address = provisioner_config["server"]
+
+if provisioner_address
+  Chef::Log.info("Checking we can ping #{provisioner_address}; " \
+                 "will wait up to 60 seconds")
+  60.times do
+    break if ::Kernel.system("ping -c 1 -w 1 -q #{provisioner_address} > /dev/null")
+    sleep 1
+  end
+end
 
 node.set["crowbar_wall"] ||= Mash.new
 node.set["crowbar_wall"]["network"] ||= Mash.new

chef/cookbooks/ntp/recipes/default.rb

@@ -15,16 +15,17 @@
 # limitations under the License.
 #
 
+local_admin_address = Barclamp::Inventory.get_network_by_type(node, "admin").address
+
 unless Chef::Config[:solo]
-  env_filter = " AND environment:#{node[:ntp][:config][:environment]}"
-  servers = search(:node, "roles:ntp\\-server#{env_filter}")
+  ntp_config = Barclamp::Config.load("core", "ntp")
+  # duplicate as we may modify it later on to remove our address and to include external servers
+  ntp_servers = ntp_config["servers"].dup || []
 else
-  servers = []
-end
-ntp_servers = []
-servers.each do |n|
-  ntp_servers.push n[:crowbar][:network][:admin][:address] if n.name != node.name
+  ntp_servers = []
 end
+
+ntp_servers.reject! { |n| n == local_admin_address }
 if node["roles"].include?("ntp-server")
   ntp_servers += node[:ntp][:external_servers]
   is_server = true
@@ -65,18 +66,16 @@
 
   user "ntp"
 
-  admin_interface = Chef::Recipe::Barclamp::Inventory.get_network_by_type(node, "admin").address
-
   template "/etc/ntp.conf" do
     owner "root"
     group "root"
     mode 0644
     source "ntp.conf.erb"
     variables(ntp_servers: ntp_servers,
-            admin_interface: admin_interface,
-            is_server: is_server,
-            fudgevalue: 10,
-            driftfile: driftfile)
+              admin_interface: local_admin_address,
+              is_server: is_server,
+              fudgevalue: 10,
+              driftfile: driftfile)
     notifies :restart, "service[ntp]"
   end
 

chef/cookbooks/provisioner/recipes/base.rb

@@ -219,8 +219,8 @@
   admin_ip = Chef::Recipe::Barclamp::Inventory.get_network_by_type(provisioner_server_node, "admin").address
   web_port = provisioner_server_node[:provisioner][:web_port]
 
-  ntp_servers = search(:node, "roles:ntp-server")
-  ntp_servers_ips = ntp_servers.map { |n| Chef::Recipe::Barclamp::Inventory.get_network_by_type(n, "admin").address }
+  ntp_config = Barclamp::Config.load("core", "ntp")
+  ntp_servers = ntp_config["servers"] || []
 
   template "/usr/sbin/crowbar_join" do
     mode 0755
@@ -229,7 +229,7 @@
     source "crowbar_join.suse.sh.erb"
     variables(admin_ip: admin_ip,
               web_port: web_port,
-              ntp_servers_ips: ntp_servers_ips,
+              ntp_servers_ips: ntp_servers,
               target_platform_version: node["platform_version"] )
   end
 

chef/cookbooks/provisioner/recipes/setup_base_images.rb

@@ -509,8 +509,8 @@
       # Add base OS install repo for suse
       node.set[:provisioner][:repositories][os][arch]["base"] = { "baseurl=#{admin_web}" => true }
 
-      ntp_servers = search(:node, "roles:ntp-server")
-      ntp_servers_ips = ntp_servers.map { |n| Chef::Recipe::Barclamp::Inventory.get_network_by_type(n, "admin").address }
+      ntp_config = Barclamp::Config.load("core", "ntp")
+      ntp_servers = ntp_config["servers"] || []
 
       target_platform_distro = os.gsub(/-.*$/, "")
       target_platform_version = os.gsub(/^.*-/, "")
@@ -522,7 +522,7 @@
         source "crowbar_join.suse.sh.erb"
         variables(admin_ip: admin_ip,
                   web_port: web_port,
-                  ntp_servers_ips: ntp_servers_ips,
+                  ntp_servers_ips: ntp_servers,
                   platform: target_platform_distro,
                   target_platform_version: target_platform_version)
       end
@@ -541,7 +541,7 @@
         variables(admin_ip: admin_ip,
                   admin_broadcast: admin_broadcast,
                   web_port: web_port,
-                  ntp_servers_ips: ntp_servers_ips,
+                  ntp_servers_ips: ntp_servers,
                   os: os,
                   arch: arch,
                   crowbar_key: crowbar_key,

chef/cookbooks/repos/recipes/default.rb

@@ -13,8 +13,18 @@
 # limitations under the License.
 #
 
-provisioners = search(:node, "roles:provisioner-server")
-provisioner = provisioners[0] if provisioners
+return if node[:platform_family] == "suse" || node[:platform_family] == "windows"
+
+# we still need to search for the provisioner node to get access to the
+# repositories, which are attributes set from the cookbook
+
+# no need to have a fallback as this recipe is run as part of the provisioner-base role
+provisioner_instance = CrowbarHelper.get_proposal_instance(node, "provisioner")
+provisioners = node_search_with_cache("roles:provisioner-server", provisioner_instance)
+provisioner = provisioners.first if provisioners
+
+provisioner_config = Barclamp::Config.load("core", "provisioner")
+
 os_token = "#{node[:platform]}-#{node[:platform_version]}"
 arch = node[:kernel][:machine]
 
@@ -24,10 +34,7 @@
   action :nothing
 end
 
-if provisioner and !CrowbarHelper.in_sledgehammer?(node)
-  web_port = provisioner["provisioner"]["web_port"]
-  address = Chef::Recipe::Barclamp::Inventory.get_network_by_type(provisioner, "admin").address
-
+if provisioner && !provisioner_config.empty? && !CrowbarHelper.in_sledgehammer?(node)
   case node[:platform_family]
   when "debian"
     repositories = provisioner["provisioner"]["repositories"][os_token][arch]
@@ -83,7 +90,7 @@
 
   if node[:platform_family] != "suse" && node[:platform_family] != "windows"
     template "/etc/gemrc" do
-      variables(admin_ip: address, web_port: web_port)
+      variables(root_url: provisioner_config["root_url"])
       mode "0644"
     end
   end

chef/cookbooks/repos/templates/default/gemrc.erb

@@ -1,3 +1,3 @@
 :sources:
-- http://<%=@admin_ip%>:<%=@web_port%>/gemsite/
+- <%= @root_url %>/gemsite/
 gem: --no-ri --no-rdoc --bindir /usr/local/bin

chef/cookbooks/resolver/recipes/default.rb

@@ -18,20 +18,14 @@
 # limitations under the License.
 #
 
-env_filter = " AND dns_config_environment:#{node[:dns][:config][:environment]}"
-nodes = search(:node, "roles:dns-server#{env_filter}")
-
-dns_list = []
-if !nodes.nil? and !nodes.empty?
-  dns_list = nodes.map { |x| Chef::Recipe::Barclamp::Inventory.get_network_by_type(x, "admin").address }
-  dns_list.sort!
-elsif !node["crowbar"].nil? and node["crowbar"]["admin_node"] and !node[:dns][:forwarders].nil?
-  dns_list << node[:dns][:forwarders]
+dns_config = Barclamp::Config.load("core", "dns")
+dns_list = dns_config["servers"] || []
+if dns_list.empty? && \
+    !node["crowbar"].nil? && node["crowbar"]["admin_node"] && \
+    !node[:dns][:forwarders].nil?
+  dns_list = (node[:dns][:forwarders] + node[:dns][:nameservers]).flatten.compact
 end
 
-dns_list << node[:dns][:nameservers]
-dns_list.flatten!
-
 unless node[:platform_family] == "windows"
   unless CrowbarHelper.in_sledgehammer?(node)
     package "dnsmasq"
@@ -41,8 +35,7 @@
       owner "root"
       group "root"
       mode 0644
-      # do a dup, because we'll insert 127.0.0.1 later on
-      variables(nameservers: dns_list.dup)
+      variables(nameservers: dns_list)
     end
 
     file "/etc/resolv-forwarders.conf" do
@@ -59,15 +52,15 @@
       end
       not_if { node["crowbar"]["admin_node"] && ::File.exist?("/var/lib/crowbar/install/disable_dns") }
     end
-
-    dns_list = dns_list.insert(0, "127.0.0.1").take(3)
   end
+  # do a dup because we modify the content
+  dns_list_with_local = dns_list.dup.insert(0, "127.0.0.1").take(3)
 
   template "/etc/resolv.conf" do
     source "resolv.conf.erb"
     owner "root"
     group "root"
     mode 0644
-    variables(nameservers: dns_list, search: node[:dns][:domain])
+    variables(nameservers: dns_list_with_local, search: node[:dns][:domain])
   end
 end

chef/cookbooks/uwsgi/providers/default.rb

@@ -3,11 +3,10 @@
   package "python-pip"
   package "python-dev"
 
-  provisioner = search(:node, "roles:provisioner-server").first
-  proxy_addr = provisioner[:fqdn]
-  proxy_port = provisioner[:provisioner][:web_port]
+  provisioner_config = BarclampLibrary::Barclamp::Config.load("core", "provisioner")
+  index_url = "#{provisioner_config['root_url']}/files/pip_cache/simple/"
 
-  execute "pip install --index-url http://#{proxy_addr}:#{proxy_port}/files/pip_cache/simple/ uwsgi" do
+  execute "pip install --index-url #{index_url} uwsgi" do
     not_if "pip freeze 2>&1 | grep -i uwsgi"
   end
 

crowbar_framework/app/models/dns_service.rb

@@ -149,6 +149,36 @@ def apply_role_pre_chef_call(old_role, role, all_nodes)
       end
     end
 
+    save_config_to_databag(old_role, role, nodes)
+
     @logger.debug("DNS apply_role_pre_chef_call: leaving")
   end
+
+  def save_config_to_databag(old_role, role, server_nodes = nil)
+    if role.nil?
+      config = nil
+    else
+      if server_nodes.nil?
+        server_nodes_names = role.override_attributes["dns"]["elements"]["dns-server"]
+        server_nodes = server_nodes_names.map { |n| NodeObject.find_node_by_name n }
+      end
+
+      addresses = server_nodes.map do |n|
+        admin_net = n.get_network_by_type("admin")
+        # admin_net may be nil in the bootstrap case, because admin server only
+        # gets its IP on hardware-installing, which is after this is first
+        # called
+        admin_net["address"] unless admin_net.nil?
+      end
+      addresses.sort!.compact!
+
+      addresses.concat(role.default_attributes["dns"]["nameservers"] || [])
+      addresses = addresses.flatten.compact
+
+      config = { servers: addresses }
+    end
+
+    instance = Crowbar::DataBagConfig.instance_from_role(old_role, role)
+    Crowbar::DataBagConfig.save("core", instance, @bc_name, config)
+  end
 end