From 4f97cdb72668a85055fd19164a79702ff02230ff Mon Sep 17 00:00:00 2001 From: Ladislav Smola Date: Wed, 22 Aug 2018 17:44:47 +0200 Subject: [PATCH 1/9] Add refresh of service catalog entities Add refresh of service catalog entities --- .../kubernetes/container_manager_mixin.rb | 9 ++ .../inventory/collector/container_manager.rb | 30 +++++ .../inventory/parser/container_manager.rb | 105 ++++++++++++++++++ .../definitions/container_collections.rb | 3 + 4 files changed, 147 insertions(+) diff --git a/app/models/manageiq/providers/kubernetes/container_manager_mixin.rb b/app/models/manageiq/providers/kubernetes/container_manager_mixin.rb index b0af8223d8..398d326b6d 100644 --- a/app/models/manageiq/providers/kubernetes/container_manager_mixin.rb +++ b/app/models/manageiq/providers/kubernetes/container_manager_mixin.rb @@ -68,6 +68,15 @@ def kubernetes_auth_options(options) def kubernetes_version 'v1' end + + def kubernetes_service_catalog_connect(hostname, port, options) + options = {:path => '/apis/servicecatalog.k8s.io', :version => service_catalog_api_version}.merge(options) + kubernetes_connect(hostname, port, options) + end + + def service_catalog_api_version + 'v1beta1' + end end PERF_ROLLUP_CHILDREN = :container_nodes diff --git a/app/models/manageiq/providers/kubernetes/inventory/collector/container_manager.rb b/app/models/manageiq/providers/kubernetes/inventory/collector/container_manager.rb index de8e8ad3ef..328d6d31e2 100644 --- a/app/models/manageiq/providers/kubernetes/inventory/collector/container_manager.rb +++ b/app/models/manageiq/providers/kubernetes/inventory/collector/container_manager.rb @@ -7,9 +7,39 @@ def pods @pods ||= connection.get_pods end + def cluster_service_classes + entities_iterator(service_catalog_connection, :cluster_service_classes) + end + + def cluster_service_plans + entities_iterator(service_catalog_connection, :cluster_service_plans) + end + + def service_instances + entities_iterator(service_catalog_connection, :service_instances) + end + private def connection @connection ||= manager.connect(:service => "kubernetes") end + + def service_catalog_connection + @service_catalog_connection ||= manager.connect(:service => "kubernetes_service_catalog") + end + + def entities_iterator(client, entity) + # TODO(lsmola) change to iterator, that will fetch paginated response from the server, never fetching everything + # at once + + begin + client.send("get_#{entity}") + rescue KubeException => e + # TODO(lsmola) the old refresh has entities that can throws error and return some :default set + _log.error("Unexpected Exception during fetching of entity #{entity}: #{e}") + _log.error(e.backtrace.join("\n")) + [] + end + end end diff --git a/app/models/manageiq/providers/kubernetes/inventory/parser/container_manager.rb b/app/models/manageiq/providers/kubernetes/inventory/parser/container_manager.rb index 10c12b9783..caf437e91f 100644 --- a/app/models/manageiq/providers/kubernetes/inventory/parser/container_manager.rb +++ b/app/models/manageiq/providers/kubernetes/inventory/parser/container_manager.rb @@ -2,6 +2,11 @@ class ManageIQ::Providers::Kubernetes::Inventory::Parser::ContainerManager < Man def parse parse_namespaces(collector.namespaces) parse_pods(collector.pods) + + # Service catalog entities + parse_container_service_classes(collector.cluster_service_classes) + parse_container_service_instances(collector.service_instances) + parse_container_service_plans(collector.cluster_service_plans) end private @@ -34,6 +39,106 @@ def parse_pod(pod) ) end + # def container_service_bindings(service_bindings) + # service_bindings.each do |service_binding| + # persister.container_service_bindings.build( + # parse_base_item(service_binding).except(:namespace).merge( + # :container_project => lazy_find_project(service_binding), + # :container_service_instance => persister.container_service_instances.lazy_find( + # service_binding.spec.serviceInstanceRef.name + # ), + # :extra => { + # :spec => service_binding.spec, + # :status => service_binding.status + # } + # ) + # ) + # end + # end + + # def container_service_brokers(service_brokers) + # service_brokers.each do |service_broker| + # persister.container_service_brokers.build( + # parse_base_item(service_broker).except(:namespace).merge( + # :url => service_broker.spec.url, + # :container_project => lazy_find_project(service_broker), + # :extra => {:status => service_broker.status} + # ) + # ) + # end + # end + + def parse_container_service_classes(service_classes) + service_classes.each do |service_class| + parse_container_service_class(service_class) + end + end + + def parse_container_service_class(service_class) + persister.container_service_classes.build( + parse_base_item(service_class).except(:namespace, :ems_ref, :name).merge( + :name => service_class.spec.externalName, + :ems_ref => service_class.spec.externalID, + :description => service_class.spec.description, + :container_project => lazy_find_project(service_class), + :extra => { + :spec => service_class.spec, + :status => service_class.status + } + ) + ) + end + + def parse_container_service_instances(service_instances) + service_instances.each do |service_instance| + parse_container_service_instance(service_instance) + end + end + + def parse_container_service_instance(service_instance) + persister.container_service_instances.build( + parse_base_item(service_instance).except(:namespace, :ems_ref).merge( + :ems_ref => service_instance.spec.externalID, + :generate_name => service_instance.metadata.generate_name, + :container_project => lazy_find_project(service_instance), + :container_service_class => persister.container_service_classes.lazy_find( + service_instance.spec.clusterServiceClassRef.name + ), + :container_service_plan => persister.container_service_plans.lazy_find( + service_instance.spec.clusterServicePlanRef.name + ), + :extra => { + :spec => service_instance.spec, + :status => service_instance.status + } + ) + ) + end + + def parse_container_service_plans(service_plans) + service_plans.each do |service_plan| + parse_container_service_plan(service_plan) + end + end + + def parse_container_service_plan(service_plan) + persister.container_service_plans.build( + parse_base_item(service_plan).except(:namespace, :ems_ref, :name).merge( + :name => service_plan.spec.externalName, + :ems_ref => service_plan.spec.externalID, + :description => service_plan.spec.description, + :container_project => lazy_find_project(service_plan), + :container_service_class => persister.container_service_classes.lazy_find( + service_plan.spec.clusterServiceClassRef.name + ), + :extra => { + :spec => service_plan.spec, + :status => service_plan.status + } + ) + ) + end + def parse_base_item(item) { :ems_ref => item.metadata.uid, diff --git a/app/models/manageiq/providers/kubernetes/inventory/persister/definitions/container_collections.rb b/app/models/manageiq/providers/kubernetes/inventory/persister/definitions/container_collections.rb index d897439ee2..dcfc8b7ddc 100644 --- a/app/models/manageiq/providers/kubernetes/inventory/persister/definitions/container_collections.rb +++ b/app/models/manageiq/providers/kubernetes/inventory/persister/definitions/container_collections.rb @@ -20,6 +20,9 @@ def initialize_container_inventory_collections container_volumes container_replicators container_routes + container_service_classes + container_service_instances + container_service_plans container_services container_service_port_configs container_templates From 112d561547c56b1a8a8689bbfc45e8a16d824252 Mon Sep 17 00:00:00 2001 From: Ladislav Smola Date: Wed, 22 Aug 2018 17:45:25 +0200 Subject: [PATCH 2/9] Remove broker and bindings --- .../inventory/parser/container_manager.rb | 29 ------------------- 1 file changed, 29 deletions(-) diff --git a/app/models/manageiq/providers/kubernetes/inventory/parser/container_manager.rb b/app/models/manageiq/providers/kubernetes/inventory/parser/container_manager.rb index caf437e91f..b6c801982b 100644 --- a/app/models/manageiq/providers/kubernetes/inventory/parser/container_manager.rb +++ b/app/models/manageiq/providers/kubernetes/inventory/parser/container_manager.rb @@ -39,35 +39,6 @@ def parse_pod(pod) ) end - # def container_service_bindings(service_bindings) - # service_bindings.each do |service_binding| - # persister.container_service_bindings.build( - # parse_base_item(service_binding).except(:namespace).merge( - # :container_project => lazy_find_project(service_binding), - # :container_service_instance => persister.container_service_instances.lazy_find( - # service_binding.spec.serviceInstanceRef.name - # ), - # :extra => { - # :spec => service_binding.spec, - # :status => service_binding.status - # } - # ) - # ) - # end - # end - - # def container_service_brokers(service_brokers) - # service_brokers.each do |service_broker| - # persister.container_service_brokers.build( - # parse_base_item(service_broker).except(:namespace).merge( - # :url => service_broker.spec.url, - # :container_project => lazy_find_project(service_broker), - # :extra => {:status => service_broker.status} - # ) - # ) - # end - # end - def parse_container_service_classes(service_classes) service_classes.each do |service_class| parse_container_service_class(service_class) From b061c45f1540e41435010c6c4e73939cc89ed820 Mon Sep 17 00:00:00 2001 From: Ladislav Smola Date: Thu, 23 Aug 2018 10:44:06 +0200 Subject: [PATCH 3/9] Cleanup entities iterator and use it for every collection Cleanup entities iterator and use it for every collection --- .../inventory/collector/container_manager.rb | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/app/models/manageiq/providers/kubernetes/inventory/collector/container_manager.rb b/app/models/manageiq/providers/kubernetes/inventory/collector/container_manager.rb index 328d6d31e2..9fb45ea6c1 100644 --- a/app/models/manageiq/providers/kubernetes/inventory/collector/container_manager.rb +++ b/app/models/manageiq/providers/kubernetes/inventory/collector/container_manager.rb @@ -1,10 +1,10 @@ class ManageIQ::Providers::Kubernetes::Inventory::Collector::ContainerManager < ManageIQ::Providers::Kubernetes::Inventory::Collector def namespaces - @namespaces ||= connection.get_namespaces + entities_iterator(connection, :namespaces) end def pods - @pods ||= connection.get_pods + entities_iterator(connection, :pods) end def cluster_service_classes @@ -32,14 +32,11 @@ def service_catalog_connection def entities_iterator(client, entity) # TODO(lsmola) change to iterator, that will fetch paginated response from the server, never fetching everything # at once - - begin - client.send("get_#{entity}") - rescue KubeException => e - # TODO(lsmola) the old refresh has entities that can throws error and return some :default set - _log.error("Unexpected Exception during fetching of entity #{entity}: #{e}") - _log.error(e.backtrace.join("\n")) - [] - end + client.send("get_#{entity}") + rescue KubeException => e + # TODO(lsmola) the old refresh has entities that can throws error and return some :default set + _log.error("Unexpected Exception during fetching of entity #{entity}: #{e}") + _log.error(e.backtrace.join("\n")) + [] end end From f1fadc1048239997493abd672fae205c452c8197 Mon Sep 17 00:00:00 2001 From: Ladislav Smola Date: Thu, 23 Aug 2018 11:18:11 +0200 Subject: [PATCH 4/9] Memoize the collections for now Memoize the collections for now, since we are getting the resourceVersion from it later. --- .../inventory/collector/container_manager.rb | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/app/models/manageiq/providers/kubernetes/inventory/collector/container_manager.rb b/app/models/manageiq/providers/kubernetes/inventory/collector/container_manager.rb index 9fb45ea6c1..5860305913 100644 --- a/app/models/manageiq/providers/kubernetes/inventory/collector/container_manager.rb +++ b/app/models/manageiq/providers/kubernetes/inventory/collector/container_manager.rb @@ -1,22 +1,22 @@ class ManageIQ::Providers::Kubernetes::Inventory::Collector::ContainerManager < ManageIQ::Providers::Kubernetes::Inventory::Collector def namespaces - entities_iterator(connection, :namespaces) + @namespaces ||= entities_iterator(connection, :namespaces) end def pods - entities_iterator(connection, :pods) + @pods ||= entities_iterator(connection, :pods) end def cluster_service_classes - entities_iterator(service_catalog_connection, :cluster_service_classes) + @cluster_service_classes ||= entities_iterator(service_catalog_connection, :cluster_service_classes) end def cluster_service_plans - entities_iterator(service_catalog_connection, :cluster_service_plans) + @cluster_service_plans ||= entities_iterator(service_catalog_connection, :cluster_service_plans) end def service_instances - entities_iterator(service_catalog_connection, :service_instances) + @service_instances ||= entities_iterator(service_catalog_connection, :service_instances) end private From c6c98f42b917a80a1dce8979e3607e5b5372dfed Mon Sep 17 00:00:00 2001 From: Ladislav Smola Date: Fri, 24 Aug 2018 16:17:02 +0200 Subject: [PATCH 5/9] Rename ServiceClass to ServiceOffering --- .../inventory/collector/container_manager.rb | 4 +- .../inventory/parser/container_manager.rb | 96 ++++++++----------- .../definitions/container_collections.rb | 8 +- 3 files changed, 48 insertions(+), 60 deletions(-) diff --git a/app/models/manageiq/providers/kubernetes/inventory/collector/container_manager.rb b/app/models/manageiq/providers/kubernetes/inventory/collector/container_manager.rb index 5860305913..9cc5d5fac4 100644 --- a/app/models/manageiq/providers/kubernetes/inventory/collector/container_manager.rb +++ b/app/models/manageiq/providers/kubernetes/inventory/collector/container_manager.rb @@ -7,8 +7,8 @@ def pods @pods ||= entities_iterator(connection, :pods) end - def cluster_service_classes - @cluster_service_classes ||= entities_iterator(service_catalog_connection, :cluster_service_classes) + def cluster_service_offerings + @cluster_service_offerings ||= entities_iterator(service_catalog_connection, :cluster_service_classes) end def cluster_service_plans diff --git a/app/models/manageiq/providers/kubernetes/inventory/parser/container_manager.rb b/app/models/manageiq/providers/kubernetes/inventory/parser/container_manager.rb index b6c801982b..a1ce12b2ed 100644 --- a/app/models/manageiq/providers/kubernetes/inventory/parser/container_manager.rb +++ b/app/models/manageiq/providers/kubernetes/inventory/parser/container_manager.rb @@ -4,9 +4,9 @@ def parse parse_pods(collector.pods) # Service catalog entities - parse_container_service_classes(collector.cluster_service_classes) - parse_container_service_instances(collector.service_instances) - parse_container_service_plans(collector.cluster_service_plans) + parse_service_offerings(collector.cluster_service_offerings) + parse_service_instances(collector.service_instances) + parse_service_plans(collector.cluster_service_plans) end private @@ -39,74 +39,62 @@ def parse_pod(pod) ) end - def parse_container_service_classes(service_classes) - service_classes.each do |service_class| - parse_container_service_class(service_class) + def parse_service_offerings(service_offeringes) + service_offeringes.each do |service_offering| + parse_service_offering(service_offering) end end - def parse_container_service_class(service_class) - persister.container_service_classes.build( - parse_base_item(service_class).except(:namespace, :ems_ref, :name).merge( - :name => service_class.spec.externalName, - :ems_ref => service_class.spec.externalID, - :description => service_class.spec.description, - :container_project => lazy_find_project(service_class), - :extra => { - :spec => service_class.spec, - :status => service_class.status - } - ) + def parse_service_offering(service_offering) + persister.service_offerings.build( + :name => service_offering.spec.externalName, + :ems_ref => service_offering.spec.externalID, + :description => service_offering.spec.description, + :extra => { + :metadata => service_offering.metadata, + :spec => service_offering.spec, + :status => service_offering.status + } ) end - def parse_container_service_instances(service_instances) + def parse_service_instances(service_instances) service_instances.each do |service_instance| - parse_container_service_instance(service_instance) + parse_service_instance(service_instance) end end - def parse_container_service_instance(service_instance) - persister.container_service_instances.build( - parse_base_item(service_instance).except(:namespace, :ems_ref).merge( - :ems_ref => service_instance.spec.externalID, - :generate_name => service_instance.metadata.generate_name, - :container_project => lazy_find_project(service_instance), - :container_service_class => persister.container_service_classes.lazy_find( - service_instance.spec.clusterServiceClassRef.name - ), - :container_service_plan => persister.container_service_plans.lazy_find( - service_instance.spec.clusterServicePlanRef.name - ), - :extra => { - :spec => service_instance.spec, - :status => service_instance.status - } - ) + def parse_service_instance(service_instance) + persister.service_instances.build( + :name => service_instance.metadata.name, + :ems_ref => service_instance.spec.externalID, + :service_offering => persister.service_offerings.lazy_find(service_instance.spec.clusterServiceClassRef.name), + :service_plan => persister.service_plans.lazy_find(service_instance.spec.clusterServicePlanRef.name), + :extra => { + :metadata => service_instance.metadata, + :spec => service_instance.spec, + :status => service_instance.status + } ) end - def parse_container_service_plans(service_plans) + def parse_service_plans(service_plans) service_plans.each do |service_plan| - parse_container_service_plan(service_plan) + parse_service_plan(service_plan) end end - def parse_container_service_plan(service_plan) - persister.container_service_plans.build( - parse_base_item(service_plan).except(:namespace, :ems_ref, :name).merge( - :name => service_plan.spec.externalName, - :ems_ref => service_plan.spec.externalID, - :description => service_plan.spec.description, - :container_project => lazy_find_project(service_plan), - :container_service_class => persister.container_service_classes.lazy_find( - service_plan.spec.clusterServiceClassRef.name - ), - :extra => { - :spec => service_plan.spec, - :status => service_plan.status - } - ) + def parse_service_plan(service_plan) + persister.service_plans.build( + :name => service_plan.spec.externalName, + :ems_ref => service_plan.spec.externalID, + :description => service_plan.spec.description, + :service_offering => persister.service_offerings.lazy_find(service_plan.spec.clusterServiceClassRef.name), + :extra => { + :metadata => service_plan.metadata, + :spec => service_plan.spec, + :status => service_plan.status + } ) end diff --git a/app/models/manageiq/providers/kubernetes/inventory/persister/definitions/container_collections.rb b/app/models/manageiq/providers/kubernetes/inventory/persister/definitions/container_collections.rb index dcfc8b7ddc..a144129b84 100644 --- a/app/models/manageiq/providers/kubernetes/inventory/persister/definitions/container_collections.rb +++ b/app/models/manageiq/providers/kubernetes/inventory/persister/definitions/container_collections.rb @@ -20,9 +20,6 @@ def initialize_container_inventory_collections container_volumes container_replicators container_routes - container_service_classes - container_service_instances - container_service_plans container_services container_service_port_configs container_templates @@ -32,7 +29,10 @@ def initialize_container_inventory_collections computer_system_operating_systems persistent_volumes persistent_volume_claims - security_contexts).each do |name| + security_contexts + service_offerings + service_instances + service_plans).each do |name| add_collection(container, name) end From e532bb9941756558b74932cb98d252249f4854f0 Mon Sep 17 00:00:00 2001 From: Ladislav Smola Date: Fri, 24 Aug 2018 16:29:27 +0200 Subject: [PATCH 6/9] Add service catalog STI subclasses --- .../providers/kubernetes/container_manager/service_instance.rb | 2 ++ .../providers/kubernetes/container_manager/service_offering.rb | 2 ++ .../providers/kubernetes/container_manager/service_plan.rb | 2 ++ 3 files changed, 6 insertions(+) create mode 100644 app/models/manageiq/providers/kubernetes/container_manager/service_instance.rb create mode 100644 app/models/manageiq/providers/kubernetes/container_manager/service_offering.rb create mode 100644 app/models/manageiq/providers/kubernetes/container_manager/service_plan.rb diff --git a/app/models/manageiq/providers/kubernetes/container_manager/service_instance.rb b/app/models/manageiq/providers/kubernetes/container_manager/service_instance.rb new file mode 100644 index 0000000000..07b3082046 --- /dev/null +++ b/app/models/manageiq/providers/kubernetes/container_manager/service_instance.rb @@ -0,0 +1,2 @@ +class ManageIQ::Providers::Kubernetes::ContainerManager::ServiceInstance < ::ServiceInstance +end diff --git a/app/models/manageiq/providers/kubernetes/container_manager/service_offering.rb b/app/models/manageiq/providers/kubernetes/container_manager/service_offering.rb new file mode 100644 index 0000000000..ef8e50f896 --- /dev/null +++ b/app/models/manageiq/providers/kubernetes/container_manager/service_offering.rb @@ -0,0 +1,2 @@ +class ManageIQ::Providers::Kubernetes::ContainerManager::ServiceOffering < ::ServiceOffering +end diff --git a/app/models/manageiq/providers/kubernetes/container_manager/service_plan.rb b/app/models/manageiq/providers/kubernetes/container_manager/service_plan.rb new file mode 100644 index 0000000000..57dc122398 --- /dev/null +++ b/app/models/manageiq/providers/kubernetes/container_manager/service_plan.rb @@ -0,0 +1,2 @@ +class ManageIQ::Providers::Kubernetes::ContainerManager::ServicePlan < ::ServicePlan +end From 5a8f5768786f6c075bc982828c259a11a614a327 Mon Sep 17 00:00:00 2001 From: Ladislav Smola Date: Mon, 27 Aug 2018 11:36:25 +0200 Subject: [PATCH 7/9] Rename ServicePlan to ServiceParamatersSet --- ...vice_plan.rb => service_parameters_set.rb} | 2 +- .../inventory/collector/container_manager.rb | 4 +-- .../inventory/parser/container_manager.rb | 28 +++++++++---------- .../definitions/container_collections.rb | 2 +- 4 files changed, 18 insertions(+), 18 deletions(-) rename app/models/manageiq/providers/kubernetes/container_manager/{service_plan.rb => service_parameters_set.rb} (63%) diff --git a/app/models/manageiq/providers/kubernetes/container_manager/service_plan.rb b/app/models/manageiq/providers/kubernetes/container_manager/service_parameters_set.rb similarity index 63% rename from app/models/manageiq/providers/kubernetes/container_manager/service_plan.rb rename to app/models/manageiq/providers/kubernetes/container_manager/service_parameters_set.rb index 57dc122398..63efb7d67a 100644 --- a/app/models/manageiq/providers/kubernetes/container_manager/service_plan.rb +++ b/app/models/manageiq/providers/kubernetes/container_manager/service_parameters_set.rb @@ -1,2 +1,2 @@ -class ManageIQ::Providers::Kubernetes::ContainerManager::ServicePlan < ::ServicePlan +class ManageIQ::Providers::Kubernetes::ContainerManager::ServiceParametersSet < ::ServiceParametersSet end diff --git a/app/models/manageiq/providers/kubernetes/inventory/collector/container_manager.rb b/app/models/manageiq/providers/kubernetes/inventory/collector/container_manager.rb index 9cc5d5fac4..0428611ac2 100644 --- a/app/models/manageiq/providers/kubernetes/inventory/collector/container_manager.rb +++ b/app/models/manageiq/providers/kubernetes/inventory/collector/container_manager.rb @@ -11,8 +11,8 @@ def cluster_service_offerings @cluster_service_offerings ||= entities_iterator(service_catalog_connection, :cluster_service_classes) end - def cluster_service_plans - @cluster_service_plans ||= entities_iterator(service_catalog_connection, :cluster_service_plans) + def cluster_service_parameters_sets + @cluster_service_parameters_sets ||= entities_iterator(service_catalog_connection, :cluster_service_plans) end def service_instances diff --git a/app/models/manageiq/providers/kubernetes/inventory/parser/container_manager.rb b/app/models/manageiq/providers/kubernetes/inventory/parser/container_manager.rb index a1ce12b2ed..3f96d199e9 100644 --- a/app/models/manageiq/providers/kubernetes/inventory/parser/container_manager.rb +++ b/app/models/manageiq/providers/kubernetes/inventory/parser/container_manager.rb @@ -6,7 +6,7 @@ def parse # Service catalog entities parse_service_offerings(collector.cluster_service_offerings) parse_service_instances(collector.service_instances) - parse_service_plans(collector.cluster_service_plans) + parse_service_parameters_sets(collector.cluster_service_parameters_sets) end private @@ -69,7 +69,7 @@ def parse_service_instance(service_instance) :name => service_instance.metadata.name, :ems_ref => service_instance.spec.externalID, :service_offering => persister.service_offerings.lazy_find(service_instance.spec.clusterServiceClassRef.name), - :service_plan => persister.service_plans.lazy_find(service_instance.spec.clusterServicePlanRef.name), + :service_parameters_set => persister.service_parameters_sets.lazy_find(service_instance.spec.clusterServicePlanRef.name), :extra => { :metadata => service_instance.metadata, :spec => service_instance.spec, @@ -78,22 +78,22 @@ def parse_service_instance(service_instance) ) end - def parse_service_plans(service_plans) - service_plans.each do |service_plan| - parse_service_plan(service_plan) + def parse_service_parameters_sets(service_parameters_sets) + service_parameters_sets.each do |service_parameters_set| + parse_service_parameters_set(service_parameters_set) end end - def parse_service_plan(service_plan) - persister.service_plans.build( - :name => service_plan.spec.externalName, - :ems_ref => service_plan.spec.externalID, - :description => service_plan.spec.description, - :service_offering => persister.service_offerings.lazy_find(service_plan.spec.clusterServiceClassRef.name), + def parse_service_parameters_set(service_parameters_set) + persister.service_parameters_sets.build( + :name => service_parameters_set.spec.externalName, + :ems_ref => service_parameters_set.spec.externalID, + :description => service_parameters_set.spec.description, + :service_offering => persister.service_offerings.lazy_find(service_parameters_set.spec.clusterServiceClassRef.name), :extra => { - :metadata => service_plan.metadata, - :spec => service_plan.spec, - :status => service_plan.status + :metadata => service_parameters_set.metadata, + :spec => service_parameters_set.spec, + :status => service_parameters_set.status } ) end diff --git a/app/models/manageiq/providers/kubernetes/inventory/persister/definitions/container_collections.rb b/app/models/manageiq/providers/kubernetes/inventory/persister/definitions/container_collections.rb index a144129b84..e522a8c672 100644 --- a/app/models/manageiq/providers/kubernetes/inventory/persister/definitions/container_collections.rb +++ b/app/models/manageiq/providers/kubernetes/inventory/persister/definitions/container_collections.rb @@ -32,7 +32,7 @@ def initialize_container_inventory_collections security_contexts service_offerings service_instances - service_plans).each do |name| + service_parameters_sets).each do |name| add_collection(container, name) end From adebdbb6c1e3351a0ca967e6e9b1d94386690500 Mon Sep 17 00:00:00 2001 From: Ladislav Smola Date: Mon, 27 Aug 2018 11:46:11 +0200 Subject: [PATCH 8/9] Add TODO for iterator, so we can properly add it later Add TODO for iterator, so we can properly add it later --- .../inventory/collector/container_manager.rb | 31 +++++++++---------- 1 file changed, 15 insertions(+), 16 deletions(-) diff --git a/app/models/manageiq/providers/kubernetes/inventory/collector/container_manager.rb b/app/models/manageiq/providers/kubernetes/inventory/collector/container_manager.rb index 0428611ac2..54392eb7fb 100644 --- a/app/models/manageiq/providers/kubernetes/inventory/collector/container_manager.rb +++ b/app/models/manageiq/providers/kubernetes/inventory/collector/container_manager.rb @@ -1,22 +1,32 @@ class ManageIQ::Providers::Kubernetes::Inventory::Collector::ContainerManager < ManageIQ::Providers::Kubernetes::Inventory::Collector + # TODO(lsmola) we need to return iterator for each collection, so we avoid fetching too many items to memory at + # once. + # Hints from cben: + # A slightly open question in kubeclient iterator interface was how it should expose whole-collection resource_version + # (and other metadata). Should iterator yield it with every item, or return it at the end? + # With streaming parse (abonas/kubeclient#254), this depends on order of json. k8s puts items last, so version is + # known by time we start yielding; this is unlikely to change but I feel weird hardcoding this assumption... + # For chunking (abonas/kubeclient#283), we'll have metadata in each chunk, so either API works. Chunking also brings + # risk of getting 410 Gone if we wait too long, not sure how to handle that. + def namespaces - @namespaces ||= entities_iterator(connection, :namespaces) + @namespaces ||= connection.get_namespaces end def pods - @pods ||= entities_iterator(connection, :pods) + @pods ||= connection.get_pods end def cluster_service_offerings - @cluster_service_offerings ||= entities_iterator(service_catalog_connection, :cluster_service_classes) + @cluster_service_offerings ||= service_catalog_connection.get_cluster_service_classes end def cluster_service_parameters_sets - @cluster_service_parameters_sets ||= entities_iterator(service_catalog_connection, :cluster_service_plans) + @cluster_service_parameters_sets ||= service_catalog_connection.get_cluster_service_plans end def service_instances - @service_instances ||= entities_iterator(service_catalog_connection, :service_instances) + @service_instances ||= service_catalog_connection.get_service_instances end private @@ -28,15 +38,4 @@ def connection def service_catalog_connection @service_catalog_connection ||= manager.connect(:service => "kubernetes_service_catalog") end - - def entities_iterator(client, entity) - # TODO(lsmola) change to iterator, that will fetch paginated response from the server, never fetching everything - # at once - client.send("get_#{entity}") - rescue KubeException => e - # TODO(lsmola) the old refresh has entities that can throws error and return some :default set - _log.error("Unexpected Exception during fetching of entity #{entity}: #{e}") - _log.error(e.backtrace.join("\n")) - [] - end end From a9937e24b4ce7d3df469d09fee8a4c81310c4a00 Mon Sep 17 00:00:00 2001 From: Ladislav Smola Date: Mon, 27 Aug 2018 11:49:24 +0200 Subject: [PATCH 9/9] Remove service instances for now --- .../container_manager/service_instance.rb | 2 -- .../inventory/collector/container_manager.rb | 4 ---- .../inventory/parser/container_manager.rb | 21 ------------------- .../definitions/container_collections.rb | 1 - 4 files changed, 28 deletions(-) delete mode 100644 app/models/manageiq/providers/kubernetes/container_manager/service_instance.rb diff --git a/app/models/manageiq/providers/kubernetes/container_manager/service_instance.rb b/app/models/manageiq/providers/kubernetes/container_manager/service_instance.rb deleted file mode 100644 index 07b3082046..0000000000 --- a/app/models/manageiq/providers/kubernetes/container_manager/service_instance.rb +++ /dev/null @@ -1,2 +0,0 @@ -class ManageIQ::Providers::Kubernetes::ContainerManager::ServiceInstance < ::ServiceInstance -end diff --git a/app/models/manageiq/providers/kubernetes/inventory/collector/container_manager.rb b/app/models/manageiq/providers/kubernetes/inventory/collector/container_manager.rb index 54392eb7fb..0d52b057fd 100644 --- a/app/models/manageiq/providers/kubernetes/inventory/collector/container_manager.rb +++ b/app/models/manageiq/providers/kubernetes/inventory/collector/container_manager.rb @@ -25,10 +25,6 @@ def cluster_service_parameters_sets @cluster_service_parameters_sets ||= service_catalog_connection.get_cluster_service_plans end - def service_instances - @service_instances ||= service_catalog_connection.get_service_instances - end - private def connection diff --git a/app/models/manageiq/providers/kubernetes/inventory/parser/container_manager.rb b/app/models/manageiq/providers/kubernetes/inventory/parser/container_manager.rb index 3f96d199e9..3d0e64b36c 100644 --- a/app/models/manageiq/providers/kubernetes/inventory/parser/container_manager.rb +++ b/app/models/manageiq/providers/kubernetes/inventory/parser/container_manager.rb @@ -5,7 +5,6 @@ def parse # Service catalog entities parse_service_offerings(collector.cluster_service_offerings) - parse_service_instances(collector.service_instances) parse_service_parameters_sets(collector.cluster_service_parameters_sets) end @@ -58,26 +57,6 @@ def parse_service_offering(service_offering) ) end - def parse_service_instances(service_instances) - service_instances.each do |service_instance| - parse_service_instance(service_instance) - end - end - - def parse_service_instance(service_instance) - persister.service_instances.build( - :name => service_instance.metadata.name, - :ems_ref => service_instance.spec.externalID, - :service_offering => persister.service_offerings.lazy_find(service_instance.spec.clusterServiceClassRef.name), - :service_parameters_set => persister.service_parameters_sets.lazy_find(service_instance.spec.clusterServicePlanRef.name), - :extra => { - :metadata => service_instance.metadata, - :spec => service_instance.spec, - :status => service_instance.status - } - ) - end - def parse_service_parameters_sets(service_parameters_sets) service_parameters_sets.each do |service_parameters_set| parse_service_parameters_set(service_parameters_set) diff --git a/app/models/manageiq/providers/kubernetes/inventory/persister/definitions/container_collections.rb b/app/models/manageiq/providers/kubernetes/inventory/persister/definitions/container_collections.rb index e522a8c672..8d9f7f7c8a 100644 --- a/app/models/manageiq/providers/kubernetes/inventory/persister/definitions/container_collections.rb +++ b/app/models/manageiq/providers/kubernetes/inventory/persister/definitions/container_collections.rb @@ -31,7 +31,6 @@ def initialize_container_inventory_collections persistent_volume_claims security_contexts service_offerings - service_instances service_parameters_sets).each do |name| add_collection(container, name)