From 98ed28c33271077530c5c99c2f28273e41ec44ab Mon Sep 17 00:00:00 2001 From: Khushi Chaudhari Date: Tue, 13 Jul 2021 16:36:26 -0700 Subject: [PATCH 01/76] added initial watch impl --- source/plugins/ruby/in_kube_podinventory.rb | 27 +++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/source/plugins/ruby/in_kube_podinventory.rb b/source/plugins/ruby/in_kube_podinventory.rb index 5598602cd..88b40c283 100644 --- a/source/plugins/ruby/in_kube_podinventory.rb +++ b/source/plugins/ruby/in_kube_podinventory.rb @@ -40,6 +40,8 @@ def initialize @controllerData = {} @podInventoryE2EProcessingLatencyMs = 0 @podsAPIE2ELatencyMs = 0 + + @noticeHash = {} @kubeperfTag = "oneagent.containerInsights.LINUX_PERF_BLOB" @kubeservicesTag = "oneagent.containerInsights.KUBE_SERVICES_BLOB" @@ -79,6 +81,7 @@ def start @condition = ConditionVariable.new @mutex = Mutex.new @thread = Thread.new(&method(:run_periodic)) + @watchthread = Thread.new(&method(:watch)) @@podTelemetryTimeTracker = DateTime.now.to_time.to_i end end @@ -94,6 +97,29 @@ def shutdown end end + def watch + $log.info("in_kube_podinventory::watch: entered watch function") + begin + KubernetesApiClient.watch_pods(resource_version: @collection_version, as: :parsed) do |notice| + if !notice["object"].nil? && !notice["object"].empty? + $log.info("in_kube_podinventory::watch: received a notice of type #{notice["type"]}") + + item = notice["object"] + record = {"name" => item["metadata"]["name"], "uid" => item["metadata"]["uid"], "status" => item["status"]["phase"], "type" => notice["type"]} + + @mutex.synchronize { + $log.info("in_kube_podinventory::watch : about to add item to noticeHash. Time: #{Time.now.utc.iso8601}") + @noticeHash[item["metadata"]["uid"]] = record + $log.info("in_kube_podinventory::watch : successfully added item to noticeHash. Time: #{Time.now.utc.iso8601}") + } + end + end + rescue => exception + $log.warn("in_kube_podinventory::watch : watch events session got broken and re-establishing the session.") + $log.debug_backtrace(exception.backtrace) + end + end + def enumerate(podList = nil) begin podInventory = podList @@ -134,6 +160,7 @@ def enumerate(podList = nil) continuationToken = nil $log.info("in_kube_podinventory::enumerate : Getting pods from Kube API @ #{Time.now.utc.iso8601}") continuationToken, podInventory = KubernetesApiClient.getResourcesAndContinuationToken("pods?limit=#{@PODS_CHUNK_SIZE}") + @collection_version = podInventory["metadata"]["resourceVersion"] $log.info("in_kube_podinventory::enumerate : Done getting pods from Kube API @ #{Time.now.utc.iso8601}") podsAPIChunkEndTime = (Time.now.to_f * 1000).to_i @podsAPIE2ELatencyMs = (podsAPIChunkEndTime - podsAPIChunkStartTime) From 7958042353171289fe36ff2c22d31cd41543cd24 Mon Sep 17 00:00:00 2001 From: Khushi Chaudhari Date: Wed, 14 Jul 2021 10:10:06 -0700 Subject: [PATCH 02/76] debugging if we get to watch --- source/plugins/ruby/in_kube_podinventory.rb | 2 ++ 1 file changed, 2 insertions(+) diff --git a/source/plugins/ruby/in_kube_podinventory.rb b/source/plugins/ruby/in_kube_podinventory.rb index 88b40c283..daee674bf 100644 --- a/source/plugins/ruby/in_kube_podinventory.rb +++ b/source/plugins/ruby/in_kube_podinventory.rb @@ -80,7 +80,9 @@ def start @finished = false @condition = ConditionVariable.new @mutex = Mutex.new + $log.info("in_kube:podinventory::start: khushi debugging about to start thread for run_periodic") @thread = Thread.new(&method(:run_periodic)) + $log.info("in_kube:podinventory::start: khushi debugging about to start thread for watch") @watchthread = Thread.new(&method(:watch)) @@podTelemetryTimeTracker = DateTime.now.to_time.to_i end From ef316a43442bb154fb1fb3147e399d4737bd8561 Mon Sep 17 00:00:00 2001 From: Khushi Chaudhari Date: Wed, 14 Jul 2021 11:17:46 -0700 Subject: [PATCH 03/76] sanity check collection version is not reason for watch session ending --- source/plugins/ruby/in_kube_podinventory.rb | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/source/plugins/ruby/in_kube_podinventory.rb b/source/plugins/ruby/in_kube_podinventory.rb index daee674bf..3a8a005a8 100644 --- a/source/plugins/ruby/in_kube_podinventory.rb +++ b/source/plugins/ruby/in_kube_podinventory.rb @@ -101,6 +101,12 @@ def shutdown def watch $log.info("in_kube_podinventory::watch: entered watch function") + + # TODO: delete later, don't want to make another API call here + continuationToken, podInventory = KubernetesApiClient.getResourcesAndContinuationToken("pods?limit=#{@PODS_CHUNK_SIZE}") + @collection_version = podInventory["metadata"]["resourceVersion"] + @log.info("in_kube_podinventory::watch : received collection version: #{@collection_version}") + begin KubernetesApiClient.watch_pods(resource_version: @collection_version, as: :parsed) do |notice| if !notice["object"].nil? && !notice["object"].empty? @@ -108,6 +114,7 @@ def watch item = notice["object"] record = {"name" => item["metadata"]["name"], "uid" => item["metadata"]["uid"], "status" => item["status"]["phase"], "type" => notice["type"]} + $log.info("in_kube_podinventory::watch: successfully created a record with notice") @mutex.synchronize { $log.info("in_kube_podinventory::watch : about to add item to noticeHash. Time: #{Time.now.utc.iso8601}") @@ -119,6 +126,7 @@ def watch rescue => exception $log.warn("in_kube_podinventory::watch : watch events session got broken and re-establishing the session.") $log.debug_backtrace(exception.backtrace) + $log.info("watch events session broken backtrace: #{exception.backtrace}") end end @@ -163,6 +171,7 @@ def enumerate(podList = nil) $log.info("in_kube_podinventory::enumerate : Getting pods from Kube API @ #{Time.now.utc.iso8601}") continuationToken, podInventory = KubernetesApiClient.getResourcesAndContinuationToken("pods?limit=#{@PODS_CHUNK_SIZE}") @collection_version = podInventory["metadata"]["resourceVersion"] + @log.info("in_kube_podinventory::enumerate : received collection version: #{@collection_version}") $log.info("in_kube_podinventory::enumerate : Done getting pods from Kube API @ #{Time.now.utc.iso8601}") podsAPIChunkEndTime = (Time.now.to_f * 1000).to_i @podsAPIE2ELatencyMs = (podsAPIChunkEndTime - podsAPIChunkStartTime) From 7e91ef798d5f6b8b09e1a5f6ac4b19c7d8b907ee Mon Sep 17 00:00:00 2001 From: Khushi Chaudhari Date: Wed, 14 Jul 2021 11:49:23 -0700 Subject: [PATCH 04/76] some logs won't be printed, why --- source/plugins/ruby/in_kube_podinventory.rb | 48 ++++++++++++--------- 1 file changed, 28 insertions(+), 20 deletions(-) diff --git a/source/plugins/ruby/in_kube_podinventory.rb b/source/plugins/ruby/in_kube_podinventory.rb index 3a8a005a8..d4f155409 100644 --- a/source/plugins/ruby/in_kube_podinventory.rb +++ b/source/plugins/ruby/in_kube_podinventory.rb @@ -103,30 +103,38 @@ def watch $log.info("in_kube_podinventory::watch: entered watch function") # TODO: delete later, don't want to make another API call here + $log.info("in_kube_pod_inventory::watch: about to make API call to get podInventory") continuationToken, podInventory = KubernetesApiClient.getResourcesAndContinuationToken("pods?limit=#{@PODS_CHUNK_SIZE}") + $log.info("in_kube_pod_inventory::watch: finished API call to get podInventory") @collection_version = podInventory["metadata"]["resourceVersion"] @log.info("in_kube_podinventory::watch : received collection version: #{@collection_version}") - begin - KubernetesApiClient.watch_pods(resource_version: @collection_version, as: :parsed) do |notice| - if !notice["object"].nil? && !notice["object"].empty? - $log.info("in_kube_podinventory::watch: received a notice of type #{notice["type"]}") - - item = notice["object"] - record = {"name" => item["metadata"]["name"], "uid" => item["metadata"]["uid"], "status" => item["status"]["phase"], "type" => notice["type"]} - $log.info("in_kube_podinventory::watch: successfully created a record with notice") - - @mutex.synchronize { - $log.info("in_kube_podinventory::watch : about to add item to noticeHash. Time: #{Time.now.utc.iso8601}") - @noticeHash[item["metadata"]["uid"]] = record - $log.info("in_kube_podinventory::watch : successfully added item to noticeHash. Time: #{Time.now.utc.iso8601}") - } - end - end - rescue => exception - $log.warn("in_kube_podinventory::watch : watch events session got broken and re-establishing the session.") - $log.debug_backtrace(exception.backtrace) - $log.info("watch events session broken backtrace: #{exception.backtrace}") + loop do + $log.info("in_kube_pod_inventory::watch: inside infinite loop for watch pods") + begin + KubernetesApiClient.watch_pods(resource_version: @collection_version, as: :parsed) do |notice| + if !notice["object"].nil? && !notice["object"].empty? + $log.info("in_kube_podinventory::watch: received a notice of type #{notice["type"]}") + + item = notice["object"] + record = {"name" => item["metadata"]["name"], "uid" => item["metadata"]["uid"], "status" => item["status"]["phase"], "type" => notice["type"]} + $log.info("in_kube_podinventory::watch: successfully created a record with notice") + + @mutex.synchronize { + $log.info("in_kube_podinventory::watch : about to add item to noticeHash. Time: #{Time.now.utc.iso8601}") + @noticeHash[item["metadata"]["uid"]] = record + $log.info("in_kube_podinventory::watch : successfully added item to noticeHash. Time: #{Time.now.utc.iso8601}") + } + else + $log.info("in_kube_pod_inventory::watch: notice object was either null or empty") + end + end + rescue => exception + $log.warn("in_kube_podinventory::watch : watch events session got broken and re-establishing the session.") + # $log.debug_backtrace(exception.backtrace) + $log.info("watch events session broken backtrace: #{exception.backtrace}") + end + sleep 300 end end From 966f3628422893d34d3562b4eb8ec0f069b0eb37 Mon Sep 17 00:00:00 2001 From: Khushi Chaudhari Date: Wed, 14 Jul 2021 15:11:54 -0700 Subject: [PATCH 05/76] adding writing to file functionality from test app --- source/plugins/ruby/in_kube_podinventory.rb | 54 ++++++++++++++++++--- 1 file changed, 48 insertions(+), 6 deletions(-) diff --git a/source/plugins/ruby/in_kube_podinventory.rb b/source/plugins/ruby/in_kube_podinventory.rb index d4f155409..3fd7c8caa 100644 --- a/source/plugins/ruby/in_kube_podinventory.rb +++ b/source/plugins/ruby/in_kube_podinventory.rb @@ -80,10 +80,10 @@ def start @finished = false @condition = ConditionVariable.new @mutex = Mutex.new - $log.info("in_kube:podinventory::start: khushi debugging about to start thread for run_periodic") - @thread = Thread.new(&method(:run_periodic)) - $log.info("in_kube:podinventory::start: khushi debugging about to start thread for watch") + $log.info("in_kube:podinventory::start: about to start thread for watch") @watchthread = Thread.new(&method(:watch)) + $log.info("in_kube:podinventory::start: about to start thread for run_periodic") + @thread = Thread.new(&method(:run_periodic)) @@podTelemetryTimeTracker = DateTime.now.to_time.to_i end end @@ -99,15 +99,57 @@ def shutdown end end + def write_to_file(podInventory) + $log.info("in_kube_podinventory::write_to_file: inside write to file function") + + File.write("testing-podinventory.json", JSON.pretty_generate(podInventory)) + + $log.info("in_kube_podinventory::write_to_file: successfully done writing to file") + + end + def watch $log.info("in_kube_podinventory::watch: entered watch function") # TODO: delete later, don't want to make another API call here - $log.info("in_kube_pod_inventory::watch: about to make API call to get podInventory") + File.open("testing-podinventory.json", "w") + $log.info("in_kube_podinventory::watch : successfully opened json file for writing") + + @podsAPIE2ELatencyMs = 0 + podsAPIChunkStartTime = (Time.now.to_f * 1000).to_i + # Initializing continuation token to nil + continuationToken = nil + $log.info("in_kube_podinventory::watch : Getting pods from Kube API @ #{Time.now.utc.iso8601}") continuationToken, podInventory = KubernetesApiClient.getResourcesAndContinuationToken("pods?limit=#{@PODS_CHUNK_SIZE}") - $log.info("in_kube_pod_inventory::watch: finished API call to get podInventory") @collection_version = podInventory["metadata"]["resourceVersion"] - @log.info("in_kube_podinventory::watch : received collection version: #{@collection_version}") + $log.info("in_kube_podinventory::watch : received collection version: #{@collection_version}") + $log.info("in_kube_podinventory::watch : Done getting pods from Kube API @ #{Time.now.utc.iso8601}") + podsAPIChunkEndTime = (Time.now.to_f * 1000).to_i + @podsAPIE2ELatencyMs = (podsAPIChunkEndTime - podsAPIChunkStartTime) + if (!podInventory.nil? && !podInventory.empty? && podInventory.key?("items") && !podInventory["items"].nil? && !podInventory["items"].empty?) + $log.info("in_kube_podinventory::watch : number of pod items :#{podInventory["items"].length} from Kube API @ #{Time.now.utc.iso8601}") + write_to_file(podInventory) + # parse_and_emit_records(podInventory, serviceRecords, continuationToken, batchTime) + else + $log.warn "in_kube_podinventory::watch:Received empty podInventory" + end + + #If we receive a continuation token, make calls, process and flush data until we have processed all data + while (!continuationToken.nil? && !continuationToken.empty?) + podsAPIChunkStartTime = (Time.now.to_f * 1000).to_i + continuationToken, podInventory = KubernetesApiClient.getResourcesAndContinuationToken("pods?limit=#{@PODS_CHUNK_SIZE}&continue=#{continuationToken}") + podsAPIChunkEndTime = (Time.now.to_f * 1000).to_i + @podsAPIE2ELatencyMs = @podsAPIE2ELatencyMs + (podsAPIChunkEndTime - podsAPIChunkStartTime) + if (!podInventory.nil? && !podInventory.empty? && podInventory.key?("items") && !podInventory["items"].nil? && !podInventory["items"].empty?) + $log.info("in_kube_podinventory::watch : number of pod items :#{podInventory["items"].length} from Kube API @ #{Time.now.utc.iso8601}") + write_to_file(podInventory) + # parse_and_emit_records(podInventory, serviceRecords, continuationToken, batchTime) + else + $log.warn "in_kube_podinventory::watch:Received empty podInventory" + end + end + + $log.info("finished initial write to pod inventory file") loop do $log.info("in_kube_pod_inventory::watch: inside infinite loop for watch pods") From 32598a03bb4fd8a30041760f4609a7745cfc1434 Mon Sep 17 00:00:00 2001 From: Khushi Chaudhari Date: Wed, 14 Jul 2021 16:47:36 -0700 Subject: [PATCH 06/76] added setup for kubeclient and log testing for watch --- kubernetes/linux/setup.sh | 3 ++ source/plugins/ruby/in_kube_podinventory.rb | 35 +++++++++++---------- 2 files changed, 22 insertions(+), 16 deletions(-) diff --git a/kubernetes/linux/setup.sh b/kubernetes/linux/setup.sh index ad7cc2232..5cb306cd6 100644 --- a/kubernetes/linux/setup.sh +++ b/kubernetes/linux/setup.sh @@ -52,6 +52,9 @@ sudo apt-get install ruby2.6 ruby2.6-dev gcc make -y gem install fluentd -v "1.12.2" --no-document fluentd --setup ./fluent gem install gyoku iso8601 --no-doc +# kubeclient gem +apt-get install libmagickwand-dev +gem install kubeclient --no-document rm -f $TMPDIR/docker-cimprov*.sh diff --git a/source/plugins/ruby/in_kube_podinventory.rb b/source/plugins/ruby/in_kube_podinventory.rb index 3fd7c8caa..457198922 100644 --- a/source/plugins/ruby/in_kube_podinventory.rb +++ b/source/plugins/ruby/in_kube_podinventory.rb @@ -3,6 +3,7 @@ require 'fluent/plugin/input' + module Fluent::Plugin require_relative "podinventory_to_mdm" @@ -20,6 +21,7 @@ def initialize require "yajl" require "set" require "time" + require "kubeclient" require_relative "kubernetes_container_inventory" require_relative "KubernetesApiClient" @@ -114,7 +116,7 @@ def watch # TODO: delete later, don't want to make another API call here File.open("testing-podinventory.json", "w") $log.info("in_kube_podinventory::watch : successfully opened json file for writing") - + @podsAPIE2ELatencyMs = 0 podsAPIChunkStartTime = (Time.now.to_f * 1000).to_i # Initializing continuation token to nil @@ -155,21 +157,22 @@ def watch $log.info("in_kube_pod_inventory::watch: inside infinite loop for watch pods") begin KubernetesApiClient.watch_pods(resource_version: @collection_version, as: :parsed) do |notice| - if !notice["object"].nil? && !notice["object"].empty? - $log.info("in_kube_podinventory::watch: received a notice of type #{notice["type"]}") - - item = notice["object"] - record = {"name" => item["metadata"]["name"], "uid" => item["metadata"]["uid"], "status" => item["status"]["phase"], "type" => notice["type"]} - $log.info("in_kube_podinventory::watch: successfully created a record with notice") - - @mutex.synchronize { - $log.info("in_kube_podinventory::watch : about to add item to noticeHash. Time: #{Time.now.utc.iso8601}") - @noticeHash[item["metadata"]["uid"]] = record - $log.info("in_kube_podinventory::watch : successfully added item to noticeHash. Time: #{Time.now.utc.iso8601}") - } - else - $log.info("in_kube_pod_inventory::watch: notice object was either null or empty") - end + $log.info("in_kube_podinventory::watch: inside watch pods! Time: #{Time.now.utc.iso8601}") + # if !notice["object"].nil? && !notice["object"].empty? + # $log.info("in_kube_podinventory::watch: received a notice of type #{notice["type"]}") + + # item = notice["object"] + # record = {"name" => item["metadata"]["name"], "uid" => item["metadata"]["uid"], "status" => item["status"]["phase"], "type" => notice["type"]} + # $log.info("in_kube_podinventory::watch: successfully created a record with notice") + + # # @mutex.synchronize { + # $log.info("in_kube_podinventory::watch : about to add item to noticeHash. Time: #{Time.now.utc.iso8601}") + # @noticeHash[item["metadata"]["uid"]] = record + # $log.info("in_kube_podinventory::watch : successfully added item to noticeHash. Time: #{Time.now.utc.iso8601}") + # # } + # else + # $log.info("in_kube_pod_inventory::watch: notice object was either null or empty") + # end end rescue => exception $log.warn("in_kube_podinventory::watch : watch events session got broken and re-establishing the session.") From 08fdfa0e3d39b19f1a71b0587cbe65cc375e4afe Mon Sep 17 00:00:00 2001 From: Khushi Chaudhari Date: Thu, 15 Jul 2021 10:08:40 -0700 Subject: [PATCH 07/76] added kubclient support --- source/plugins/ruby/in_kube_podinventory.rb | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/source/plugins/ruby/in_kube_podinventory.rb b/source/plugins/ruby/in_kube_podinventory.rb index 457198922..0bbfde9d8 100644 --- a/source/plugins/ruby/in_kube_podinventory.rb +++ b/source/plugins/ruby/in_kube_podinventory.rb @@ -78,6 +78,14 @@ def start $log.warn("in_kube_podinventory::start: setting to default value since got PODS_EMIT_STREAM_BATCH_SIZE nil or empty") @PODS_EMIT_STREAM_BATCH_SIZE = 200 end + # create kubernetes watch client + ssl_options = { + ca_file: "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt", + verify_ssl: OpenSSL::SSL::VERIFY_PEER, + } + getTokenStr = "Bearer " + KubernetesApiClient.getTokenStr + auth_options = { bearer_token: KubernetesApiClient.getTokenStr } + @KubernetesWatchClient = Kubeclient::Client.new('https://#{ENV["KUBERNETES_SERVICE_HOST"]}:#{ENV["KUBERNETES_PORT_443_TCP_PORT"]}/api/', "v1", ssl_options: ssl_options, auth_options: auth_options) $log.info("in_kube_podinventory::start: PODS_EMIT_STREAM_BATCH_SIZE @ #{@PODS_EMIT_STREAM_BATCH_SIZE}") @finished = false @condition = ConditionVariable.new @@ -156,7 +164,7 @@ def watch loop do $log.info("in_kube_pod_inventory::watch: inside infinite loop for watch pods") begin - KubernetesApiClient.watch_pods(resource_version: @collection_version, as: :parsed) do |notice| + @KubernetesWatchClient.watch_pods(resource_version: @collection_version, as: :parsed) do |notice| $log.info("in_kube_podinventory::watch: inside watch pods! Time: #{Time.now.utc.iso8601}") # if !notice["object"].nil? && !notice["object"].empty? # $log.info("in_kube_podinventory::watch: received a notice of type #{notice["type"]}") From 62c7c6304972902e02a0ffe3481bb1afb2a9b337 Mon Sep 17 00:00:00 2001 From: Khushi Chaudhari Date: Thu, 15 Jul 2021 19:02:13 -0700 Subject: [PATCH 08/76] potentially fixed kubeclient issue --- kubernetes/linux/setup.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kubernetes/linux/setup.sh b/kubernetes/linux/setup.sh index 5cb306cd6..c91e0bd29 100644 --- a/kubernetes/linux/setup.sh +++ b/kubernetes/linux/setup.sh @@ -53,7 +53,7 @@ gem install fluentd -v "1.12.2" --no-document fluentd --setup ./fluent gem install gyoku iso8601 --no-doc # kubeclient gem -apt-get install libmagickwand-dev +apt-get install libmagickwand-dev -y gem install kubeclient --no-document From 1a0a7abd30328d2c49cfd33f43201558a254f8c1 Mon Sep 17 00:00:00 2001 From: Khushi Chaudhari Date: Fri, 16 Jul 2021 12:47:53 -0700 Subject: [PATCH 09/76] debugging watch not being triggered --- khushi-test | 20 ++++++++ .../existingClusterOnboarding.json | 44 ++++++++++++++++ .../dockerbuild/existingClusterParam.json | 15 ++++++ kubernetes/linux/dockerbuild/khushi-test | 20 ++++++++ kubernetes/omsagent.yaml | 50 +++++++++---------- source/plugins/ruby/in_kube_podinventory.rb | 31 ++++++------ 6 files changed, 140 insertions(+), 40 deletions(-) create mode 100644 khushi-test create mode 100644 kubernetes/linux/dockerbuild/existingClusterOnboarding.json create mode 100644 kubernetes/linux/dockerbuild/existingClusterParam.json create mode 100644 kubernetes/linux/dockerbuild/khushi-test diff --git a/khushi-test b/khushi-test new file mode 100644 index 000000000..4a07d0fb9 --- /dev/null +++ b/khushi-test @@ -0,0 +1,20 @@ +apiVersion: v1 +clusters: +- cluster: + certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUU2VENDQXRHZ0F3SUJBZ0lSQUozU3djVWpyaUg5blhaWnZHVHFmWWN3RFFZSktvWklodmNOQVFFTEJRQXcKRFRFTE1Ba0dBMVVFQXhNQ1kyRXdJQmNOTWpFd05qSTBNVFl5TVRBMVdoZ1BNakExTVRBMk1qUXhOak14TURWYQpNQTB4Q3pBSkJnTlZCQU1UQW1OaE1JSUNJakFOQmdrcWhraUc5dzBCQVFFRkFBT0NBZzhBTUlJQ0NnS0NBZ0VBCnUvZFhmRnZvYi9zREdVa0VoVWNYcDVGVndBR2VyR01hOXJocVhPM3JSbFJQYlltTW9oOVUrQWV6Q0U4ckJ5WGwKbCtib2JNKzQzU2x1Y3BickkrOVpZcGEvYnYrY2dYRXRGOENzdDMwbm9LYjlGeno2cVFicHNuQ09hNllhdVRhVQovWVU2aVhCQm4vRnhYZ1BNRGFucTRFcVVnNXNVZkhuVFJCNFdhZmhBaXI2T2g4RjBoZzhyRS83dXJ2cVFhNllPCklNN2kzUHY5dG1GN1ZjRThocUFYZTdNUE4xd0R2Q21SUGZoZlZTejAxdDFlQlFtcDJ4YitHQ2lBRStYUHVVZ3EKZ3RkU21OL2tnYVdRMFFIWm9sRWdEbE1Hb1BjSXNkeXZvd2x4WThEMmlkNlluMS9jeEtvNHBmUjkrUVdieUZrRQp5b0VDM0M0MGVBZVJTZUtLdjlxSnJyNlp2WGRxZGs1WHpRREU2d1hqakorUUhGZHJyeHFqdE01Z0dkbThiRDdiCmg0WjA5c201djBQd1BQZWE5SG5WbURmbjlsQmtjZGRSTDJuV1NGSVZsbWdDMVdrV3huZXBmSGV3YmN4MnpnK1UKazc2UjMxamMyWW9NRnZpNmtXaTFLejYwZmh5N0VIVWxsRTRlbm9VZm5TM2dDZ2pxeFVJcWlKZFpzbGluSTdNbgpFS0sxZ21DRk96bmtMU3MzQ0FXTGFKNEN6VFJsQkZKWDYzTE5NR2U0NXBTRUJIVEFlTUdtbjY5Tjc2Y2lrejNiCmthYTJXZ05sTEFZVDBEZUtQR0RJNmFsU2pOckFaS3lkNS9rTzBYTGo4MFVtT2JZY1NpeXlwYlhZZ1Q5bWxna3IKVEhPZ29HKzlEaDQ3bWdWczJlaURVYWJ6RVpTUlBUbEorTk9iSkJRczQ1RUNBd0VBQWFOQ01FQXdEZ1lEVlIwUApBUUgvQkFRREFnS2tNQThHQTFVZEV3RUIvd1FGTUFNQkFmOHdIUVlEVlIwT0JCWUVGRWEzTFFUSHJFeHBZQ0VJCkJZNEEzd3k4S2ZqTU1BMEdDU3FHU0liM0RRRUJDd1VBQTRJQ0FRQlp2b0IwdVpZa3dYclBvTnp3LytmMEc1bFUKMENWYU56STNkQlNyd29lT1VMTnBJdDFad0E3REhxNTZNa0pUTzNlNnNQbzZKczFIbHAzRTRJaFJxZTI0NnNyQgpaam51czdTNDY1aHBFcDBhd1lHMzdnY05NMlBxbFo3dDZNdDhSQjVneHJPb3JNS1BSSWZiYnd6NDNvWHB6VHVpCkZ0QUk5Z2xjR3V1VlZ5Yk42a2N2Y2MyNE9zb0VGQ3VHU0dBMzhaUTZqY1NIRDUvQW9rcGd0cTZ4M0trNHR1TjIKS3dWcjdtekdvZGZrOTlKSVhkTWREVy9qTGt5RlRRQWhoNlY2ZThmZHNJbmtEZFdKOUI2UlFqSzFiMjRiQmowbwpjaGZkald3L2pLRzQ4YTFIS2tlWHk2MG5oU1FKaGFaRjBvTEprNElpU3lqNUtQMlpHUGg1WTZLSUlRT0oreE5HCmJiN3BTQ2h6dFFVUjR4dEZkc2pxZk9BZzdCZitYU3ZTNXExUU5FS3JaOHE2M2VMQVQ5aFUxVDVDVE5QVW1Zd2YKbjNQVEJHdVUrNThpWFM3VGVxVGVwYlZxOFlvU2hCeVc1cXdWa1loU1VWNzk5VGNVb2oreTJ5SDAxd3F2a0JlUQovZExXZ0srVE4xZnpsMVBXTk5CWW03SFgwbWpuZVRrWFB2QWFKMzNsS0RzdWpsR0RwYmkza09tbGN1TVJqYStUCm5iSjNRRkd5aDR2UWt6N3hMWEtDdFFBOHVUeXVNVWdTVzJZZ2ZZYWlNWG1wMG95RHNQRnI3aFFKMXN2THd0dksKd2M3Z1ZMV3ZIWTlYOVhqN0dSbFFwdlBmV1B4MkxESThFeEU1eTVQSXFqYjhTMFZ1aVU4REpZTlVEaDlodGhoVgpqRmxuVTd6ekw5dEE4RDZ6RWc9PQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0tCg== + server: https://khushi-test-dns-92573251.hcp.westus2.azmk8s.io:443 + name: khushi-test +contexts: +- context: + cluster: khushi-test + user: clusterUser_khushi-test_khushi-test + name: khushi-test +current-context: khushi-test +kind: Config +preferences: {} +users: +- name: clusterUser_khushi-test_khushi-test + user: + client-certificate-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUZIakNDQXdhZ0F3SUJBZ0lSQVBHQzNPazBaT2pxcXppS01CT3M3bEF3RFFZSktvWklodmNOQVFFTEJRQXcKRFRFTE1Ba0dBMVVFQXhNQ1kyRXdIaGNOTWpFd05qSTBNVFl5TVRBMVdoY05Nak13TmpJME1UWXpNVEExV2pBdwpNUmN3RlFZRFZRUUtFdzV6ZVhOMFpXMDZiV0Z6ZEdWeWN6RVZNQk1HQTFVRUF4TU1iV0Z6ZEdWeVkyeHBaVzUwCk1JSUNJakFOQmdrcWhraUc5dzBCQVFFRkFBT0NBZzhBTUlJQ0NnS0NBZ0VBdE1GUVhjQkdJYmduK1dleWlMOWkKYkg1Z2JvQ2w4QkZyb3dRWnpZbjRsSnBxQVVuRzR4b0QwbUZSNklBbkdQd0szTVdqNXJmcFNQUEdLZ29kbHBMTQo4azZSeVBNN1BwYTlzck1NWGJlMHZMQmJxb1lNeFc4aHFyTldBcEtYWitJUWI0TnkrY0VFbklMNlY4aHN4ME9kCmVVYTZJK2xib3k1VXVjN0FsL1pFVSt6NHRGaEhtNjc1VHNhWU9kOXozSTJpZ0pmQS9EZ044Q3ZWVWFHeVNwVHAKOGczOWpsaTJTNllaT0xRSlRSVnBZUEFLck1DQnlQclV1L3FhbFVMSXVSdE5SLyszdkpNS2pyZWQzQTd6MXYwdwo4NGw4ZTlJQWI0TDNwRDFTL1FzcnNldi9aTnN4dXljZVczNjlGeXFrNEZwWnZaVEwvb3A4OUdZa2ZaZmpXNmpiCmp1MXdFaklSYTJ3Z3BJdDlNSitTRHc3YlFtdmxNQVNhZmlwVnEyRDBXYmMveWhySFk5MGRwNEVIY1dOcDNkUlIKSU5YZ0U3cmdEa1RzK3drdlNlVUN5ckZZUTBBck1nZUl0NDNJR0srdVU5UW5HaWhNOG1Wa2tGVjM1SitQWTZTawpQNW5JRk83OGJYNkk4UVRsOXhEaVJsM3hGTHJGcE53Sjk2bWFlVkE5ZDd6NjdoZnFvc3o2c25YZ0I4L085UkhjCmhrUWhWZWt6c2o4T250WU9vK3V6K2llandaRGlVREV6N0dQeGI3NUw0aks4RERSalFtYys0dUZTOFh3ZFRrUEoKS3lZc3FkTGVJb3REWjlkYm9sUTF6WjlZS3hsNTcrQ3l2aU50ZGFCL0FRVUZ0dnNnZXY2bzdmYkhzY1VoZTRvbgpZcDRFOHVWRHcyMlFuakdweUlNRzZJY0NBd0VBQWFOV01GUXdEZ1lEVlIwUEFRSC9CQVFEQWdXZ01CTUdBMVVkCkpRUU1NQW9HQ0NzR0FRVUZCd01DTUF3R0ExVWRFd0VCL3dRQ01BQXdId1lEVlIwakJCZ3dGb0FVUnJjdEJNZXMKVEdsZ0lRZ0ZqZ0RmREx3cCtNd3dEUVlKS29aSWh2Y05BUUVMQlFBRGdnSUJBQTRubkxXUERBaUlka0QxV3NiegoxTnVlSnhhVzRzd2VEdWtyNDdxVEFweHA1dFVyK1N1OGVCQkg5WVorTW5XUHBFUEtkclQ5Q0xRNElyTzJXbEJXCmNmV1lZTGxOU0hFSjVOL1pBY0k2Q2Y2MklnOUZlbHdzYk5SNjhOWGozL21OZEpqa3dzVitVL0FwVXM4RTVXU2sKMUUzaU4xOWxEUnh6cFNXZ0R5dUJmQzhvWWRSZzMzZ0wyd3FjcmRQc3MvdlZnc013S2tBd2NwVHJ1ei84ZjZTSgpPdTFtWmlDRGFVUzdQdm9IQ2YvMEU0eDJzZmdqTE9scjE1eTc4K0dKVVdOcXBpVXQ2aWxHNVVKWVZqd1JCS0U4CkxJTGw4dW0xeTUxU2pmUzNkSnQ4b0l4WENSWW9uTStNRDA0ZEYzVHduSFRhVWljS05Va2x5b1VMQWhVN1VmREYKWkNqU1FVckFhZm5JajB0cFdDRHVuK1hEQ1pHRVVzV3BDeTdINWhJOFhqYjBRbEw3Vm5KRGk5alpBeWwwalJhVAp5OExrSTA2SXNwbVhlM2xJRzI5S0FRVmdGOWNsNjFDVTVWR25CM2ZHbUNzOUdtMStmTTNkc2hya055eHkvN1JNClBBUUhyVzhiWVo1c0NiQ09SUHBqc3NOandSVk16VjNtU2dhcmI0b1h1T3V2NTZoSnYxSnAxc2N1bS84SVlrQm4KdkE5RS9qa0E5RTVqd05uV0FNam9wRGtTZlVNN0NiVDdJMjZ5R001THdLdDc2NXY4bUJTVVBGcWt0amM5SVpoOApXZ2NKcGxuRHExOEhyTGtTQ3gvamRkR2I0WWphRDN1ajhyb0JMQ3NPRFR2K3NzWUo0Z21heWpHbkZRbFRLZlhpCnNLV2hwL0RvUXR6cWZtbHZyWDdlRnUrUwotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0tCg== + client-key-data: LS0tLS1CRUdJTiBSU0EgUFJJVkFURSBLRVktLS0tLQpNSUlKS0FJQkFBS0NBZ0VBdE1GUVhjQkdJYmduK1dleWlMOWliSDVnYm9DbDhCRnJvd1FaelluNGxKcHFBVW5HCjR4b0QwbUZSNklBbkdQd0szTVdqNXJmcFNQUEdLZ29kbHBMTThrNlJ5UE03UHBhOXNyTU1YYmUwdkxCYnFvWU0KeFc4aHFyTldBcEtYWitJUWI0TnkrY0VFbklMNlY4aHN4ME9kZVVhNkkrbGJveTVVdWM3QWwvWkVVK3o0dEZoSAptNjc1VHNhWU9kOXozSTJpZ0pmQS9EZ044Q3ZWVWFHeVNwVHA4ZzM5amxpMlM2WVpPTFFKVFJWcFlQQUtyTUNCCnlQclV1L3FhbFVMSXVSdE5SLyszdkpNS2pyZWQzQTd6MXYwdzg0bDhlOUlBYjRMM3BEMVMvUXNyc2V2L1pOc3gKdXljZVczNjlGeXFrNEZwWnZaVEwvb3A4OUdZa2ZaZmpXNmpianUxd0VqSVJhMndncEl0OU1KK1NEdzdiUW12bApNQVNhZmlwVnEyRDBXYmMveWhySFk5MGRwNEVIY1dOcDNkUlJJTlhnRTdyZ0RrVHMrd2t2U2VVQ3lyRllRMEFyCk1nZUl0NDNJR0srdVU5UW5HaWhNOG1Wa2tGVjM1SitQWTZTa1A1bklGTzc4Ylg2SThRVGw5eERpUmwzeEZMckYKcE53Sjk2bWFlVkE5ZDd6NjdoZnFvc3o2c25YZ0I4L085UkhjaGtRaFZla3pzajhPbnRZT28rdXoraWVqd1pEaQpVREV6N0dQeGI3NUw0aks4RERSalFtYys0dUZTOFh3ZFRrUEpLeVlzcWRMZUlvdERaOWRib2xRMXpaOVlLeGw1CjcrQ3l2aU50ZGFCL0FRVUZ0dnNnZXY2bzdmYkhzY1VoZTRvbllwNEU4dVZEdzIyUW5qR3B5SU1HNkljQ0F3RUEKQVFLQ0FnRUFoeHFMRUZmNzA0NUxTcTdJa2svQ3FHZFplcDdyNk9HTWUzOUFMaEkzWEJPQ3NrM0x1OG1oRzZtawpTRTBwYkJKUHFDNk4wSkNjYk92UXJYRjhKaVM1elU4T3hvMlBOYndFTkVLQ05yWnVBQzh0aE9ySE52ZFd5YmxiCnkzWjRkcXhSTHBpbzRxYno1R3c5NUVrcGpWTXJyQTlDYUQ4dlFHd0w1Z24waUdFSk4xWERtMEQwM1JmT3ZxUFAKK2tuYjF2bUNnZTFyME5teENWZ3FET2I0VmFpeXFEcnV6MnZkYmZQTG5FZHRRRkdGdjNBTCtvMjFORnpQajJYRQpEMWFvb3pzVjYzL21mS3ZIVTE5SlBjZHJXTUZNeW10MFFUUjVJT0JhYTBkdTlUWlpSWmFwQVdDWTdsTlI3SDl4CnBKUTZRbnNsNG44S2JzcGlsV2FYVmN5bW9zYzJ1Mzlzb3EyVUd3cjJoOHJ3MEdFNGdEY21wZ0RwY3FuRXlEVloKRUxoNG9CSmEvc1F0K0tiVk1zRVZUbUZkOXJyaDNEbGRuaHNPTVNvUldxWTZYZ01pNzQ1OVJPWWxPSEZ3YTdFTgpHUzFqMWF5dndDZmVUSGcrNHhFVUYybXcyQmd6UDNXOTZ0VmpOOFgxTmdFbHZLT1lCTTFEc1crYndSR2RldjErCm41R2N4WEZja3A4ZDY5QWtkQThsT2hOM2dzTVZ2Vk5rVmFBQWwzdzMrbkNIVHlNcStjeXJ6V1orSnZNNjZZdnIKZlQvSkQyZ3g0RW05ZEdkTjlxT0ZxTWtkMXN1dk9HOWJHbVB2N2JEb2hYV2dwL2FUMkk3WW1CYW1yemwvNGpPYwo0VTI3Z2JwTysxS2NjeDB6Zk9TYkJGZU91N2thYkJpc0JsdkExeThoQlBxVjRyUy9XZmtDZ2dFQkFOemFSODlXCnlsRlFsUjNhZXF6c3o2QlRLRW9ISndPelZvYXBRdzZZYjNiaTROczY1eXFoUllGWjlBV1BJdC9wQ090TEsyWnQKRG1oRTVVcUJreklQZ3EwK292OUtIUUk1aEVCV241U0dUWnNWSDdpMDdBUGhVeDVVWDROTEpMNWM5SklINlRZZAoyVC9CWGV3c1JFSiswOU5sN3NMTU53eWZlMUhGQU5ldUFXZ2hhNW1YVTNWOFBLeEY1N3NzcGI5YzlrZzVhc1dhClByRHRMRFRmd0hCQ0tlc3ZZMkVnUU9Tc0VNb0l1SWN0WnQvWDd3UzFjMldSbStzRis5WWJYV1lBaHNTQ0tpSGcKYlpJdG53cUg0YVJzWmRmTjlWWnpMTG8zdWpiK3N0RXA5RWtyNWJkeDRFZEUwSkZrblVTZzRYTHN2Q1RyektEMQpaV2dmdGQ0VFVXYmJMdE1DZ2dFQkFOR0ZiVm1GazNiYld1bFVKOXhZclpWNDBmZXZhUVFnQjg1Nm01VGxVN2JKCktKSms1U2J2TThCcEMrSkFHRk1vWXdlZENYQlppV3YxWHExV20vTURFaWl3V1lBZW1HZ0tsM1Bzam9aNEE3WXcKNkRicmRTcDY1WXJ0aFh6aW01Z3RrbkhoVDhmVGhSUUZFZmZ1MnFpY1lOMTFSZ0liVW1NU0xyc0c3aEtJN3o4dApUaEVqbmZQSXRob1R2WkN3R0ZjMi92eEhYTXl0ancra0pNTnlXL0pCQUgyUU1rdmJHSHdDckE0Y1ovTWtSc1l0CldEemkrUTZTcStWeDVyUDVRVzNUUFhDeGhNMWgrZ1FGcmsyVTYrcG9waHQvQ0NCVzVRT3duRVdkamtvcXoyRm0KTW9ZblpDUmpjMXdJWmJKYjNjT2thNG0yR3I2VWRHWkhpREhWU1Z6ZWx2MENnZ0VBTDB6QW5ITm1HVXc4NDR2cgpRaVhsd2R5TmdHMVB1Y2NxL09RN2JHRFJYcER3djRNNkJ6TkxBNDBVYWl4aHQ3cmRJaWVUUC9pVFlWbUFFaGoyClFYNTQ2a25EOXhFVWJXbU1IWXR2Mmd0NWk0RldyZVJ4dDlNR1RqU2IwQzdEeTlGMHYrSFozTEUvMmh1NzFkMFYKTVhpZ1BXSlRaSDBBUFQvYTFZUnZ6Y0dDbzU2a3dOeGNENGx3SlNWRG9CN0wwTlJFR3V2Ukpra0dyRFhmaGVOMQovNVRzRFdhMHpoUnVVWXRZZlN5bytLcU1vZHhKWUtUTDlqRGRSOVNjUHhyVlFnZFMxYnRhOHE1eVNTQmtNNGt1CmxseHc0MnRFNUNMK3JWTktHb1VxSHdueE84cWZCRTdyVGRnN0l5aDlWamx1ajRLTU9Rd1hZMUdyU2tXZ1dkT2gKVjNvUjZRS0NBUUIrT3VEU21vTnBqR2FOUk1KdklvNFNDNlNVa1AyNEdvUVpoZ3pOL0F3Zmg3b0YyUHVtSlY4TQpnVDFEcGxJQlo4ZWlBcDU1TGJoNWJtOElCN0lWQ3JpYUF3VXBCMFhUQ082T1RPcmhZTlQ3b3lPRVV2OVlCSXJlCjFKTWoxbkxma0RVMm53UXQ0K1c5VmdDcFd5SXBpMjFnZnhNYW1kTmRGTXRLcXJZc1BJdkhpRWdNVTJVS1J0WlAKaDYwaHlyWWZkV2JTSHp4dG8vTVkvVStZNFFFOFJxVkJBT3NNZ0llVHp4aFJ1Qk40alZhR2R6TStHNURwa3J3MQpsQlhtUURtSExVbEs1K2lXeEh5WWg4T253aDBRTU5SQ3BKb0V3ZFZnRDd0V3hjSEFOV0tUejdxSTBKcW93Ykh5CnRLeHdkS0I3Q2tndnc2TUJSdmVKMTNiYWJ5R3Y3ZnNSQW9JQkFITjFtZjNldmpQbTZmQ1IwZGZycGVEdnExUWQKbzFmOWFQVUd1dFFCZUplQXVkeXVHZlRRdkN2WlRVMUxVdytBUHJzSnRBUlVEcU5JYXY2ZFYrVUdUWnNxeFdqWQpwaS83KzFiOWx5N2VzSHBrMnY2ckx0MHJHSUgvQmhPWUZVeEpEWmRib0RpemxZc3BSaE1nSWtGb3hjVUwxSDVlCkx1MzJDaVNERUgwWU9RNkJGSi9NZGVtOHRQWmtuc3pXejc4b3hrMWdmb1VFNXQ3SGJzRmt2dDI2cWNDWjVYOWEKdHBHY040dXQ3emE1NXIxQUxEUzhlTDhjRHd2a3d0T3JxS1lVSTcvTHByNXNjWlZwaFFXbmp4NUptdUF3UFFrMwpkSXB2R2Nabk8yOFo1cWdrYi9CSnhrWGJLSkNVSTJGUVRCbEZnVzlVNXA5NWJHdUlCNk41T0hqbmplTT0KLS0tLS1FTkQgUlNBIFBSSVZBVEUgS0VZLS0tLS0K + token: c38fbc7d6fdc12cf07893b421e510078b2daad26d2e4233f2251b7743e50401398024d59bf573c912768d086766cd8f0f92baf4d43e2eb63a97f9868bfd1ad47 diff --git a/kubernetes/linux/dockerbuild/existingClusterOnboarding.json b/kubernetes/linux/dockerbuild/existingClusterOnboarding.json new file mode 100644 index 000000000..73c8398dd --- /dev/null +++ b/kubernetes/linux/dockerbuild/existingClusterOnboarding.json @@ -0,0 +1,44 @@ +{ + "$schema": "https://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#", + "contentVersion": "1.0.0.0", + "parameters": { + "aksResourceId": { + "type": "string", + "metadata": { + "description": "AKS Cluster Resource ID" + } + }, + "aksResourceLocation": { + "type": "string", + "metadata": { + "description": "Location of the AKS resource e.g. \"East US\"" + } + }, + "workspaceResourceId": { + "type": "string", + "metadata": { + "description": "Azure Monitor Log Analytics Resource ID" + } + } + }, + "resources": [ + { + "name": "[split(parameters('aksResourceId'),'/')[8]]", + "type": "Microsoft.ContainerService/managedClusters", + "location": "[parameters('aksResourceLocation')]", + "apiVersion": "2018-03-31", + "properties": { + "mode": "Incremental", + "id": "[parameters('aksResourceId')]", + "addonProfiles": { + "omsagent": { + "enabled": false, + "config": { + "logAnalyticsWorkspaceResourceID": "[parameters('workspaceResourceId')]" + } + } + } + } + } + ] +} \ No newline at end of file diff --git a/kubernetes/linux/dockerbuild/existingClusterParam.json b/kubernetes/linux/dockerbuild/existingClusterParam.json new file mode 100644 index 000000000..011b95337 --- /dev/null +++ b/kubernetes/linux/dockerbuild/existingClusterParam.json @@ -0,0 +1,15 @@ +{ + "$schema": "https://schema.management.azure.com/schemas/2015-01-01/deploymentParameters.json#", + "contentVersion": "1.0.0.0", + "parameters": { + "aksResourceId": { + "value": "/subscriptions/b4b0cbab-64dc-42bc-9d95-ea7c8d0cf188/resourcegroups/khushi-test/providers/Microsoft.ContainerService/managedClusters/khushi-test" + }, + "aksResourceLocation": { + "value": "West US 2" + }, + "workspaceResourceId": { + "value": "/subscriptions/b4b0cbab-64dc-42bc-9d95-ea7c8d0cf188/resourcegroups/defaultresourcegroup-eus/providers/microsoft.operationalinsights/workspaces/defaultworkspace-b4b0cbab-64dc-42bc-9d95-ea7c8d0cf188-eus" + } + } +} \ No newline at end of file diff --git a/kubernetes/linux/dockerbuild/khushi-test b/kubernetes/linux/dockerbuild/khushi-test new file mode 100644 index 000000000..4a07d0fb9 --- /dev/null +++ b/kubernetes/linux/dockerbuild/khushi-test @@ -0,0 +1,20 @@ +apiVersion: v1 +clusters: +- cluster: + certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUU2VENDQXRHZ0F3SUJBZ0lSQUozU3djVWpyaUg5blhaWnZHVHFmWWN3RFFZSktvWklodmNOQVFFTEJRQXcKRFRFTE1Ba0dBMVVFQXhNQ1kyRXdJQmNOTWpFd05qSTBNVFl5TVRBMVdoZ1BNakExTVRBMk1qUXhOak14TURWYQpNQTB4Q3pBSkJnTlZCQU1UQW1OaE1JSUNJakFOQmdrcWhraUc5dzBCQVFFRkFBT0NBZzhBTUlJQ0NnS0NBZ0VBCnUvZFhmRnZvYi9zREdVa0VoVWNYcDVGVndBR2VyR01hOXJocVhPM3JSbFJQYlltTW9oOVUrQWV6Q0U4ckJ5WGwKbCtib2JNKzQzU2x1Y3BickkrOVpZcGEvYnYrY2dYRXRGOENzdDMwbm9LYjlGeno2cVFicHNuQ09hNllhdVRhVQovWVU2aVhCQm4vRnhYZ1BNRGFucTRFcVVnNXNVZkhuVFJCNFdhZmhBaXI2T2g4RjBoZzhyRS83dXJ2cVFhNllPCklNN2kzUHY5dG1GN1ZjRThocUFYZTdNUE4xd0R2Q21SUGZoZlZTejAxdDFlQlFtcDJ4YitHQ2lBRStYUHVVZ3EKZ3RkU21OL2tnYVdRMFFIWm9sRWdEbE1Hb1BjSXNkeXZvd2x4WThEMmlkNlluMS9jeEtvNHBmUjkrUVdieUZrRQp5b0VDM0M0MGVBZVJTZUtLdjlxSnJyNlp2WGRxZGs1WHpRREU2d1hqakorUUhGZHJyeHFqdE01Z0dkbThiRDdiCmg0WjA5c201djBQd1BQZWE5SG5WbURmbjlsQmtjZGRSTDJuV1NGSVZsbWdDMVdrV3huZXBmSGV3YmN4MnpnK1UKazc2UjMxamMyWW9NRnZpNmtXaTFLejYwZmh5N0VIVWxsRTRlbm9VZm5TM2dDZ2pxeFVJcWlKZFpzbGluSTdNbgpFS0sxZ21DRk96bmtMU3MzQ0FXTGFKNEN6VFJsQkZKWDYzTE5NR2U0NXBTRUJIVEFlTUdtbjY5Tjc2Y2lrejNiCmthYTJXZ05sTEFZVDBEZUtQR0RJNmFsU2pOckFaS3lkNS9rTzBYTGo4MFVtT2JZY1NpeXlwYlhZZ1Q5bWxna3IKVEhPZ29HKzlEaDQ3bWdWczJlaURVYWJ6RVpTUlBUbEorTk9iSkJRczQ1RUNBd0VBQWFOQ01FQXdEZ1lEVlIwUApBUUgvQkFRREFnS2tNQThHQTFVZEV3RUIvd1FGTUFNQkFmOHdIUVlEVlIwT0JCWUVGRWEzTFFUSHJFeHBZQ0VJCkJZNEEzd3k4S2ZqTU1BMEdDU3FHU0liM0RRRUJDd1VBQTRJQ0FRQlp2b0IwdVpZa3dYclBvTnp3LytmMEc1bFUKMENWYU56STNkQlNyd29lT1VMTnBJdDFad0E3REhxNTZNa0pUTzNlNnNQbzZKczFIbHAzRTRJaFJxZTI0NnNyQgpaam51czdTNDY1aHBFcDBhd1lHMzdnY05NMlBxbFo3dDZNdDhSQjVneHJPb3JNS1BSSWZiYnd6NDNvWHB6VHVpCkZ0QUk5Z2xjR3V1VlZ5Yk42a2N2Y2MyNE9zb0VGQ3VHU0dBMzhaUTZqY1NIRDUvQW9rcGd0cTZ4M0trNHR1TjIKS3dWcjdtekdvZGZrOTlKSVhkTWREVy9qTGt5RlRRQWhoNlY2ZThmZHNJbmtEZFdKOUI2UlFqSzFiMjRiQmowbwpjaGZkald3L2pLRzQ4YTFIS2tlWHk2MG5oU1FKaGFaRjBvTEprNElpU3lqNUtQMlpHUGg1WTZLSUlRT0oreE5HCmJiN3BTQ2h6dFFVUjR4dEZkc2pxZk9BZzdCZitYU3ZTNXExUU5FS3JaOHE2M2VMQVQ5aFUxVDVDVE5QVW1Zd2YKbjNQVEJHdVUrNThpWFM3VGVxVGVwYlZxOFlvU2hCeVc1cXdWa1loU1VWNzk5VGNVb2oreTJ5SDAxd3F2a0JlUQovZExXZ0srVE4xZnpsMVBXTk5CWW03SFgwbWpuZVRrWFB2QWFKMzNsS0RzdWpsR0RwYmkza09tbGN1TVJqYStUCm5iSjNRRkd5aDR2UWt6N3hMWEtDdFFBOHVUeXVNVWdTVzJZZ2ZZYWlNWG1wMG95RHNQRnI3aFFKMXN2THd0dksKd2M3Z1ZMV3ZIWTlYOVhqN0dSbFFwdlBmV1B4MkxESThFeEU1eTVQSXFqYjhTMFZ1aVU4REpZTlVEaDlodGhoVgpqRmxuVTd6ekw5dEE4RDZ6RWc9PQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0tCg== + server: https://khushi-test-dns-92573251.hcp.westus2.azmk8s.io:443 + name: khushi-test +contexts: +- context: + cluster: khushi-test + user: clusterUser_khushi-test_khushi-test + name: khushi-test +current-context: khushi-test +kind: Config +preferences: {} +users: +- name: clusterUser_khushi-test_khushi-test + user: + client-certificate-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUZIakNDQXdhZ0F3SUJBZ0lSQVBHQzNPazBaT2pxcXppS01CT3M3bEF3RFFZSktvWklodmNOQVFFTEJRQXcKRFRFTE1Ba0dBMVVFQXhNQ1kyRXdIaGNOTWpFd05qSTBNVFl5TVRBMVdoY05Nak13TmpJME1UWXpNVEExV2pBdwpNUmN3RlFZRFZRUUtFdzV6ZVhOMFpXMDZiV0Z6ZEdWeWN6RVZNQk1HQTFVRUF4TU1iV0Z6ZEdWeVkyeHBaVzUwCk1JSUNJakFOQmdrcWhraUc5dzBCQVFFRkFBT0NBZzhBTUlJQ0NnS0NBZ0VBdE1GUVhjQkdJYmduK1dleWlMOWkKYkg1Z2JvQ2w4QkZyb3dRWnpZbjRsSnBxQVVuRzR4b0QwbUZSNklBbkdQd0szTVdqNXJmcFNQUEdLZ29kbHBMTQo4azZSeVBNN1BwYTlzck1NWGJlMHZMQmJxb1lNeFc4aHFyTldBcEtYWitJUWI0TnkrY0VFbklMNlY4aHN4ME9kCmVVYTZJK2xib3k1VXVjN0FsL1pFVSt6NHRGaEhtNjc1VHNhWU9kOXozSTJpZ0pmQS9EZ044Q3ZWVWFHeVNwVHAKOGczOWpsaTJTNllaT0xRSlRSVnBZUEFLck1DQnlQclV1L3FhbFVMSXVSdE5SLyszdkpNS2pyZWQzQTd6MXYwdwo4NGw4ZTlJQWI0TDNwRDFTL1FzcnNldi9aTnN4dXljZVczNjlGeXFrNEZwWnZaVEwvb3A4OUdZa2ZaZmpXNmpiCmp1MXdFaklSYTJ3Z3BJdDlNSitTRHc3YlFtdmxNQVNhZmlwVnEyRDBXYmMveWhySFk5MGRwNEVIY1dOcDNkUlIKSU5YZ0U3cmdEa1RzK3drdlNlVUN5ckZZUTBBck1nZUl0NDNJR0srdVU5UW5HaWhNOG1Wa2tGVjM1SitQWTZTawpQNW5JRk83OGJYNkk4UVRsOXhEaVJsM3hGTHJGcE53Sjk2bWFlVkE5ZDd6NjdoZnFvc3o2c25YZ0I4L085UkhjCmhrUWhWZWt6c2o4T250WU9vK3V6K2llandaRGlVREV6N0dQeGI3NUw0aks4RERSalFtYys0dUZTOFh3ZFRrUEoKS3lZc3FkTGVJb3REWjlkYm9sUTF6WjlZS3hsNTcrQ3l2aU50ZGFCL0FRVUZ0dnNnZXY2bzdmYkhzY1VoZTRvbgpZcDRFOHVWRHcyMlFuakdweUlNRzZJY0NBd0VBQWFOV01GUXdEZ1lEVlIwUEFRSC9CQVFEQWdXZ01CTUdBMVVkCkpRUU1NQW9HQ0NzR0FRVUZCd01DTUF3R0ExVWRFd0VCL3dRQ01BQXdId1lEVlIwakJCZ3dGb0FVUnJjdEJNZXMKVEdsZ0lRZ0ZqZ0RmREx3cCtNd3dEUVlKS29aSWh2Y05BUUVMQlFBRGdnSUJBQTRubkxXUERBaUlka0QxV3NiegoxTnVlSnhhVzRzd2VEdWtyNDdxVEFweHA1dFVyK1N1OGVCQkg5WVorTW5XUHBFUEtkclQ5Q0xRNElyTzJXbEJXCmNmV1lZTGxOU0hFSjVOL1pBY0k2Q2Y2MklnOUZlbHdzYk5SNjhOWGozL21OZEpqa3dzVitVL0FwVXM4RTVXU2sKMUUzaU4xOWxEUnh6cFNXZ0R5dUJmQzhvWWRSZzMzZ0wyd3FjcmRQc3MvdlZnc013S2tBd2NwVHJ1ei84ZjZTSgpPdTFtWmlDRGFVUzdQdm9IQ2YvMEU0eDJzZmdqTE9scjE1eTc4K0dKVVdOcXBpVXQ2aWxHNVVKWVZqd1JCS0U4CkxJTGw4dW0xeTUxU2pmUzNkSnQ4b0l4WENSWW9uTStNRDA0ZEYzVHduSFRhVWljS05Va2x5b1VMQWhVN1VmREYKWkNqU1FVckFhZm5JajB0cFdDRHVuK1hEQ1pHRVVzV3BDeTdINWhJOFhqYjBRbEw3Vm5KRGk5alpBeWwwalJhVAp5OExrSTA2SXNwbVhlM2xJRzI5S0FRVmdGOWNsNjFDVTVWR25CM2ZHbUNzOUdtMStmTTNkc2hya055eHkvN1JNClBBUUhyVzhiWVo1c0NiQ09SUHBqc3NOandSVk16VjNtU2dhcmI0b1h1T3V2NTZoSnYxSnAxc2N1bS84SVlrQm4KdkE5RS9qa0E5RTVqd05uV0FNam9wRGtTZlVNN0NiVDdJMjZ5R001THdLdDc2NXY4bUJTVVBGcWt0amM5SVpoOApXZ2NKcGxuRHExOEhyTGtTQ3gvamRkR2I0WWphRDN1ajhyb0JMQ3NPRFR2K3NzWUo0Z21heWpHbkZRbFRLZlhpCnNLV2hwL0RvUXR6cWZtbHZyWDdlRnUrUwotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0tCg== + client-key-data: LS0tLS1CRUdJTiBSU0EgUFJJVkFURSBLRVktLS0tLQpNSUlKS0FJQkFBS0NBZ0VBdE1GUVhjQkdJYmduK1dleWlMOWliSDVnYm9DbDhCRnJvd1FaelluNGxKcHFBVW5HCjR4b0QwbUZSNklBbkdQd0szTVdqNXJmcFNQUEdLZ29kbHBMTThrNlJ5UE03UHBhOXNyTU1YYmUwdkxCYnFvWU0KeFc4aHFyTldBcEtYWitJUWI0TnkrY0VFbklMNlY4aHN4ME9kZVVhNkkrbGJveTVVdWM3QWwvWkVVK3o0dEZoSAptNjc1VHNhWU9kOXozSTJpZ0pmQS9EZ044Q3ZWVWFHeVNwVHA4ZzM5amxpMlM2WVpPTFFKVFJWcFlQQUtyTUNCCnlQclV1L3FhbFVMSXVSdE5SLyszdkpNS2pyZWQzQTd6MXYwdzg0bDhlOUlBYjRMM3BEMVMvUXNyc2V2L1pOc3gKdXljZVczNjlGeXFrNEZwWnZaVEwvb3A4OUdZa2ZaZmpXNmpianUxd0VqSVJhMndncEl0OU1KK1NEdzdiUW12bApNQVNhZmlwVnEyRDBXYmMveWhySFk5MGRwNEVIY1dOcDNkUlJJTlhnRTdyZ0RrVHMrd2t2U2VVQ3lyRllRMEFyCk1nZUl0NDNJR0srdVU5UW5HaWhNOG1Wa2tGVjM1SitQWTZTa1A1bklGTzc4Ylg2SThRVGw5eERpUmwzeEZMckYKcE53Sjk2bWFlVkE5ZDd6NjdoZnFvc3o2c25YZ0I4L085UkhjaGtRaFZla3pzajhPbnRZT28rdXoraWVqd1pEaQpVREV6N0dQeGI3NUw0aks4RERSalFtYys0dUZTOFh3ZFRrUEpLeVlzcWRMZUlvdERaOWRib2xRMXpaOVlLeGw1CjcrQ3l2aU50ZGFCL0FRVUZ0dnNnZXY2bzdmYkhzY1VoZTRvbllwNEU4dVZEdzIyUW5qR3B5SU1HNkljQ0F3RUEKQVFLQ0FnRUFoeHFMRUZmNzA0NUxTcTdJa2svQ3FHZFplcDdyNk9HTWUzOUFMaEkzWEJPQ3NrM0x1OG1oRzZtawpTRTBwYkJKUHFDNk4wSkNjYk92UXJYRjhKaVM1elU4T3hvMlBOYndFTkVLQ05yWnVBQzh0aE9ySE52ZFd5YmxiCnkzWjRkcXhSTHBpbzRxYno1R3c5NUVrcGpWTXJyQTlDYUQ4dlFHd0w1Z24waUdFSk4xWERtMEQwM1JmT3ZxUFAKK2tuYjF2bUNnZTFyME5teENWZ3FET2I0VmFpeXFEcnV6MnZkYmZQTG5FZHRRRkdGdjNBTCtvMjFORnpQajJYRQpEMWFvb3pzVjYzL21mS3ZIVTE5SlBjZHJXTUZNeW10MFFUUjVJT0JhYTBkdTlUWlpSWmFwQVdDWTdsTlI3SDl4CnBKUTZRbnNsNG44S2JzcGlsV2FYVmN5bW9zYzJ1Mzlzb3EyVUd3cjJoOHJ3MEdFNGdEY21wZ0RwY3FuRXlEVloKRUxoNG9CSmEvc1F0K0tiVk1zRVZUbUZkOXJyaDNEbGRuaHNPTVNvUldxWTZYZ01pNzQ1OVJPWWxPSEZ3YTdFTgpHUzFqMWF5dndDZmVUSGcrNHhFVUYybXcyQmd6UDNXOTZ0VmpOOFgxTmdFbHZLT1lCTTFEc1crYndSR2RldjErCm41R2N4WEZja3A4ZDY5QWtkQThsT2hOM2dzTVZ2Vk5rVmFBQWwzdzMrbkNIVHlNcStjeXJ6V1orSnZNNjZZdnIKZlQvSkQyZ3g0RW05ZEdkTjlxT0ZxTWtkMXN1dk9HOWJHbVB2N2JEb2hYV2dwL2FUMkk3WW1CYW1yemwvNGpPYwo0VTI3Z2JwTysxS2NjeDB6Zk9TYkJGZU91N2thYkJpc0JsdkExeThoQlBxVjRyUy9XZmtDZ2dFQkFOemFSODlXCnlsRlFsUjNhZXF6c3o2QlRLRW9ISndPelZvYXBRdzZZYjNiaTROczY1eXFoUllGWjlBV1BJdC9wQ090TEsyWnQKRG1oRTVVcUJreklQZ3EwK292OUtIUUk1aEVCV241U0dUWnNWSDdpMDdBUGhVeDVVWDROTEpMNWM5SklINlRZZAoyVC9CWGV3c1JFSiswOU5sN3NMTU53eWZlMUhGQU5ldUFXZ2hhNW1YVTNWOFBLeEY1N3NzcGI5YzlrZzVhc1dhClByRHRMRFRmd0hCQ0tlc3ZZMkVnUU9Tc0VNb0l1SWN0WnQvWDd3UzFjMldSbStzRis5WWJYV1lBaHNTQ0tpSGcKYlpJdG53cUg0YVJzWmRmTjlWWnpMTG8zdWpiK3N0RXA5RWtyNWJkeDRFZEUwSkZrblVTZzRYTHN2Q1RyektEMQpaV2dmdGQ0VFVXYmJMdE1DZ2dFQkFOR0ZiVm1GazNiYld1bFVKOXhZclpWNDBmZXZhUVFnQjg1Nm01VGxVN2JKCktKSms1U2J2TThCcEMrSkFHRk1vWXdlZENYQlppV3YxWHExV20vTURFaWl3V1lBZW1HZ0tsM1Bzam9aNEE3WXcKNkRicmRTcDY1WXJ0aFh6aW01Z3RrbkhoVDhmVGhSUUZFZmZ1MnFpY1lOMTFSZ0liVW1NU0xyc0c3aEtJN3o4dApUaEVqbmZQSXRob1R2WkN3R0ZjMi92eEhYTXl0ancra0pNTnlXL0pCQUgyUU1rdmJHSHdDckE0Y1ovTWtSc1l0CldEemkrUTZTcStWeDVyUDVRVzNUUFhDeGhNMWgrZ1FGcmsyVTYrcG9waHQvQ0NCVzVRT3duRVdkamtvcXoyRm0KTW9ZblpDUmpjMXdJWmJKYjNjT2thNG0yR3I2VWRHWkhpREhWU1Z6ZWx2MENnZ0VBTDB6QW5ITm1HVXc4NDR2cgpRaVhsd2R5TmdHMVB1Y2NxL09RN2JHRFJYcER3djRNNkJ6TkxBNDBVYWl4aHQ3cmRJaWVUUC9pVFlWbUFFaGoyClFYNTQ2a25EOXhFVWJXbU1IWXR2Mmd0NWk0RldyZVJ4dDlNR1RqU2IwQzdEeTlGMHYrSFozTEUvMmh1NzFkMFYKTVhpZ1BXSlRaSDBBUFQvYTFZUnZ6Y0dDbzU2a3dOeGNENGx3SlNWRG9CN0wwTlJFR3V2Ukpra0dyRFhmaGVOMQovNVRzRFdhMHpoUnVVWXRZZlN5bytLcU1vZHhKWUtUTDlqRGRSOVNjUHhyVlFnZFMxYnRhOHE1eVNTQmtNNGt1CmxseHc0MnRFNUNMK3JWTktHb1VxSHdueE84cWZCRTdyVGRnN0l5aDlWamx1ajRLTU9Rd1hZMUdyU2tXZ1dkT2gKVjNvUjZRS0NBUUIrT3VEU21vTnBqR2FOUk1KdklvNFNDNlNVa1AyNEdvUVpoZ3pOL0F3Zmg3b0YyUHVtSlY4TQpnVDFEcGxJQlo4ZWlBcDU1TGJoNWJtOElCN0lWQ3JpYUF3VXBCMFhUQ082T1RPcmhZTlQ3b3lPRVV2OVlCSXJlCjFKTWoxbkxma0RVMm53UXQ0K1c5VmdDcFd5SXBpMjFnZnhNYW1kTmRGTXRLcXJZc1BJdkhpRWdNVTJVS1J0WlAKaDYwaHlyWWZkV2JTSHp4dG8vTVkvVStZNFFFOFJxVkJBT3NNZ0llVHp4aFJ1Qk40alZhR2R6TStHNURwa3J3MQpsQlhtUURtSExVbEs1K2lXeEh5WWg4T253aDBRTU5SQ3BKb0V3ZFZnRDd0V3hjSEFOV0tUejdxSTBKcW93Ykh5CnRLeHdkS0I3Q2tndnc2TUJSdmVKMTNiYWJ5R3Y3ZnNSQW9JQkFITjFtZjNldmpQbTZmQ1IwZGZycGVEdnExUWQKbzFmOWFQVUd1dFFCZUplQXVkeXVHZlRRdkN2WlRVMUxVdytBUHJzSnRBUlVEcU5JYXY2ZFYrVUdUWnNxeFdqWQpwaS83KzFiOWx5N2VzSHBrMnY2ckx0MHJHSUgvQmhPWUZVeEpEWmRib0RpemxZc3BSaE1nSWtGb3hjVUwxSDVlCkx1MzJDaVNERUgwWU9RNkJGSi9NZGVtOHRQWmtuc3pXejc4b3hrMWdmb1VFNXQ3SGJzRmt2dDI2cWNDWjVYOWEKdHBHY040dXQ3emE1NXIxQUxEUzhlTDhjRHd2a3d0T3JxS1lVSTcvTHByNXNjWlZwaFFXbmp4NUptdUF3UFFrMwpkSXB2R2Nabk8yOFo1cWdrYi9CSnhrWGJLSkNVSTJGUVRCbEZnVzlVNXA5NWJHdUlCNk41T0hqbmplTT0KLS0tLS1FTkQgUlNBIFBSSVZBVEUgS0VZLS0tLS0K + token: c38fbc7d6fdc12cf07893b421e510078b2daad26d2e4233f2251b7743e50401398024d59bf573c912768d086766cd8f0f92baf4d43e2eb63a97f9868bfd1ad47 diff --git a/kubernetes/omsagent.yaml b/kubernetes/omsagent.yaml index 617c81f38..80f121a5d 100644 --- a/kubernetes/omsagent.yaml +++ b/kubernetes/omsagent.yaml @@ -333,8 +333,8 @@ metadata: type: Opaque data: #BASE64 ENCODED (Both WSID & KEY) INSIDE DOUBLE QUOTE ("") - WSID: "VALUE_WSID" - KEY: "VALUE_KEY" + WSID: "ZTNkYmI1OTItNDhhNS00N2IwLWE2MTAtMTUwYzU1ZmM1Y2ZmCg==" + KEY: "U3htc215eXY3UXYrZzk3QXZVdlZzd3FlSW1ENHoyRlpjdHJGamtHUUl5NUx5bDhYUEc1RmxuQU5wRkhFLzNJczBWSjd2elFuS3FIa2NDOVhwWU5UR3c9PQo=" --- apiVersion: apps/v1 kind: DaemonSet @@ -368,8 +368,8 @@ spec: value: "3" containers: - name: omsagent - image: "mcr.microsoft.com/azuremonitor/containerinsights/ciprod:ciprod06112021" - imagePullPolicy: IfNotPresent + image: "mcr.microsoft.com/azuremonitor/containerinsights/cidev:khushi-test-5" + imagePullPolicy: Always resources: limits: cpu: 500m @@ -380,9 +380,9 @@ spec: env: # azure devops pipeline uses AKS_RESOURCE_ID and AKS_REGION hence ensure to uncomment these - name: AKS_RESOURCE_ID - value: "VALUE_AKS_RESOURCE_ID_VALUE" + value: "/subscriptions/b4b0cbab-64dc-42bc-9d95-ea7c8d0cf188/resourcegroups/khushi-test/providers/Microsoft.ContainerService/managedClusters/khushi-test" - name: AKS_REGION - value: "VALUE_AKS_RESOURCE_REGION_VALUE" + value: "West US 2" # this used for e2e test and setting this just emits some additional log statements which used for the e2e tests - name: ISTEST value: "true" @@ -446,8 +446,8 @@ spec: timeoutSeconds: 15 #Only in sidecar scraping mode - name: omsagent-prometheus - image: "mcr.microsoft.com/azuremonitor/containerinsights/ciprod:ciprod06112021" - imagePullPolicy: IfNotPresent + image: "mcr.microsoft.com/azuremonitor/containerinsights/cidev:khushi-test-5" + imagePullPolicy: Always resources: limits: cpu: 500m @@ -458,9 +458,9 @@ spec: env: # azure devops pipeline uses AKS_RESOURCE_ID and AKS_REGION hence ensure to uncomment these - name: AKS_RESOURCE_ID - value: "VALUE_AKS_RESOURCE_ID_VALUE" + value: "/subscriptions/b4b0cbab-64dc-42bc-9d95-ea7c8d0cf188/resourcegroups/khushi-test/providers/Microsoft.ContainerService/managedClusters/khushi-test" - name: AKS_REGION - value: "VALUE_AKS_RESOURCE_REGION_VALUE" + value: "West US 2" #Uncomment below two lines for ACS clusters and set the cluster names manually. Also comment out the above two lines for ACS clusters #- name: ACS_RESOURCE_NAME # value: "my_acs_cluster_name" @@ -589,8 +589,8 @@ spec: serviceAccountName: omsagent containers: - name: omsagent - image: "mcr.microsoft.com/azuremonitor/containerinsights/ciprod:ciprod06112021" - imagePullPolicy: IfNotPresent + image: "mcr.microsoft.com/azuremonitor/containerinsights/cidev:khushi-test-5" + imagePullPolicy: Always resources: limits: cpu: 1 @@ -600,9 +600,9 @@ spec: memory: 250Mi env: - name: AKS_RESOURCE_ID - value: "VALUE_AKS_RESOURCE_ID_VALUE" + value: "/subscriptions/b4b0cbab-64dc-42bc-9d95-ea7c8d0cf188/resourcegroups/khushi-test/providers/Microsoft.ContainerService/managedClusters/khushi-test" - name: AKS_REGION - value: "VALUE_AKS_RESOURCE_REGION_VALUE" + value: "West US 2" # this used for e2e test and setting this just emits some additional log statements which used for the e2e tests - name: ISTEST value: "true" @@ -651,15 +651,15 @@ spec: - mountPath: /etc/config/osm-settings name: osm-settings-vol-config readOnly: true - livenessProbe: - exec: - command: - - /bin/bash - - -c - - /opt/livenessprobe.sh - initialDelaySeconds: 60 - periodSeconds: 60 - timeoutSeconds: 15 + # livenessProbe: + # exec: + # command: + # - /bin/bash + # - -c + # - /opt/livenessprobe.sh + # initialDelaySeconds: 60 + # periodSeconds: 60 + # timeoutSeconds: 15 affinity: nodeAffinity: # affinity to schedule on to ephemeral os node if its available @@ -769,9 +769,9 @@ spec: env: # azure devops pipeline uses AKS_RESOURCE_ID and AKS_REGION hence ensure to uncomment these - name: AKS_RESOURCE_ID - value: "VALUE_AKS_RESOURCE_ID_VALUE" + value: "/subscriptions/b4b0cbab-64dc-42bc-9d95-ea7c8d0cf188/resourcegroups/khushi-test/providers/Microsoft.ContainerService/managedClusters/khushi-test" - name: AKS_REGION - value: "VALUE_AKS_RESOURCE_REGION_VALUE" + value: "West US 2" #- name: ACS_RESOURCE_NAME # value: "my_acs_cluster_name" - name: CONTROLLER_TYPE diff --git a/source/plugins/ruby/in_kube_podinventory.rb b/source/plugins/ruby/in_kube_podinventory.rb index 0bbfde9d8..4f8eb2983 100644 --- a/source/plugins/ruby/in_kube_podinventory.rb +++ b/source/plugins/ruby/in_kube_podinventory.rb @@ -85,7 +85,8 @@ def start } getTokenStr = "Bearer " + KubernetesApiClient.getTokenStr auth_options = { bearer_token: KubernetesApiClient.getTokenStr } - @KubernetesWatchClient = Kubeclient::Client.new('https://#{ENV["KUBERNETES_SERVICE_HOST"]}:#{ENV["KUBERNETES_PORT_443_TCP_PORT"]}/api/', "v1", ssl_options: ssl_options, auth_options: auth_options) + @KubernetesWatchClient = Kubeclient::Client.new("https://#{ENV["KUBERNETES_SERVICE_HOST"]}:#{ENV["KUBERNETES_PORT_443_TCP_PORT"]}/api/", "v1", ssl_options: ssl_options, auth_options: auth_options, as: :parsed) + $log.info("in_kube:podinventory::start: successfully created kubernetes watch client") $log.info("in_kube_podinventory::start: PODS_EMIT_STREAM_BATCH_SIZE @ #{@PODS_EMIT_STREAM_BATCH_SIZE}") @finished = false @condition = ConditionVariable.new @@ -109,21 +110,18 @@ def shutdown end end - def write_to_file(podInventory) - $log.info("in_kube_podinventory::write_to_file: inside write to file function") - - File.write("testing-podinventory.json", JSON.pretty_generate(podInventory)) - - $log.info("in_kube_podinventory::write_to_file: successfully done writing to file") - - end + # def write_to_file(podInventory) + # $log.info("in_kube_podinventory::write_to_file: inside write to file function") + # File.write("testing-podinventory.json", JSON.pretty_generate(podInventory)) + # $log.info("in_kube_podinventory::write_to_file: successfully done writing to file") + # end def watch $log.info("in_kube_podinventory::watch: entered watch function") # TODO: delete later, don't want to make another API call here - File.open("testing-podinventory.json", "w") - $log.info("in_kube_podinventory::watch : successfully opened json file for writing") + # File.open("testing-podinventory.json", "w") + # $log.info("in_kube_podinventory::watch : successfully opened json file for writing") @podsAPIE2ELatencyMs = 0 podsAPIChunkStartTime = (Time.now.to_f * 1000).to_i @@ -138,7 +136,8 @@ def watch @podsAPIE2ELatencyMs = (podsAPIChunkEndTime - podsAPIChunkStartTime) if (!podInventory.nil? && !podInventory.empty? && podInventory.key?("items") && !podInventory["items"].nil? && !podInventory["items"].empty?) $log.info("in_kube_podinventory::watch : number of pod items :#{podInventory["items"].length} from Kube API @ #{Time.now.utc.iso8601}") - write_to_file(podInventory) + $log.info("in_kube_podinventory::watch : time to write to a file and emit to backend - functionality later") + # write_to_file(podInventory) # parse_and_emit_records(podInventory, serviceRecords, continuationToken, batchTime) else $log.warn "in_kube_podinventory::watch:Received empty podInventory" @@ -146,26 +145,28 @@ def watch #If we receive a continuation token, make calls, process and flush data until we have processed all data while (!continuationToken.nil? && !continuationToken.empty?) + $log.info("in_kube_podinventory::watch : continuation token was not null or empty so round two of gettings pod inventory") podsAPIChunkStartTime = (Time.now.to_f * 1000).to_i continuationToken, podInventory = KubernetesApiClient.getResourcesAndContinuationToken("pods?limit=#{@PODS_CHUNK_SIZE}&continue=#{continuationToken}") podsAPIChunkEndTime = (Time.now.to_f * 1000).to_i @podsAPIE2ELatencyMs = @podsAPIE2ELatencyMs + (podsAPIChunkEndTime - podsAPIChunkStartTime) if (!podInventory.nil? && !podInventory.empty? && podInventory.key?("items") && !podInventory["items"].nil? && !podInventory["items"].empty?) $log.info("in_kube_podinventory::watch : number of pod items :#{podInventory["items"].length} from Kube API @ #{Time.now.utc.iso8601}") - write_to_file(podInventory) + # write_to_file(podInventory) # parse_and_emit_records(podInventory, serviceRecords, continuationToken, batchTime) else $log.warn "in_kube_podinventory::watch:Received empty podInventory" end end - $log.info("finished initial write to pod inventory file") + # $log.info("finished initial write to pod inventory file") + $log.info("in_kube_podinventory::watch : finished getting pods, about to begin infinite loop for watch") loop do $log.info("in_kube_pod_inventory::watch: inside infinite loop for watch pods") begin @KubernetesWatchClient.watch_pods(resource_version: @collection_version, as: :parsed) do |notice| - $log.info("in_kube_podinventory::watch: inside watch pods! Time: #{Time.now.utc.iso8601}") + $log.info("in_kube_podinventory::watch: received a notice, inside watch pods! Time: #{Time.now.utc.iso8601}") # if !notice["object"].nil? && !notice["object"].empty? # $log.info("in_kube_podinventory::watch: received a notice of type #{notice["type"]}") From 006c86815f1c175b62fc4566c384dd1495444b57 Mon Sep 17 00:00:00 2001 From: Khushi Chaudhari Date: Mon, 19 Jul 2021 08:54:00 -0700 Subject: [PATCH 10/76] porting working test code to agent --- kubernetes/linux/setup.sh | 4 +- source/plugins/ruby/in_kube_podinventory.rb | 50 ++++++++++++--------- 2 files changed, 31 insertions(+), 23 deletions(-) diff --git a/kubernetes/linux/setup.sh b/kubernetes/linux/setup.sh index c91e0bd29..d140e1603 100644 --- a/kubernetes/linux/setup.sh +++ b/kubernetes/linux/setup.sh @@ -53,8 +53,8 @@ gem install fluentd -v "1.12.2" --no-document fluentd --setup ./fluent gem install gyoku iso8601 --no-doc # kubeclient gem -apt-get install libmagickwand-dev -y -gem install kubeclient --no-document +sudo apt-get install libmagickwand-dev -y +sudo gem install kubeclient --no-document rm -f $TMPDIR/docker-cimprov*.sh diff --git a/source/plugins/ruby/in_kube_podinventory.rb b/source/plugins/ruby/in_kube_podinventory.rb index 4f8eb2983..6ea5e3807 100644 --- a/source/plugins/ruby/in_kube_podinventory.rb +++ b/source/plugins/ruby/in_kube_podinventory.rb @@ -105,59 +105,67 @@ def shutdown @finished = true @condition.signal } + @watchthread.join @thread.join super # This super must be at the end of shutdown method end end - # def write_to_file(podInventory) - # $log.info("in_kube_podinventory::write_to_file: inside write to file function") - # File.write("testing-podinventory.json", JSON.pretty_generate(podInventory)) - # $log.info("in_kube_podinventory::write_to_file: successfully done writing to file") - # end - - def watch - $log.info("in_kube_podinventory::watch: entered watch function") + def write_to_file(podInventory) + $log.info("in_kube_podinventory:: write_to_file : inside write to file function") + File.write("testing-podinventory.json", JSON.pretty_generate(podInventory)) + $log.info("in_kube_podinventory:: write_to_file : successfully done writing to file") + end - # TODO: delete later, don't want to make another API call here - # File.open("testing-podinventory.json", "w") - # $log.info("in_kube_podinventory::watch : successfully opened json file for writing") + def initial_write + $log.info("in_kube_podinventory::watch - initial_write: entered initial write function") @podsAPIE2ELatencyMs = 0 podsAPIChunkStartTime = (Time.now.to_f * 1000).to_i + # Initializing continuation token to nil continuationToken = nil - $log.info("in_kube_podinventory::watch : Getting pods from Kube API @ #{Time.now.utc.iso8601}") + $log.info("in_kube_podinventory::watch - initial_write : Getting pods from Kube API @ #{Time.now.utc.iso8601}") + continuationToken, podInventory = KubernetesApiClient.getResourcesAndContinuationToken("pods?limit=#{@PODS_CHUNK_SIZE}") @collection_version = podInventory["metadata"]["resourceVersion"] - $log.info("in_kube_podinventory::watch : received collection version: #{@collection_version}") + + $log.info("in_kube_podinventory::watch - initial_write : received collection version: #{@collection_version}") $log.info("in_kube_podinventory::watch : Done getting pods from Kube API @ #{Time.now.utc.iso8601}") + podsAPIChunkEndTime = (Time.now.to_f * 1000).to_i @podsAPIE2ELatencyMs = (podsAPIChunkEndTime - podsAPIChunkStartTime) + if (!podInventory.nil? && !podInventory.empty? && podInventory.key?("items") && !podInventory["items"].nil? && !podInventory["items"].empty?) - $log.info("in_kube_podinventory::watch : number of pod items :#{podInventory["items"].length} from Kube API @ #{Time.now.utc.iso8601}") - $log.info("in_kube_podinventory::watch : time to write to a file and emit to backend - functionality later") - # write_to_file(podInventory) + $log.info("in_kube_podinventory::watch - initial_write : number of pod items :#{podInventory["items"].length} from Kube API @ #{Time.now.utc.iso8601}") + $log.info("in_kube_podinventory::watch - initial_write : time to write to a file and emit to backend - functionality later") + write_to_file(podInventory) # parse_and_emit_records(podInventory, serviceRecords, continuationToken, batchTime) else - $log.warn "in_kube_podinventory::watch:Received empty podInventory" + $log.warn "in_kube_podinventory::watch - initial_write : Received empty podInventory" end #If we receive a continuation token, make calls, process and flush data until we have processed all data while (!continuationToken.nil? && !continuationToken.empty?) - $log.info("in_kube_podinventory::watch : continuation token was not null or empty so round two of gettings pod inventory") + $log.info("in_kube_podinventory::watch - initial_write : continuation token was not null or empty so round two of gettings pod inventory") podsAPIChunkStartTime = (Time.now.to_f * 1000).to_i continuationToken, podInventory = KubernetesApiClient.getResourcesAndContinuationToken("pods?limit=#{@PODS_CHUNK_SIZE}&continue=#{continuationToken}") podsAPIChunkEndTime = (Time.now.to_f * 1000).to_i @podsAPIE2ELatencyMs = @podsAPIE2ELatencyMs + (podsAPIChunkEndTime - podsAPIChunkStartTime) if (!podInventory.nil? && !podInventory.empty? && podInventory.key?("items") && !podInventory["items"].nil? && !podInventory["items"].empty?) - $log.info("in_kube_podinventory::watch : number of pod items :#{podInventory["items"].length} from Kube API @ #{Time.now.utc.iso8601}") - # write_to_file(podInventory) + $log.info("in_kube_podinventory::watch - initial_write : number of pod items :#{podInventory["items"].length} from Kube API @ #{Time.now.utc.iso8601}") + write_to_file(podInventory) # parse_and_emit_records(podInventory, serviceRecords, continuationToken, batchTime) else - $log.warn "in_kube_podinventory::watch:Received empty podInventory" + $log.warn "in_kube_podinventory::watch - initial_write : Received empty podInventory" end end + end + + + def watch + $log.info("in_kube_podinventory::watch : entered watch function - about to call initial write") + initial_write # $log.info("finished initial write to pod inventory file") $log.info("in_kube_podinventory::watch : finished getting pods, about to begin infinite loop for watch") From ccc51fb000c65f60e44421dda7ee6d163bf3fabb Mon Sep 17 00:00:00 2001 From: Khushi Chaudhari Date: Mon, 19 Jul 2021 11:47:33 -0700 Subject: [PATCH 11/76] fixed yaml file --- .../dockerbuild/existingClusterParam.json | 2 +- kubernetes/linux/dockerbuild/khushi-test | 20 ------------------- .../linux/dockerbuild/khushi-watch-test | 20 +++++++++++++++++++ kubernetes/omsagent.yaml | 14 ++++++------- 4 files changed, 28 insertions(+), 28 deletions(-) delete mode 100644 kubernetes/linux/dockerbuild/khushi-test create mode 100644 kubernetes/linux/dockerbuild/khushi-watch-test diff --git a/kubernetes/linux/dockerbuild/existingClusterParam.json b/kubernetes/linux/dockerbuild/existingClusterParam.json index 011b95337..409e82c1e 100644 --- a/kubernetes/linux/dockerbuild/existingClusterParam.json +++ b/kubernetes/linux/dockerbuild/existingClusterParam.json @@ -3,7 +3,7 @@ "contentVersion": "1.0.0.0", "parameters": { "aksResourceId": { - "value": "/subscriptions/b4b0cbab-64dc-42bc-9d95-ea7c8d0cf188/resourcegroups/khushi-test/providers/Microsoft.ContainerService/managedClusters/khushi-test" + "value": "/subscriptions/b4b0cbab-64dc-42bc-9d95-ea7c8d0cf188/resourcegroups/khushi-watch-test/providers/Microsoft.ContainerService/managedClusters/khushi-watch-test" }, "aksResourceLocation": { "value": "West US 2" diff --git a/kubernetes/linux/dockerbuild/khushi-test b/kubernetes/linux/dockerbuild/khushi-test deleted file mode 100644 index 4a07d0fb9..000000000 --- a/kubernetes/linux/dockerbuild/khushi-test +++ /dev/null @@ -1,20 +0,0 @@ -apiVersion: v1 -clusters: -- cluster: - certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUU2VENDQXRHZ0F3SUJBZ0lSQUozU3djVWpyaUg5blhaWnZHVHFmWWN3RFFZSktvWklodmNOQVFFTEJRQXcKRFRFTE1Ba0dBMVVFQXhNQ1kyRXdJQmNOTWpFd05qSTBNVFl5TVRBMVdoZ1BNakExTVRBMk1qUXhOak14TURWYQpNQTB4Q3pBSkJnTlZCQU1UQW1OaE1JSUNJakFOQmdrcWhraUc5dzBCQVFFRkFBT0NBZzhBTUlJQ0NnS0NBZ0VBCnUvZFhmRnZvYi9zREdVa0VoVWNYcDVGVndBR2VyR01hOXJocVhPM3JSbFJQYlltTW9oOVUrQWV6Q0U4ckJ5WGwKbCtib2JNKzQzU2x1Y3BickkrOVpZcGEvYnYrY2dYRXRGOENzdDMwbm9LYjlGeno2cVFicHNuQ09hNllhdVRhVQovWVU2aVhCQm4vRnhYZ1BNRGFucTRFcVVnNXNVZkhuVFJCNFdhZmhBaXI2T2g4RjBoZzhyRS83dXJ2cVFhNllPCklNN2kzUHY5dG1GN1ZjRThocUFYZTdNUE4xd0R2Q21SUGZoZlZTejAxdDFlQlFtcDJ4YitHQ2lBRStYUHVVZ3EKZ3RkU21OL2tnYVdRMFFIWm9sRWdEbE1Hb1BjSXNkeXZvd2x4WThEMmlkNlluMS9jeEtvNHBmUjkrUVdieUZrRQp5b0VDM0M0MGVBZVJTZUtLdjlxSnJyNlp2WGRxZGs1WHpRREU2d1hqakorUUhGZHJyeHFqdE01Z0dkbThiRDdiCmg0WjA5c201djBQd1BQZWE5SG5WbURmbjlsQmtjZGRSTDJuV1NGSVZsbWdDMVdrV3huZXBmSGV3YmN4MnpnK1UKazc2UjMxamMyWW9NRnZpNmtXaTFLejYwZmh5N0VIVWxsRTRlbm9VZm5TM2dDZ2pxeFVJcWlKZFpzbGluSTdNbgpFS0sxZ21DRk96bmtMU3MzQ0FXTGFKNEN6VFJsQkZKWDYzTE5NR2U0NXBTRUJIVEFlTUdtbjY5Tjc2Y2lrejNiCmthYTJXZ05sTEFZVDBEZUtQR0RJNmFsU2pOckFaS3lkNS9rTzBYTGo4MFVtT2JZY1NpeXlwYlhZZ1Q5bWxna3IKVEhPZ29HKzlEaDQ3bWdWczJlaURVYWJ6RVpTUlBUbEorTk9iSkJRczQ1RUNBd0VBQWFOQ01FQXdEZ1lEVlIwUApBUUgvQkFRREFnS2tNQThHQTFVZEV3RUIvd1FGTUFNQkFmOHdIUVlEVlIwT0JCWUVGRWEzTFFUSHJFeHBZQ0VJCkJZNEEzd3k4S2ZqTU1BMEdDU3FHU0liM0RRRUJDd1VBQTRJQ0FRQlp2b0IwdVpZa3dYclBvTnp3LytmMEc1bFUKMENWYU56STNkQlNyd29lT1VMTnBJdDFad0E3REhxNTZNa0pUTzNlNnNQbzZKczFIbHAzRTRJaFJxZTI0NnNyQgpaam51czdTNDY1aHBFcDBhd1lHMzdnY05NMlBxbFo3dDZNdDhSQjVneHJPb3JNS1BSSWZiYnd6NDNvWHB6VHVpCkZ0QUk5Z2xjR3V1VlZ5Yk42a2N2Y2MyNE9zb0VGQ3VHU0dBMzhaUTZqY1NIRDUvQW9rcGd0cTZ4M0trNHR1TjIKS3dWcjdtekdvZGZrOTlKSVhkTWREVy9qTGt5RlRRQWhoNlY2ZThmZHNJbmtEZFdKOUI2UlFqSzFiMjRiQmowbwpjaGZkald3L2pLRzQ4YTFIS2tlWHk2MG5oU1FKaGFaRjBvTEprNElpU3lqNUtQMlpHUGg1WTZLSUlRT0oreE5HCmJiN3BTQ2h6dFFVUjR4dEZkc2pxZk9BZzdCZitYU3ZTNXExUU5FS3JaOHE2M2VMQVQ5aFUxVDVDVE5QVW1Zd2YKbjNQVEJHdVUrNThpWFM3VGVxVGVwYlZxOFlvU2hCeVc1cXdWa1loU1VWNzk5VGNVb2oreTJ5SDAxd3F2a0JlUQovZExXZ0srVE4xZnpsMVBXTk5CWW03SFgwbWpuZVRrWFB2QWFKMzNsS0RzdWpsR0RwYmkza09tbGN1TVJqYStUCm5iSjNRRkd5aDR2UWt6N3hMWEtDdFFBOHVUeXVNVWdTVzJZZ2ZZYWlNWG1wMG95RHNQRnI3aFFKMXN2THd0dksKd2M3Z1ZMV3ZIWTlYOVhqN0dSbFFwdlBmV1B4MkxESThFeEU1eTVQSXFqYjhTMFZ1aVU4REpZTlVEaDlodGhoVgpqRmxuVTd6ekw5dEE4RDZ6RWc9PQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0tCg== - server: https://khushi-test-dns-92573251.hcp.westus2.azmk8s.io:443 - name: khushi-test -contexts: -- context: - cluster: khushi-test - user: clusterUser_khushi-test_khushi-test - name: khushi-test -current-context: khushi-test -kind: Config -preferences: {} -users: -- name: clusterUser_khushi-test_khushi-test - user: - client-certificate-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUZIakNDQXdhZ0F3SUJBZ0lSQVBHQzNPazBaT2pxcXppS01CT3M3bEF3RFFZSktvWklodmNOQVFFTEJRQXcKRFRFTE1Ba0dBMVVFQXhNQ1kyRXdIaGNOTWpFd05qSTBNVFl5TVRBMVdoY05Nak13TmpJME1UWXpNVEExV2pBdwpNUmN3RlFZRFZRUUtFdzV6ZVhOMFpXMDZiV0Z6ZEdWeWN6RVZNQk1HQTFVRUF4TU1iV0Z6ZEdWeVkyeHBaVzUwCk1JSUNJakFOQmdrcWhraUc5dzBCQVFFRkFBT0NBZzhBTUlJQ0NnS0NBZ0VBdE1GUVhjQkdJYmduK1dleWlMOWkKYkg1Z2JvQ2w4QkZyb3dRWnpZbjRsSnBxQVVuRzR4b0QwbUZSNklBbkdQd0szTVdqNXJmcFNQUEdLZ29kbHBMTQo4azZSeVBNN1BwYTlzck1NWGJlMHZMQmJxb1lNeFc4aHFyTldBcEtYWitJUWI0TnkrY0VFbklMNlY4aHN4ME9kCmVVYTZJK2xib3k1VXVjN0FsL1pFVSt6NHRGaEhtNjc1VHNhWU9kOXozSTJpZ0pmQS9EZ044Q3ZWVWFHeVNwVHAKOGczOWpsaTJTNllaT0xRSlRSVnBZUEFLck1DQnlQclV1L3FhbFVMSXVSdE5SLyszdkpNS2pyZWQzQTd6MXYwdwo4NGw4ZTlJQWI0TDNwRDFTL1FzcnNldi9aTnN4dXljZVczNjlGeXFrNEZwWnZaVEwvb3A4OUdZa2ZaZmpXNmpiCmp1MXdFaklSYTJ3Z3BJdDlNSitTRHc3YlFtdmxNQVNhZmlwVnEyRDBXYmMveWhySFk5MGRwNEVIY1dOcDNkUlIKSU5YZ0U3cmdEa1RzK3drdlNlVUN5ckZZUTBBck1nZUl0NDNJR0srdVU5UW5HaWhNOG1Wa2tGVjM1SitQWTZTawpQNW5JRk83OGJYNkk4UVRsOXhEaVJsM3hGTHJGcE53Sjk2bWFlVkE5ZDd6NjdoZnFvc3o2c25YZ0I4L085UkhjCmhrUWhWZWt6c2o4T250WU9vK3V6K2llandaRGlVREV6N0dQeGI3NUw0aks4RERSalFtYys0dUZTOFh3ZFRrUEoKS3lZc3FkTGVJb3REWjlkYm9sUTF6WjlZS3hsNTcrQ3l2aU50ZGFCL0FRVUZ0dnNnZXY2bzdmYkhzY1VoZTRvbgpZcDRFOHVWRHcyMlFuakdweUlNRzZJY0NBd0VBQWFOV01GUXdEZ1lEVlIwUEFRSC9CQVFEQWdXZ01CTUdBMVVkCkpRUU1NQW9HQ0NzR0FRVUZCd01DTUF3R0ExVWRFd0VCL3dRQ01BQXdId1lEVlIwakJCZ3dGb0FVUnJjdEJNZXMKVEdsZ0lRZ0ZqZ0RmREx3cCtNd3dEUVlKS29aSWh2Y05BUUVMQlFBRGdnSUJBQTRubkxXUERBaUlka0QxV3NiegoxTnVlSnhhVzRzd2VEdWtyNDdxVEFweHA1dFVyK1N1OGVCQkg5WVorTW5XUHBFUEtkclQ5Q0xRNElyTzJXbEJXCmNmV1lZTGxOU0hFSjVOL1pBY0k2Q2Y2MklnOUZlbHdzYk5SNjhOWGozL21OZEpqa3dzVitVL0FwVXM4RTVXU2sKMUUzaU4xOWxEUnh6cFNXZ0R5dUJmQzhvWWRSZzMzZ0wyd3FjcmRQc3MvdlZnc013S2tBd2NwVHJ1ei84ZjZTSgpPdTFtWmlDRGFVUzdQdm9IQ2YvMEU0eDJzZmdqTE9scjE1eTc4K0dKVVdOcXBpVXQ2aWxHNVVKWVZqd1JCS0U4CkxJTGw4dW0xeTUxU2pmUzNkSnQ4b0l4WENSWW9uTStNRDA0ZEYzVHduSFRhVWljS05Va2x5b1VMQWhVN1VmREYKWkNqU1FVckFhZm5JajB0cFdDRHVuK1hEQ1pHRVVzV3BDeTdINWhJOFhqYjBRbEw3Vm5KRGk5alpBeWwwalJhVAp5OExrSTA2SXNwbVhlM2xJRzI5S0FRVmdGOWNsNjFDVTVWR25CM2ZHbUNzOUdtMStmTTNkc2hya055eHkvN1JNClBBUUhyVzhiWVo1c0NiQ09SUHBqc3NOandSVk16VjNtU2dhcmI0b1h1T3V2NTZoSnYxSnAxc2N1bS84SVlrQm4KdkE5RS9qa0E5RTVqd05uV0FNam9wRGtTZlVNN0NiVDdJMjZ5R001THdLdDc2NXY4bUJTVVBGcWt0amM5SVpoOApXZ2NKcGxuRHExOEhyTGtTQ3gvamRkR2I0WWphRDN1ajhyb0JMQ3NPRFR2K3NzWUo0Z21heWpHbkZRbFRLZlhpCnNLV2hwL0RvUXR6cWZtbHZyWDdlRnUrUwotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0tCg== - client-key-data: LS0tLS1CRUdJTiBSU0EgUFJJVkFURSBLRVktLS0tLQpNSUlKS0FJQkFBS0NBZ0VBdE1GUVhjQkdJYmduK1dleWlMOWliSDVnYm9DbDhCRnJvd1FaelluNGxKcHFBVW5HCjR4b0QwbUZSNklBbkdQd0szTVdqNXJmcFNQUEdLZ29kbHBMTThrNlJ5UE03UHBhOXNyTU1YYmUwdkxCYnFvWU0KeFc4aHFyTldBcEtYWitJUWI0TnkrY0VFbklMNlY4aHN4ME9kZVVhNkkrbGJveTVVdWM3QWwvWkVVK3o0dEZoSAptNjc1VHNhWU9kOXozSTJpZ0pmQS9EZ044Q3ZWVWFHeVNwVHA4ZzM5amxpMlM2WVpPTFFKVFJWcFlQQUtyTUNCCnlQclV1L3FhbFVMSXVSdE5SLyszdkpNS2pyZWQzQTd6MXYwdzg0bDhlOUlBYjRMM3BEMVMvUXNyc2V2L1pOc3gKdXljZVczNjlGeXFrNEZwWnZaVEwvb3A4OUdZa2ZaZmpXNmpianUxd0VqSVJhMndncEl0OU1KK1NEdzdiUW12bApNQVNhZmlwVnEyRDBXYmMveWhySFk5MGRwNEVIY1dOcDNkUlJJTlhnRTdyZ0RrVHMrd2t2U2VVQ3lyRllRMEFyCk1nZUl0NDNJR0srdVU5UW5HaWhNOG1Wa2tGVjM1SitQWTZTa1A1bklGTzc4Ylg2SThRVGw5eERpUmwzeEZMckYKcE53Sjk2bWFlVkE5ZDd6NjdoZnFvc3o2c25YZ0I4L085UkhjaGtRaFZla3pzajhPbnRZT28rdXoraWVqd1pEaQpVREV6N0dQeGI3NUw0aks4RERSalFtYys0dUZTOFh3ZFRrUEpLeVlzcWRMZUlvdERaOWRib2xRMXpaOVlLeGw1CjcrQ3l2aU50ZGFCL0FRVUZ0dnNnZXY2bzdmYkhzY1VoZTRvbllwNEU4dVZEdzIyUW5qR3B5SU1HNkljQ0F3RUEKQVFLQ0FnRUFoeHFMRUZmNzA0NUxTcTdJa2svQ3FHZFplcDdyNk9HTWUzOUFMaEkzWEJPQ3NrM0x1OG1oRzZtawpTRTBwYkJKUHFDNk4wSkNjYk92UXJYRjhKaVM1elU4T3hvMlBOYndFTkVLQ05yWnVBQzh0aE9ySE52ZFd5YmxiCnkzWjRkcXhSTHBpbzRxYno1R3c5NUVrcGpWTXJyQTlDYUQ4dlFHd0w1Z24waUdFSk4xWERtMEQwM1JmT3ZxUFAKK2tuYjF2bUNnZTFyME5teENWZ3FET2I0VmFpeXFEcnV6MnZkYmZQTG5FZHRRRkdGdjNBTCtvMjFORnpQajJYRQpEMWFvb3pzVjYzL21mS3ZIVTE5SlBjZHJXTUZNeW10MFFUUjVJT0JhYTBkdTlUWlpSWmFwQVdDWTdsTlI3SDl4CnBKUTZRbnNsNG44S2JzcGlsV2FYVmN5bW9zYzJ1Mzlzb3EyVUd3cjJoOHJ3MEdFNGdEY21wZ0RwY3FuRXlEVloKRUxoNG9CSmEvc1F0K0tiVk1zRVZUbUZkOXJyaDNEbGRuaHNPTVNvUldxWTZYZ01pNzQ1OVJPWWxPSEZ3YTdFTgpHUzFqMWF5dndDZmVUSGcrNHhFVUYybXcyQmd6UDNXOTZ0VmpOOFgxTmdFbHZLT1lCTTFEc1crYndSR2RldjErCm41R2N4WEZja3A4ZDY5QWtkQThsT2hOM2dzTVZ2Vk5rVmFBQWwzdzMrbkNIVHlNcStjeXJ6V1orSnZNNjZZdnIKZlQvSkQyZ3g0RW05ZEdkTjlxT0ZxTWtkMXN1dk9HOWJHbVB2N2JEb2hYV2dwL2FUMkk3WW1CYW1yemwvNGpPYwo0VTI3Z2JwTysxS2NjeDB6Zk9TYkJGZU91N2thYkJpc0JsdkExeThoQlBxVjRyUy9XZmtDZ2dFQkFOemFSODlXCnlsRlFsUjNhZXF6c3o2QlRLRW9ISndPelZvYXBRdzZZYjNiaTROczY1eXFoUllGWjlBV1BJdC9wQ090TEsyWnQKRG1oRTVVcUJreklQZ3EwK292OUtIUUk1aEVCV241U0dUWnNWSDdpMDdBUGhVeDVVWDROTEpMNWM5SklINlRZZAoyVC9CWGV3c1JFSiswOU5sN3NMTU53eWZlMUhGQU5ldUFXZ2hhNW1YVTNWOFBLeEY1N3NzcGI5YzlrZzVhc1dhClByRHRMRFRmd0hCQ0tlc3ZZMkVnUU9Tc0VNb0l1SWN0WnQvWDd3UzFjMldSbStzRis5WWJYV1lBaHNTQ0tpSGcKYlpJdG53cUg0YVJzWmRmTjlWWnpMTG8zdWpiK3N0RXA5RWtyNWJkeDRFZEUwSkZrblVTZzRYTHN2Q1RyektEMQpaV2dmdGQ0VFVXYmJMdE1DZ2dFQkFOR0ZiVm1GazNiYld1bFVKOXhZclpWNDBmZXZhUVFnQjg1Nm01VGxVN2JKCktKSms1U2J2TThCcEMrSkFHRk1vWXdlZENYQlppV3YxWHExV20vTURFaWl3V1lBZW1HZ0tsM1Bzam9aNEE3WXcKNkRicmRTcDY1WXJ0aFh6aW01Z3RrbkhoVDhmVGhSUUZFZmZ1MnFpY1lOMTFSZ0liVW1NU0xyc0c3aEtJN3o4dApUaEVqbmZQSXRob1R2WkN3R0ZjMi92eEhYTXl0ancra0pNTnlXL0pCQUgyUU1rdmJHSHdDckE0Y1ovTWtSc1l0CldEemkrUTZTcStWeDVyUDVRVzNUUFhDeGhNMWgrZ1FGcmsyVTYrcG9waHQvQ0NCVzVRT3duRVdkamtvcXoyRm0KTW9ZblpDUmpjMXdJWmJKYjNjT2thNG0yR3I2VWRHWkhpREhWU1Z6ZWx2MENnZ0VBTDB6QW5ITm1HVXc4NDR2cgpRaVhsd2R5TmdHMVB1Y2NxL09RN2JHRFJYcER3djRNNkJ6TkxBNDBVYWl4aHQ3cmRJaWVUUC9pVFlWbUFFaGoyClFYNTQ2a25EOXhFVWJXbU1IWXR2Mmd0NWk0RldyZVJ4dDlNR1RqU2IwQzdEeTlGMHYrSFozTEUvMmh1NzFkMFYKTVhpZ1BXSlRaSDBBUFQvYTFZUnZ6Y0dDbzU2a3dOeGNENGx3SlNWRG9CN0wwTlJFR3V2Ukpra0dyRFhmaGVOMQovNVRzRFdhMHpoUnVVWXRZZlN5bytLcU1vZHhKWUtUTDlqRGRSOVNjUHhyVlFnZFMxYnRhOHE1eVNTQmtNNGt1CmxseHc0MnRFNUNMK3JWTktHb1VxSHdueE84cWZCRTdyVGRnN0l5aDlWamx1ajRLTU9Rd1hZMUdyU2tXZ1dkT2gKVjNvUjZRS0NBUUIrT3VEU21vTnBqR2FOUk1KdklvNFNDNlNVa1AyNEdvUVpoZ3pOL0F3Zmg3b0YyUHVtSlY4TQpnVDFEcGxJQlo4ZWlBcDU1TGJoNWJtOElCN0lWQ3JpYUF3VXBCMFhUQ082T1RPcmhZTlQ3b3lPRVV2OVlCSXJlCjFKTWoxbkxma0RVMm53UXQ0K1c5VmdDcFd5SXBpMjFnZnhNYW1kTmRGTXRLcXJZc1BJdkhpRWdNVTJVS1J0WlAKaDYwaHlyWWZkV2JTSHp4dG8vTVkvVStZNFFFOFJxVkJBT3NNZ0llVHp4aFJ1Qk40alZhR2R6TStHNURwa3J3MQpsQlhtUURtSExVbEs1K2lXeEh5WWg4T253aDBRTU5SQ3BKb0V3ZFZnRDd0V3hjSEFOV0tUejdxSTBKcW93Ykh5CnRLeHdkS0I3Q2tndnc2TUJSdmVKMTNiYWJ5R3Y3ZnNSQW9JQkFITjFtZjNldmpQbTZmQ1IwZGZycGVEdnExUWQKbzFmOWFQVUd1dFFCZUplQXVkeXVHZlRRdkN2WlRVMUxVdytBUHJzSnRBUlVEcU5JYXY2ZFYrVUdUWnNxeFdqWQpwaS83KzFiOWx5N2VzSHBrMnY2ckx0MHJHSUgvQmhPWUZVeEpEWmRib0RpemxZc3BSaE1nSWtGb3hjVUwxSDVlCkx1MzJDaVNERUgwWU9RNkJGSi9NZGVtOHRQWmtuc3pXejc4b3hrMWdmb1VFNXQ3SGJzRmt2dDI2cWNDWjVYOWEKdHBHY040dXQ3emE1NXIxQUxEUzhlTDhjRHd2a3d0T3JxS1lVSTcvTHByNXNjWlZwaFFXbmp4NUptdUF3UFFrMwpkSXB2R2Nabk8yOFo1cWdrYi9CSnhrWGJLSkNVSTJGUVRCbEZnVzlVNXA5NWJHdUlCNk41T0hqbmplTT0KLS0tLS1FTkQgUlNBIFBSSVZBVEUgS0VZLS0tLS0K - token: c38fbc7d6fdc12cf07893b421e510078b2daad26d2e4233f2251b7743e50401398024d59bf573c912768d086766cd8f0f92baf4d43e2eb63a97f9868bfd1ad47 diff --git a/kubernetes/linux/dockerbuild/khushi-watch-test b/kubernetes/linux/dockerbuild/khushi-watch-test new file mode 100644 index 000000000..69729d1ec --- /dev/null +++ b/kubernetes/linux/dockerbuild/khushi-watch-test @@ -0,0 +1,20 @@ +apiVersion: v1 +clusters: +- cluster: + certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUU2RENDQXRDZ0F3SUJBZ0lRWDdub2hxOWthLzY4OHBzSHh0b1RHekFOQmdrcWhraUc5dzBCQVFzRkFEQU4KTVFzd0NRWURWUVFERXdKallUQWdGdzB5TVRBM01UWXlNekl4TVRoYUdBOHlNRFV4TURjeE5qSXpNekV4T0ZvdwpEVEVMTUFrR0ExVUVBeE1DWTJFd2dnSWlNQTBHQ1NxR1NJYjNEUUVCQVFVQUE0SUNEd0F3Z2dJS0FvSUNBUUMvCmR1aE1wZkxhWURuTHoyUjNrVW55M2hwbHREdGQ4cmlvakE1ZUZteXUveUJZcUl5Qi9LUDNMQWd4ajVwLzRtYlQKT2RqT0pseElKcW8vRlVGdU51b1E3YkRpSlVpZ0I3UVVQVVl0dVZCTUtFUHpuZk8yUFB1VXY3UVloZjNXMmpQegp1cUNGWkFRVnZUeDZmVTZLRjNhNkt1aC9pSkZLY2c1YnZGUVpnOWVNSy91QjlBdXZUSEx0aS9VRUJjWmM5UXBICm9hbklsUGo1N3dVV01zV25veWxRTndwSk1WczE4aDhaMStYS1pYakkvN0dVRTZhaEZwTVdmaU5GWDgybGNlbEQKQ0NwdzZSL3NiRyt1dXRuRURXQVpPRGFSOERpdFRBc3hyUHJMNHdDWXI5d05tY0tSQ3JwUXFsb3p6WFNGZGN5bApwbXZDMExYOVUwSGdZOURMMTZHaDUxbHRrSnN0dS9velhCd1BSM1JrdVVJZ3Q3WmpkOERSTmtlbVVvdGlyZHkvCm16VTJmb2hpSWE0a3VKR1JjVnR0RHZ0eVFSYTNPMzNHMzNQRVJTbkxCblpHSzRHMkVMbTBxTDh1WVFOZ1IveHoKeWlTdmozSlUrOVZoRFplaFlyOWNMWXFuSXRLci84anhVSFozU3RkNjhpSkFvQUJwZzFFWFRDc3pxamdxckJnagpLaG5YNHFCTlpQVmVUZjNrQjVkZ0VsZlJtMXIvcGsxblpFQmMrQllkcjBoSmNKTGgzSEYyR2hCM1BLc1NwUXE0CmVibVhQalZmanNxeFhSYVN2MTdzUTloSmhHRXN6WlBHU29SUjA1b3UwaU04ZHF3M0hjTnI4NzJuYW5uK2pXK24KTS9EaWxrSnJRY25WUVJLc1IxVkJ2ZnhKNjJzOFA2Ynl6V2FWL3NMNm1RSURBUUFCbzBJd1FEQU9CZ05WSFE4QgpBZjhFQkFNQ0FxUXdEd1lEVlIwVEFRSC9CQVV3QXdFQi96QWRCZ05WSFE0RUZnUVVCRG1ORE9VczRrVE9OLzlVClI3bkQ0MGNnR1VVd0RRWUpLb1pJaHZjTkFRRUxCUUFEZ2dJQkFEdGZ6M2pjV2lNeE1zaDIyT3I1RWV2R1pTRU0Kd0JDZmQ0QWMwUXlENkRmVkdaUFlwZmU3aDIzV1dKUWs5T05ncUszdFJhY1FUTTA0N2R3WUJGL3FTOEU2M3pKSQpINmZxL1lwQW9adHpZRkdaZTU1bUFaWWFQTlZWeDdvaHdoMzV2RTdFY0JaZ3JuS1pUN0EzOXlOWHRka2F2cThYCmhHQzdMV1lIMUxjQlhxckxHZlkvL1FpNnYraXdzZWlFdDkvUnRrSjhJK0VFd2JTWU9EazY5Z1lhSUtyK2RJaDgKbXdZM0d3VTZJUXl6MkFqb3pCaEcxbzQ1dzh1K1dKOTVyZEtrS0lEVG0ySTBNMEQ3dWhnZ0ZDT2srY25sOWNtSQozNkdqM2hIMlNFWC9Ea0tKN0orZ1o4bG1rc01HbGh3bGZ3K3FqRXVkVEQ4bCs3a1QrWWpMbEl3dTZRVkhvOGxWCkwyZ3diTG9tTzNJdU9IdHo4VUkxNVRJYSt4QmNsNDVSek1ZWWo0NHcvY2JiZStxUDE5THFQSUZXaXl1OG51UE4KUVhtcUtVR2wyOE83RmtteWFOckpuQURKNmZ5YU5nY21Bb1RrSU1oSk5IMHd2UkNjcTBGaXdJNkFYSGhEczJXYQpDNUkvS0c4NDg3dnlIRUJON3RhcldLN2RMMEhjcWt5cGl4cVkyNERRSnZLVUlIdUlHVk5sMGtza1Z4OWRKekxOCnkrd25SWndUTEVxRkNLSlMzU1F4eFl5Z0xLd09uZGVGTG1TSG1vbGJYM3IzYllDdkZLQ3BlNFA2V1Iwb0M1b20KOGZvMUpwLzI4SnF3K091RTJQZkFhRGFBaFNtK2tmb3V3WnIyWUNHSm50RjFTOEdlSFQxanFFRkJSWnp1UWd3eQp3R3cySnB0cjFEdmJvWmhaCi0tLS0tRU5EIENFUlRJRklDQVRFLS0tLS0K + server: https://khushi-watch-test-dns-9d093a4f.hcp.westus2.azmk8s.io:443 + name: khushi-watch-test +contexts: +- context: + cluster: khushi-watch-test + user: clusterUser_khushi-watch-test_khushi-watch-test + name: khushi-watch-test +current-context: khushi-watch-test +kind: Config +preferences: {} +users: +- name: clusterUser_khushi-watch-test_khushi-watch-test + user: + client-certificate-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUZIakNDQXdhZ0F3SUJBZ0lSQVArdktyZDVaRms2ajFycDVCS01adjh3RFFZSktvWklodmNOQVFFTEJRQXcKRFRFTE1Ba0dBMVVFQXhNQ1kyRXdIaGNOTWpFd056RTJNak15TVRFNFdoY05Nak13TnpFMk1qTXpNVEU0V2pBdwpNUmN3RlFZRFZRUUtFdzV6ZVhOMFpXMDZiV0Z6ZEdWeWN6RVZNQk1HQTFVRUF4TU1iV0Z6ZEdWeVkyeHBaVzUwCk1JSUNJakFOQmdrcWhraUc5dzBCQVFFRkFBT0NBZzhBTUlJQ0NnS0NBZ0VBcDhoV0piY3F4emFrejkrZjdRdG0KV0xvVzB3RFo0czlacmlDM25xZ2ZzaFBiYkMwSWlCdGxtQ05lYU1pRTVXeDBnby90VXQ5VUtvUlAwbzQ0Rlh2SApEZGVhU1J1OEllZnh2K1VUOXpkR0VwZDc4ejFBdlF3djhKYysyUjk5b2RnbWtYS0ZXeVhZb0UxRXpSMk9mbVVJCnVOVnNlNldoUHdCbFhNWXZmL0IwYU0rcHg0QzNZOGxnT2p4QTdybFRYYW1pNUlIemxZclQ0aVdtbHI1c05LczAKbkJkVWpicGtsdngxZk5DalR4S0tRaGFNRGdMc1dUUXBaVjI1b0FzYXhIcUx0Z0g3TytZaGZ2bnBJNUxLV3JNego1by9TQ3Vya0phK1RyWnVTbnJKWmgvR21NK09wczhXT1NJYVJpcEtLZlVtajNONzlUeEhkWTJVdThpNnZiTS9MCjhHMVZVYThRV3BnZ0wrbmtSWWRZQmdjSENKVjJZRUovSnlmSm9oK3c0RFF1aUV5RmZyNndmWlpKZGNnRlVRcTgKQllJZ29HNnZGQ3hMNEZwRW5ueWJHcFBLemczVUtmVmxCMGJ0Zkw0TjRZM3ZPYzhPeDgrWHZBWFMzSDdXV1I2TQpUeTFvMGxua1RRYW1uUVo1U2Y4TFU0OERHU2R5NFIzUTdiNW9UTU5TYmVHc0t0WXlxYURPQjE4WW0zTE9CQ0dvCnU4VE4vTFlsaFRHY0NSVklTOFE2SEoyYUZ0L1Z6b04zbXZlYitsUDF3LzNNaTltYjIyOTFsQ25hUG9pcE1memUKdFpoVUNOUURmUGpVRlFyS0dKODNhVzZkTWRRTVY0dXNzWG1Dbno5NXlWeEx6eEtRSVEzM0RKYWhVNlQ1VUtxRQpqY3ZqY09MWmFTdDE5Z21DMXh5QlF4OENBd0VBQWFOV01GUXdEZ1lEVlIwUEFRSC9CQVFEQWdXZ01CTUdBMVVkCkpRUU1NQW9HQ0NzR0FRVUZCd01DTUF3R0ExVWRFd0VCL3dRQ01BQXdId1lEVlIwakJCZ3dGb0FVQkRtTkRPVXMKNGtUT04vOVVSN25ENDBjZ0dVVXdEUVlKS29aSWh2Y05BUUVMQlFBRGdnSUJBRS9XNDdtU21PS0ZSYmJQV01aVQppTmROdmpzOVQ5QTVSc25WYk9uKzN5WFdGSk9pM0JucFBlOHNRc094MTlva0c4K2pIN2lpZWFmOW1SNEtUKzFJCjNMSy9uMWRNTVUzZXRTdE9zcmpob2ZYS3dNbVB3YklDYXAwaGs1T2w5Wk0xaFVIWEtGWGVMME9qQ2piVG5wa0gKYzdTd0JEd1NLcS83bDhYaWdYN29wclFxRTZUNFRLMWI5SDFQQ1JjZWw0UFJZZ0lwTXdHTWZOckFuK2F3dkJpVApZTmpudXljdEpHdXlXT2xGYVl6V281Skh3b2JSQUpJaG9sN1RFckM5Z28yUlpTL1dwSXNYbHhLREFaSnNmUGFkCmsrSmQzNlV6eU1QRnNvUHZqUWhlajlIVnl0Wk8rY1piYVlYbkd1eWx0U3Y2dVErUTd2cEtJVmoxTEN5cVJUL0kKWkxjM2ovMTNUNGFwLzExKytOaHRYNmtRM0JJRDdQS2VXd2YxeFF2NVRPTExIbG5VMTQ0WXI3b1NqK1QzWEdCSAp0bENqdHF5clFxMERZaitGaVhyb3JuUGJRMFFybG0zVE5lTVRLbktUSzdWMDl4a2FMRW8xalgrOHkxMzkwUzVMCk03bWRVTmlzS1A4T09EUTNMSVVETWdYRWEwWFFxdnJmMnJlVG9pVFhwZlJaK3JZU0RMVitsT3VoYzhlSFN6aVQKWXE0dDhJMlplMHdUUzdrTkhiMzlPYVVHYTc1Y2ZEdHZBcWxNWUk3UXhkU3hVYXNFd2Y0NytwUXdMZEl0UWFXcgp3cFZ2dk5nNnBIZEdVbG1VQVJ0dEhNMnR5YWhRNERXVmFoM0lubFlDYzRqVEtuNGc1aldxZkhBNWFsOEtNTWhQCmpNVWFVRGMzVkN5RGFacHRDeWRHQUgrLwotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0tCg== + client-key-data: LS0tLS1CRUdJTiBSU0EgUFJJVkFURSBLRVktLS0tLQpNSUlKS0FJQkFBS0NBZ0VBcDhoV0piY3F4emFrejkrZjdRdG1XTG9XMHdEWjRzOVpyaUMzbnFnZnNoUGJiQzBJCmlCdGxtQ05lYU1pRTVXeDBnby90VXQ5VUtvUlAwbzQ0Rlh2SERkZWFTUnU4SWVmeHYrVVQ5emRHRXBkNzh6MUEKdlF3djhKYysyUjk5b2RnbWtYS0ZXeVhZb0UxRXpSMk9mbVVJdU5Wc2U2V2hQd0JsWE1ZdmYvQjBhTStweDRDMwpZOGxnT2p4QTdybFRYYW1pNUlIemxZclQ0aVdtbHI1c05LczBuQmRVamJwa2x2eDFmTkNqVHhLS1FoYU1EZ0xzCldUUXBaVjI1b0FzYXhIcUx0Z0g3TytZaGZ2bnBJNUxLV3JNejVvL1NDdXJrSmErVHJadVNuckpaaC9HbU0rT3AKczhXT1NJYVJpcEtLZlVtajNONzlUeEhkWTJVdThpNnZiTS9MOEcxVlVhOFFXcGdnTCtua1JZZFlCZ2NIQ0pWMgpZRUovSnlmSm9oK3c0RFF1aUV5RmZyNndmWlpKZGNnRlVRcThCWUlnb0c2dkZDeEw0RnBFbm55YkdwUEt6ZzNVCktmVmxCMGJ0Zkw0TjRZM3ZPYzhPeDgrWHZBWFMzSDdXV1I2TVR5MW8wbG5rVFFhbW5RWjVTZjhMVTQ4REdTZHkKNFIzUTdiNW9UTU5TYmVHc0t0WXlxYURPQjE4WW0zTE9CQ0dvdThUTi9MWWxoVEdjQ1JWSVM4UTZISjJhRnQvVgp6b04zbXZlYitsUDF3LzNNaTltYjIyOTFsQ25hUG9pcE1memV0WmhVQ05RRGZQalVGUXJLR0o4M2FXNmRNZFFNClY0dXNzWG1Dbno5NXlWeEx6eEtRSVEzM0RKYWhVNlQ1VUtxRWpjdmpjT0xaYVN0MTlnbUMxeHlCUXg4Q0F3RUEKQVFLQ0FnQUhyaXdEWGZjZlYya0Qrd0NmSVQ1MklVNmFLaGZQUWg2ZzBlNlYzS3hXd29IdzJiN2lQQjdTY0F3SwpUK09GZlFsNFVJNVVsTlNOZmJFSnVtam0wdHV4em9USmcvT0F1ZFZmSzJWV2s3a3BjTFhEMUxINTlXemNYcEFKCjhGOFg0WVVpYzFPWGNJd1NDbmR6ekQ2Um1wNWpsNkYzcDRWU0ZQcU8zS09mLzZuVWdtMExMT0U0T0NlbmdzcVcKSXZXbCsvWHc3K0h1bm9SRWZlUzZVYzB5UEFRVWdSemx2L3FLenRPeCt2cit5Nko0ay8rbFJJejlLRjRjdmNXWgpoZWlieGVCUFhKZmJqaFZLY3JZeFlxN1FxQk5nSW9WQzZxMjI2K3FlcjlodVcwdXQ1V251UmlXbWpReU9WVnFrCk5VRVJxeVZOSHdnSmJvL3IxTjNwU0NuUG1WT0Vnd0t2Qk1iN24zZFE2ME1TZHIxVkNKOVl5ZkI3TGlVODVOdHYKY01CWWt3Y1FocWxPR1o0WlZJa2VjaWxvZnBmY2lJVWlzWmJpMTlsUnB4SHBoRzUvR0N3K3p4VVJ2U0xud0Rqcgo2Zy81ZCthWjQ0bFlSSzIwS09OQ003VnA1dHRnZHJwcmtnK211cldBZ0tIcThJQTB6KzhzZlVYWDNXUFoyelVTCnppb2NGcmlmdXhENm1aMEFHZEhRaFdqK1dMSEpYdnJtaENtYmFsUEk5NllVVWRxZlo0RHdkL1ZZcXlLOEI2TjkKM0RFVnI4SlhKU1piUFV1MEZDcS80amV2R29JekJPVHBrTVU1RGtSUlA4ODgwY1ZLMGoxNmwxYXhiajNYb3hVVQpuWmQ1UDhSMWx6dTNONzdYUHNydXBPRDloUmJXRDFiQnVKN0E4aER0bWlQZ3hwTUxRUUtDQVFFQXkrWmZHMWJqClpnNUZqcWhOclR2dDVENmlmWmorUkkrL0YyYlpLdjdsdVRRRkgwemkvazhMSk44VFQ2b3ZYQTFHM29vQ1ZNZnAKNUZEYmphL3l1azJKejFwcCtiZ1pYeHdwRUdpSm0vRjZRVGQvM2Q0RzhGL1hnQnZIb3FHSXZCTzRFRlBCcTFlTgppRGNCMEUzdnZTWEM5bTFsWGgvSEVHczYySVg2bG5Nd2tYbUcrRTZMT0NTRTNQcTFUMDQxaWdUOTNYTmIzQnZxCm1YWWNBeGlzZVJMUGY3aExpNVVaNTJ1ekJLUDJ1OHg0R2lEOVlPQm5oc1g2dnBIUVExOWozUjlLMm52U3ppRjMKMWlTWVVLbWhIU0xwY3RHY0ZzYTdRYWJvVklpdGYzOWEzRHl4dHZXNndPOEx1TWVGRnMzdytqR3JrbTVIZ1BjZgpMY1A0T203T2xOa2JmUUtDQVFFQTBxZHZqUkYrMWxlRHZJUFhuSy9wTk5kTUNza21DTko1b25MTllKTWxNMVJTCk4velBZbjg3bXRsbnBzb3QrR0ZXS0pEaHkwTHlhUk4zZERacHJvRWxzOFVnNUt5NFZiZTV3OFloKzR5K0FrMFQKM3lwSTVDdUllNFpaTndpbEF5RkVWTkNhTXJaU3Y1eGpaR0pZQU1aa05mRE5XU3NSbFowUEkycFhYRDFCVnA5UQo4djVDWG5UTVZrTSs4NVZVN3dKMHl3QU1IRDdqVGM1RGVnaGZJT3NlZzVIc3RBTTJSbXc3dHU5YVVlL0t5eGVDCjZPY2NOMGhtT3VuZ2lPakE5MXYxWm5nV0lxaUZMZFVUUG9YS0JsWlRwbSt5QUJCTC92M0dWYm5NcUdzcnFlV3kKaS9ZQ1ZoSlVXcGZPa2t2djVUTTJXU0hKTGZYejU3Q25oL1Jodi80RHl3S0NBUUVBbGVlYWlJa0t5ZDg0T0RkNApWQ0k1TlpMdTk1UGhiQnFhak9QcVNYZjVBSDVFUGN6VEhkQ1RDdHFPWWdWbXFEQ0NwOTJpOVIyODBVUzVCYVFUCnVmQ0RudFNFRVRuT1BXU0F0RFdHNWdWVXNsblJRaGFYMTJVL0ZFcFlMVExCU2pUZEgvUFQ4TnAvaldPVGk4ZWcKdDlqcFN5OEdWWHJiYVRETXBKOXJxZXlxQ21ua1Z1MjM0T0RJWllaVWdpZW5xUDhlZlE3d3ZCUXlGLzBEZnl0RwpzZ1NvVk9memNuMTkrK3ZzUXo4Z2lOVThmMGs5djFsOEExUE9rd2kwcXJPWXdkcmR0MTlOam9xQWhnbUZpZGdNClFWV0hlQWl1enZmd2Q4WDBEU0luSEJOUGc1ZUR1RlZVcGIrWlNKSTNRSnJMemNWeXRFY3JmcDh3WnY4cW9oc1EKK2RIY0tRS0NBUUIydGF5RFRzeXZkWG5qamxpL0Q5TFgyRXdkOStUYksrRW56cEkyVGpXMGkxd1orUG5WZytYUApDWjBEdlFQUzZPZG96Tnl4Y3ZTd2lpdlM3YWI0bEFidkc3UEJxaVBuQ2paQlFUSjlVMzd3UlFkaHg2NTBCcVJQCkdCTEdsTFNJNHdKaTJYdE1BTHI1QitScStaQ21QeWJSenZXcHZqK1dsSCtuY1pIeGhRT0JFUjdKRU1mTit2djcKME1GMCt6NTF3bWlXelZ3RnZ0clJTZDkwc2FzS0hmV0FKNGZBbWQ3SUtNNHQ5UXh0Q0RpNzRtLy9WOTNxdVg0Qwp6VEdmZGFyb3VvVzhUUWdNVi9OVk5MSTNsYVdYbEtabk1LS2FycFJsQ2hYdTBWbW52MTFIelEybEdlbVBINTB4CndFTEU1czQxMTNwVkdGa0s4WU9WbVZPUHBnUS84YUdwQW9JQkFFa256SzNMeElSRmowVHB4TlJaSEp1NVNVdHkKTGRGaDQwd3kyUjNJWS9SSGpBbmx1TTBmUUQydjN1ZTY1VHhaenY4Y2tiZ1pKTkE0eGxTd2M0a3U5M3pkSkxYZwpLZzZPTU52cWZBbW9nMzkvekV3cEZ2MEZsdXpYTWdEdjVaSk5EYm0yMjhMaFJaQ1E3TFdKNWZMZXREY1BVaFRVCmd0UW9WVGZ6Zm5jTVMwa3IzQzdFSnQxam91UXMvNFhrVlEvZTkweTVkNTkwRGV5YmRSNnlDMWt5bU9sWVZJWDgKK0drdkxuZnFpMnZRZzdVR1hNTEY1LzRRNjZYRWl3NzVxcEdDTUFTdnNSa216L2FWYkpFNzNvR2dnZ1BGc3ZOOQpVcEFuR1NqaUFpd2Z5RmJ5d0NRV2hnUG1SbDhXV2xjTStZM0QxL1lLTElTVU5xQURnMGZXVXdkYVhFMD0KLS0tLS1FTkQgUlNBIFBSSVZBVEUgS0VZLS0tLS0K + token: 71b6ec454624539f8cf4a07c38d4c0efb015da0d5a9720766ab1e1d7c68c5920906350d2f4745a395a55abc5620c5f0002124c2f870358b61670078f137b5aa9 diff --git a/kubernetes/omsagent.yaml b/kubernetes/omsagent.yaml index 80f121a5d..c1a300a99 100644 --- a/kubernetes/omsagent.yaml +++ b/kubernetes/omsagent.yaml @@ -368,7 +368,7 @@ spec: value: "3" containers: - name: omsagent - image: "mcr.microsoft.com/azuremonitor/containerinsights/cidev:khushi-test-5" + image: "mcr.microsoft.com/azuremonitor/containerinsights/cidev:khushi-test-6" imagePullPolicy: Always resources: limits: @@ -380,7 +380,7 @@ spec: env: # azure devops pipeline uses AKS_RESOURCE_ID and AKS_REGION hence ensure to uncomment these - name: AKS_RESOURCE_ID - value: "/subscriptions/b4b0cbab-64dc-42bc-9d95-ea7c8d0cf188/resourcegroups/khushi-test/providers/Microsoft.ContainerService/managedClusters/khushi-test" + value: "/subscriptions/b4b0cbab-64dc-42bc-9d95-ea7c8d0cf188/resourcegroups/khushi-watch-test/providers/Microsoft.ContainerService/managedClusters/khushi-watch-test" - name: AKS_REGION value: "West US 2" # this used for e2e test and setting this just emits some additional log statements which used for the e2e tests @@ -446,7 +446,7 @@ spec: timeoutSeconds: 15 #Only in sidecar scraping mode - name: omsagent-prometheus - image: "mcr.microsoft.com/azuremonitor/containerinsights/cidev:khushi-test-5" + image: "mcr.microsoft.com/azuremonitor/containerinsights/cidev:khushi-test-6" imagePullPolicy: Always resources: limits: @@ -458,7 +458,7 @@ spec: env: # azure devops pipeline uses AKS_RESOURCE_ID and AKS_REGION hence ensure to uncomment these - name: AKS_RESOURCE_ID - value: "/subscriptions/b4b0cbab-64dc-42bc-9d95-ea7c8d0cf188/resourcegroups/khushi-test/providers/Microsoft.ContainerService/managedClusters/khushi-test" + value: "/subscriptions/b4b0cbab-64dc-42bc-9d95-ea7c8d0cf188/resourcegroups/khushi-watch-test/providers/Microsoft.ContainerService/managedClusters/khushi-watch-test" - name: AKS_REGION value: "West US 2" #Uncomment below two lines for ACS clusters and set the cluster names manually. Also comment out the above two lines for ACS clusters @@ -589,7 +589,7 @@ spec: serviceAccountName: omsagent containers: - name: omsagent - image: "mcr.microsoft.com/azuremonitor/containerinsights/cidev:khushi-test-5" + image: "mcr.microsoft.com/azuremonitor/containerinsights/cidev:khushi-test-6" imagePullPolicy: Always resources: limits: @@ -600,7 +600,7 @@ spec: memory: 250Mi env: - name: AKS_RESOURCE_ID - value: "/subscriptions/b4b0cbab-64dc-42bc-9d95-ea7c8d0cf188/resourcegroups/khushi-test/providers/Microsoft.ContainerService/managedClusters/khushi-test" + value: "/subscriptions/b4b0cbab-64dc-42bc-9d95-ea7c8d0cf188/resourcegroups/khushi-watch-test/providers/Microsoft.ContainerService/managedClusters/khushi-watch-test" - name: AKS_REGION value: "West US 2" # this used for e2e test and setting this just emits some additional log statements which used for the e2e tests @@ -769,7 +769,7 @@ spec: env: # azure devops pipeline uses AKS_RESOURCE_ID and AKS_REGION hence ensure to uncomment these - name: AKS_RESOURCE_ID - value: "/subscriptions/b4b0cbab-64dc-42bc-9d95-ea7c8d0cf188/resourcegroups/khushi-test/providers/Microsoft.ContainerService/managedClusters/khushi-test" + value: "/subscriptions/b4b0cbab-64dc-42bc-9d95-ea7c8d0cf188/resourcegroups/khushi-watch-test/providers/Microsoft.ContainerService/managedClusters/khushi-watch-test" - name: AKS_REGION value: "West US 2" #- name: ACS_RESOURCE_NAME From d653e2b366b1f0f70980d7a62bd8318eafc04ddc Mon Sep 17 00:00:00 2001 From: Khushi Chaudhari Date: Tue, 20 Jul 2021 13:39:21 -0700 Subject: [PATCH 12/76] e2e added - merging included --- khushi-test | 20 ---- kubernetes/omsagent.yaml | 6 +- source/plugins/ruby/in_kube_podinventory.rb | 104 ++++++++++++++++---- 3 files changed, 87 insertions(+), 43 deletions(-) delete mode 100644 khushi-test diff --git a/khushi-test b/khushi-test deleted file mode 100644 index 4a07d0fb9..000000000 --- a/khushi-test +++ /dev/null @@ -1,20 +0,0 @@ -apiVersion: v1 -clusters: -- cluster: - certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUU2VENDQXRHZ0F3SUJBZ0lSQUozU3djVWpyaUg5blhaWnZHVHFmWWN3RFFZSktvWklodmNOQVFFTEJRQXcKRFRFTE1Ba0dBMVVFQXhNQ1kyRXdJQmNOTWpFd05qSTBNVFl5TVRBMVdoZ1BNakExTVRBMk1qUXhOak14TURWYQpNQTB4Q3pBSkJnTlZCQU1UQW1OaE1JSUNJakFOQmdrcWhraUc5dzBCQVFFRkFBT0NBZzhBTUlJQ0NnS0NBZ0VBCnUvZFhmRnZvYi9zREdVa0VoVWNYcDVGVndBR2VyR01hOXJocVhPM3JSbFJQYlltTW9oOVUrQWV6Q0U4ckJ5WGwKbCtib2JNKzQzU2x1Y3BickkrOVpZcGEvYnYrY2dYRXRGOENzdDMwbm9LYjlGeno2cVFicHNuQ09hNllhdVRhVQovWVU2aVhCQm4vRnhYZ1BNRGFucTRFcVVnNXNVZkhuVFJCNFdhZmhBaXI2T2g4RjBoZzhyRS83dXJ2cVFhNllPCklNN2kzUHY5dG1GN1ZjRThocUFYZTdNUE4xd0R2Q21SUGZoZlZTejAxdDFlQlFtcDJ4YitHQ2lBRStYUHVVZ3EKZ3RkU21OL2tnYVdRMFFIWm9sRWdEbE1Hb1BjSXNkeXZvd2x4WThEMmlkNlluMS9jeEtvNHBmUjkrUVdieUZrRQp5b0VDM0M0MGVBZVJTZUtLdjlxSnJyNlp2WGRxZGs1WHpRREU2d1hqakorUUhGZHJyeHFqdE01Z0dkbThiRDdiCmg0WjA5c201djBQd1BQZWE5SG5WbURmbjlsQmtjZGRSTDJuV1NGSVZsbWdDMVdrV3huZXBmSGV3YmN4MnpnK1UKazc2UjMxamMyWW9NRnZpNmtXaTFLejYwZmh5N0VIVWxsRTRlbm9VZm5TM2dDZ2pxeFVJcWlKZFpzbGluSTdNbgpFS0sxZ21DRk96bmtMU3MzQ0FXTGFKNEN6VFJsQkZKWDYzTE5NR2U0NXBTRUJIVEFlTUdtbjY5Tjc2Y2lrejNiCmthYTJXZ05sTEFZVDBEZUtQR0RJNmFsU2pOckFaS3lkNS9rTzBYTGo4MFVtT2JZY1NpeXlwYlhZZ1Q5bWxna3IKVEhPZ29HKzlEaDQ3bWdWczJlaURVYWJ6RVpTUlBUbEorTk9iSkJRczQ1RUNBd0VBQWFOQ01FQXdEZ1lEVlIwUApBUUgvQkFRREFnS2tNQThHQTFVZEV3RUIvd1FGTUFNQkFmOHdIUVlEVlIwT0JCWUVGRWEzTFFUSHJFeHBZQ0VJCkJZNEEzd3k4S2ZqTU1BMEdDU3FHU0liM0RRRUJDd1VBQTRJQ0FRQlp2b0IwdVpZa3dYclBvTnp3LytmMEc1bFUKMENWYU56STNkQlNyd29lT1VMTnBJdDFad0E3REhxNTZNa0pUTzNlNnNQbzZKczFIbHAzRTRJaFJxZTI0NnNyQgpaam51czdTNDY1aHBFcDBhd1lHMzdnY05NMlBxbFo3dDZNdDhSQjVneHJPb3JNS1BSSWZiYnd6NDNvWHB6VHVpCkZ0QUk5Z2xjR3V1VlZ5Yk42a2N2Y2MyNE9zb0VGQ3VHU0dBMzhaUTZqY1NIRDUvQW9rcGd0cTZ4M0trNHR1TjIKS3dWcjdtekdvZGZrOTlKSVhkTWREVy9qTGt5RlRRQWhoNlY2ZThmZHNJbmtEZFdKOUI2UlFqSzFiMjRiQmowbwpjaGZkald3L2pLRzQ4YTFIS2tlWHk2MG5oU1FKaGFaRjBvTEprNElpU3lqNUtQMlpHUGg1WTZLSUlRT0oreE5HCmJiN3BTQ2h6dFFVUjR4dEZkc2pxZk9BZzdCZitYU3ZTNXExUU5FS3JaOHE2M2VMQVQ5aFUxVDVDVE5QVW1Zd2YKbjNQVEJHdVUrNThpWFM3VGVxVGVwYlZxOFlvU2hCeVc1cXdWa1loU1VWNzk5VGNVb2oreTJ5SDAxd3F2a0JlUQovZExXZ0srVE4xZnpsMVBXTk5CWW03SFgwbWpuZVRrWFB2QWFKMzNsS0RzdWpsR0RwYmkza09tbGN1TVJqYStUCm5iSjNRRkd5aDR2UWt6N3hMWEtDdFFBOHVUeXVNVWdTVzJZZ2ZZYWlNWG1wMG95RHNQRnI3aFFKMXN2THd0dksKd2M3Z1ZMV3ZIWTlYOVhqN0dSbFFwdlBmV1B4MkxESThFeEU1eTVQSXFqYjhTMFZ1aVU4REpZTlVEaDlodGhoVgpqRmxuVTd6ekw5dEE4RDZ6RWc9PQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0tCg== - server: https://khushi-test-dns-92573251.hcp.westus2.azmk8s.io:443 - name: khushi-test -contexts: -- context: - cluster: khushi-test - user: clusterUser_khushi-test_khushi-test - name: khushi-test -current-context: khushi-test -kind: Config -preferences: {} -users: -- name: clusterUser_khushi-test_khushi-test - user: - client-certificate-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUZIakNDQXdhZ0F3SUJBZ0lSQVBHQzNPazBaT2pxcXppS01CT3M3bEF3RFFZSktvWklodmNOQVFFTEJRQXcKRFRFTE1Ba0dBMVVFQXhNQ1kyRXdIaGNOTWpFd05qSTBNVFl5TVRBMVdoY05Nak13TmpJME1UWXpNVEExV2pBdwpNUmN3RlFZRFZRUUtFdzV6ZVhOMFpXMDZiV0Z6ZEdWeWN6RVZNQk1HQTFVRUF4TU1iV0Z6ZEdWeVkyeHBaVzUwCk1JSUNJakFOQmdrcWhraUc5dzBCQVFFRkFBT0NBZzhBTUlJQ0NnS0NBZ0VBdE1GUVhjQkdJYmduK1dleWlMOWkKYkg1Z2JvQ2w4QkZyb3dRWnpZbjRsSnBxQVVuRzR4b0QwbUZSNklBbkdQd0szTVdqNXJmcFNQUEdLZ29kbHBMTQo4azZSeVBNN1BwYTlzck1NWGJlMHZMQmJxb1lNeFc4aHFyTldBcEtYWitJUWI0TnkrY0VFbklMNlY4aHN4ME9kCmVVYTZJK2xib3k1VXVjN0FsL1pFVSt6NHRGaEhtNjc1VHNhWU9kOXozSTJpZ0pmQS9EZ044Q3ZWVWFHeVNwVHAKOGczOWpsaTJTNllaT0xRSlRSVnBZUEFLck1DQnlQclV1L3FhbFVMSXVSdE5SLyszdkpNS2pyZWQzQTd6MXYwdwo4NGw4ZTlJQWI0TDNwRDFTL1FzcnNldi9aTnN4dXljZVczNjlGeXFrNEZwWnZaVEwvb3A4OUdZa2ZaZmpXNmpiCmp1MXdFaklSYTJ3Z3BJdDlNSitTRHc3YlFtdmxNQVNhZmlwVnEyRDBXYmMveWhySFk5MGRwNEVIY1dOcDNkUlIKSU5YZ0U3cmdEa1RzK3drdlNlVUN5ckZZUTBBck1nZUl0NDNJR0srdVU5UW5HaWhNOG1Wa2tGVjM1SitQWTZTawpQNW5JRk83OGJYNkk4UVRsOXhEaVJsM3hGTHJGcE53Sjk2bWFlVkE5ZDd6NjdoZnFvc3o2c25YZ0I4L085UkhjCmhrUWhWZWt6c2o4T250WU9vK3V6K2llandaRGlVREV6N0dQeGI3NUw0aks4RERSalFtYys0dUZTOFh3ZFRrUEoKS3lZc3FkTGVJb3REWjlkYm9sUTF6WjlZS3hsNTcrQ3l2aU50ZGFCL0FRVUZ0dnNnZXY2bzdmYkhzY1VoZTRvbgpZcDRFOHVWRHcyMlFuakdweUlNRzZJY0NBd0VBQWFOV01GUXdEZ1lEVlIwUEFRSC9CQVFEQWdXZ01CTUdBMVVkCkpRUU1NQW9HQ0NzR0FRVUZCd01DTUF3R0ExVWRFd0VCL3dRQ01BQXdId1lEVlIwakJCZ3dGb0FVUnJjdEJNZXMKVEdsZ0lRZ0ZqZ0RmREx3cCtNd3dEUVlKS29aSWh2Y05BUUVMQlFBRGdnSUJBQTRubkxXUERBaUlka0QxV3NiegoxTnVlSnhhVzRzd2VEdWtyNDdxVEFweHA1dFVyK1N1OGVCQkg5WVorTW5XUHBFUEtkclQ5Q0xRNElyTzJXbEJXCmNmV1lZTGxOU0hFSjVOL1pBY0k2Q2Y2MklnOUZlbHdzYk5SNjhOWGozL21OZEpqa3dzVitVL0FwVXM4RTVXU2sKMUUzaU4xOWxEUnh6cFNXZ0R5dUJmQzhvWWRSZzMzZ0wyd3FjcmRQc3MvdlZnc013S2tBd2NwVHJ1ei84ZjZTSgpPdTFtWmlDRGFVUzdQdm9IQ2YvMEU0eDJzZmdqTE9scjE1eTc4K0dKVVdOcXBpVXQ2aWxHNVVKWVZqd1JCS0U4CkxJTGw4dW0xeTUxU2pmUzNkSnQ4b0l4WENSWW9uTStNRDA0ZEYzVHduSFRhVWljS05Va2x5b1VMQWhVN1VmREYKWkNqU1FVckFhZm5JajB0cFdDRHVuK1hEQ1pHRVVzV3BDeTdINWhJOFhqYjBRbEw3Vm5KRGk5alpBeWwwalJhVAp5OExrSTA2SXNwbVhlM2xJRzI5S0FRVmdGOWNsNjFDVTVWR25CM2ZHbUNzOUdtMStmTTNkc2hya055eHkvN1JNClBBUUhyVzhiWVo1c0NiQ09SUHBqc3NOandSVk16VjNtU2dhcmI0b1h1T3V2NTZoSnYxSnAxc2N1bS84SVlrQm4KdkE5RS9qa0E5RTVqd05uV0FNam9wRGtTZlVNN0NiVDdJMjZ5R001THdLdDc2NXY4bUJTVVBGcWt0amM5SVpoOApXZ2NKcGxuRHExOEhyTGtTQ3gvamRkR2I0WWphRDN1ajhyb0JMQ3NPRFR2K3NzWUo0Z21heWpHbkZRbFRLZlhpCnNLV2hwL0RvUXR6cWZtbHZyWDdlRnUrUwotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0tCg== - client-key-data: LS0tLS1CRUdJTiBSU0EgUFJJVkFURSBLRVktLS0tLQpNSUlKS0FJQkFBS0NBZ0VBdE1GUVhjQkdJYmduK1dleWlMOWliSDVnYm9DbDhCRnJvd1FaelluNGxKcHFBVW5HCjR4b0QwbUZSNklBbkdQd0szTVdqNXJmcFNQUEdLZ29kbHBMTThrNlJ5UE03UHBhOXNyTU1YYmUwdkxCYnFvWU0KeFc4aHFyTldBcEtYWitJUWI0TnkrY0VFbklMNlY4aHN4ME9kZVVhNkkrbGJveTVVdWM3QWwvWkVVK3o0dEZoSAptNjc1VHNhWU9kOXozSTJpZ0pmQS9EZ044Q3ZWVWFHeVNwVHA4ZzM5amxpMlM2WVpPTFFKVFJWcFlQQUtyTUNCCnlQclV1L3FhbFVMSXVSdE5SLyszdkpNS2pyZWQzQTd6MXYwdzg0bDhlOUlBYjRMM3BEMVMvUXNyc2V2L1pOc3gKdXljZVczNjlGeXFrNEZwWnZaVEwvb3A4OUdZa2ZaZmpXNmpianUxd0VqSVJhMndncEl0OU1KK1NEdzdiUW12bApNQVNhZmlwVnEyRDBXYmMveWhySFk5MGRwNEVIY1dOcDNkUlJJTlhnRTdyZ0RrVHMrd2t2U2VVQ3lyRllRMEFyCk1nZUl0NDNJR0srdVU5UW5HaWhNOG1Wa2tGVjM1SitQWTZTa1A1bklGTzc4Ylg2SThRVGw5eERpUmwzeEZMckYKcE53Sjk2bWFlVkE5ZDd6NjdoZnFvc3o2c25YZ0I4L085UkhjaGtRaFZla3pzajhPbnRZT28rdXoraWVqd1pEaQpVREV6N0dQeGI3NUw0aks4RERSalFtYys0dUZTOFh3ZFRrUEpLeVlzcWRMZUlvdERaOWRib2xRMXpaOVlLeGw1CjcrQ3l2aU50ZGFCL0FRVUZ0dnNnZXY2bzdmYkhzY1VoZTRvbllwNEU4dVZEdzIyUW5qR3B5SU1HNkljQ0F3RUEKQVFLQ0FnRUFoeHFMRUZmNzA0NUxTcTdJa2svQ3FHZFplcDdyNk9HTWUzOUFMaEkzWEJPQ3NrM0x1OG1oRzZtawpTRTBwYkJKUHFDNk4wSkNjYk92UXJYRjhKaVM1elU4T3hvMlBOYndFTkVLQ05yWnVBQzh0aE9ySE52ZFd5YmxiCnkzWjRkcXhSTHBpbzRxYno1R3c5NUVrcGpWTXJyQTlDYUQ4dlFHd0w1Z24waUdFSk4xWERtMEQwM1JmT3ZxUFAKK2tuYjF2bUNnZTFyME5teENWZ3FET2I0VmFpeXFEcnV6MnZkYmZQTG5FZHRRRkdGdjNBTCtvMjFORnpQajJYRQpEMWFvb3pzVjYzL21mS3ZIVTE5SlBjZHJXTUZNeW10MFFUUjVJT0JhYTBkdTlUWlpSWmFwQVdDWTdsTlI3SDl4CnBKUTZRbnNsNG44S2JzcGlsV2FYVmN5bW9zYzJ1Mzlzb3EyVUd3cjJoOHJ3MEdFNGdEY21wZ0RwY3FuRXlEVloKRUxoNG9CSmEvc1F0K0tiVk1zRVZUbUZkOXJyaDNEbGRuaHNPTVNvUldxWTZYZ01pNzQ1OVJPWWxPSEZ3YTdFTgpHUzFqMWF5dndDZmVUSGcrNHhFVUYybXcyQmd6UDNXOTZ0VmpOOFgxTmdFbHZLT1lCTTFEc1crYndSR2RldjErCm41R2N4WEZja3A4ZDY5QWtkQThsT2hOM2dzTVZ2Vk5rVmFBQWwzdzMrbkNIVHlNcStjeXJ6V1orSnZNNjZZdnIKZlQvSkQyZ3g0RW05ZEdkTjlxT0ZxTWtkMXN1dk9HOWJHbVB2N2JEb2hYV2dwL2FUMkk3WW1CYW1yemwvNGpPYwo0VTI3Z2JwTysxS2NjeDB6Zk9TYkJGZU91N2thYkJpc0JsdkExeThoQlBxVjRyUy9XZmtDZ2dFQkFOemFSODlXCnlsRlFsUjNhZXF6c3o2QlRLRW9ISndPelZvYXBRdzZZYjNiaTROczY1eXFoUllGWjlBV1BJdC9wQ090TEsyWnQKRG1oRTVVcUJreklQZ3EwK292OUtIUUk1aEVCV241U0dUWnNWSDdpMDdBUGhVeDVVWDROTEpMNWM5SklINlRZZAoyVC9CWGV3c1JFSiswOU5sN3NMTU53eWZlMUhGQU5ldUFXZ2hhNW1YVTNWOFBLeEY1N3NzcGI5YzlrZzVhc1dhClByRHRMRFRmd0hCQ0tlc3ZZMkVnUU9Tc0VNb0l1SWN0WnQvWDd3UzFjMldSbStzRis5WWJYV1lBaHNTQ0tpSGcKYlpJdG53cUg0YVJzWmRmTjlWWnpMTG8zdWpiK3N0RXA5RWtyNWJkeDRFZEUwSkZrblVTZzRYTHN2Q1RyektEMQpaV2dmdGQ0VFVXYmJMdE1DZ2dFQkFOR0ZiVm1GazNiYld1bFVKOXhZclpWNDBmZXZhUVFnQjg1Nm01VGxVN2JKCktKSms1U2J2TThCcEMrSkFHRk1vWXdlZENYQlppV3YxWHExV20vTURFaWl3V1lBZW1HZ0tsM1Bzam9aNEE3WXcKNkRicmRTcDY1WXJ0aFh6aW01Z3RrbkhoVDhmVGhSUUZFZmZ1MnFpY1lOMTFSZ0liVW1NU0xyc0c3aEtJN3o4dApUaEVqbmZQSXRob1R2WkN3R0ZjMi92eEhYTXl0ancra0pNTnlXL0pCQUgyUU1rdmJHSHdDckE0Y1ovTWtSc1l0CldEemkrUTZTcStWeDVyUDVRVzNUUFhDeGhNMWgrZ1FGcmsyVTYrcG9waHQvQ0NCVzVRT3duRVdkamtvcXoyRm0KTW9ZblpDUmpjMXdJWmJKYjNjT2thNG0yR3I2VWRHWkhpREhWU1Z6ZWx2MENnZ0VBTDB6QW5ITm1HVXc4NDR2cgpRaVhsd2R5TmdHMVB1Y2NxL09RN2JHRFJYcER3djRNNkJ6TkxBNDBVYWl4aHQ3cmRJaWVUUC9pVFlWbUFFaGoyClFYNTQ2a25EOXhFVWJXbU1IWXR2Mmd0NWk0RldyZVJ4dDlNR1RqU2IwQzdEeTlGMHYrSFozTEUvMmh1NzFkMFYKTVhpZ1BXSlRaSDBBUFQvYTFZUnZ6Y0dDbzU2a3dOeGNENGx3SlNWRG9CN0wwTlJFR3V2Ukpra0dyRFhmaGVOMQovNVRzRFdhMHpoUnVVWXRZZlN5bytLcU1vZHhKWUtUTDlqRGRSOVNjUHhyVlFnZFMxYnRhOHE1eVNTQmtNNGt1CmxseHc0MnRFNUNMK3JWTktHb1VxSHdueE84cWZCRTdyVGRnN0l5aDlWamx1ajRLTU9Rd1hZMUdyU2tXZ1dkT2gKVjNvUjZRS0NBUUIrT3VEU21vTnBqR2FOUk1KdklvNFNDNlNVa1AyNEdvUVpoZ3pOL0F3Zmg3b0YyUHVtSlY4TQpnVDFEcGxJQlo4ZWlBcDU1TGJoNWJtOElCN0lWQ3JpYUF3VXBCMFhUQ082T1RPcmhZTlQ3b3lPRVV2OVlCSXJlCjFKTWoxbkxma0RVMm53UXQ0K1c5VmdDcFd5SXBpMjFnZnhNYW1kTmRGTXRLcXJZc1BJdkhpRWdNVTJVS1J0WlAKaDYwaHlyWWZkV2JTSHp4dG8vTVkvVStZNFFFOFJxVkJBT3NNZ0llVHp4aFJ1Qk40alZhR2R6TStHNURwa3J3MQpsQlhtUURtSExVbEs1K2lXeEh5WWg4T253aDBRTU5SQ3BKb0V3ZFZnRDd0V3hjSEFOV0tUejdxSTBKcW93Ykh5CnRLeHdkS0I3Q2tndnc2TUJSdmVKMTNiYWJ5R3Y3ZnNSQW9JQkFITjFtZjNldmpQbTZmQ1IwZGZycGVEdnExUWQKbzFmOWFQVUd1dFFCZUplQXVkeXVHZlRRdkN2WlRVMUxVdytBUHJzSnRBUlVEcU5JYXY2ZFYrVUdUWnNxeFdqWQpwaS83KzFiOWx5N2VzSHBrMnY2ckx0MHJHSUgvQmhPWUZVeEpEWmRib0RpemxZc3BSaE1nSWtGb3hjVUwxSDVlCkx1MzJDaVNERUgwWU9RNkJGSi9NZGVtOHRQWmtuc3pXejc4b3hrMWdmb1VFNXQ3SGJzRmt2dDI2cWNDWjVYOWEKdHBHY040dXQ3emE1NXIxQUxEUzhlTDhjRHd2a3d0T3JxS1lVSTcvTHByNXNjWlZwaFFXbmp4NUptdUF3UFFrMwpkSXB2R2Nabk8yOFo1cWdrYi9CSnhrWGJLSkNVSTJGUVRCbEZnVzlVNXA5NWJHdUlCNk41T0hqbmplTT0KLS0tLS1FTkQgUlNBIFBSSVZBVEUgS0VZLS0tLS0K - token: c38fbc7d6fdc12cf07893b421e510078b2daad26d2e4233f2251b7743e50401398024d59bf573c912768d086766cd8f0f92baf4d43e2eb63a97f9868bfd1ad47 diff --git a/kubernetes/omsagent.yaml b/kubernetes/omsagent.yaml index c1a300a99..7b87ad7ea 100644 --- a/kubernetes/omsagent.yaml +++ b/kubernetes/omsagent.yaml @@ -368,7 +368,7 @@ spec: value: "3" containers: - name: omsagent - image: "mcr.microsoft.com/azuremonitor/containerinsights/cidev:khushi-test-6" + image: "mcr.microsoft.com/azuremonitor/containerinsights/cidev:khushi-test-8" imagePullPolicy: Always resources: limits: @@ -446,7 +446,7 @@ spec: timeoutSeconds: 15 #Only in sidecar scraping mode - name: omsagent-prometheus - image: "mcr.microsoft.com/azuremonitor/containerinsights/cidev:khushi-test-6" + image: "mcr.microsoft.com/azuremonitor/containerinsights/cidev:khushi-test-8" imagePullPolicy: Always resources: limits: @@ -589,7 +589,7 @@ spec: serviceAccountName: omsagent containers: - name: omsagent - image: "mcr.microsoft.com/azuremonitor/containerinsights/cidev:khushi-test-6" + image: "mcr.microsoft.com/azuremonitor/containerinsights/cidev:khushi-test-8" imagePullPolicy: Always resources: limits: diff --git a/source/plugins/ruby/in_kube_podinventory.rb b/source/plugins/ruby/in_kube_podinventory.rb index 6ea5e3807..0041fd17c 100644 --- a/source/plugins/ruby/in_kube_podinventory.rb +++ b/source/plugins/ruby/in_kube_podinventory.rb @@ -128,6 +128,10 @@ def initial_write $log.info("in_kube_podinventory::watch - initial_write : Getting pods from Kube API @ #{Time.now.utc.iso8601}") continuationToken, podInventory = KubernetesApiClient.getResourcesAndContinuationToken("pods?limit=#{@PODS_CHUNK_SIZE}") + if (continuationToken.nil? || continuationToken.empty?) + $log.info("in_kube_podinventory::watch - initial_write : continuation token is null or empty rip") + end + @collection_version = podInventory["metadata"]["resourceVersion"] $log.info("in_kube_podinventory::watch - initial_write : received collection version: #{@collection_version}") @@ -171,26 +175,23 @@ def watch $log.info("in_kube_podinventory::watch : finished getting pods, about to begin infinite loop for watch") loop do - $log.info("in_kube_pod_inventory::watch: inside infinite loop for watch pods") + $log.info("in_kube_pod_inventory::watch: inside infinite loop for watch pods. collection version: #{@collection_version}") begin @KubernetesWatchClient.watch_pods(resource_version: @collection_version, as: :parsed) do |notice| - $log.info("in_kube_podinventory::watch: received a notice, inside watch pods! Time: #{Time.now.utc.iso8601}") - # if !notice["object"].nil? && !notice["object"].empty? - # $log.info("in_kube_podinventory::watch: received a notice of type #{notice["type"]}") - - # item = notice["object"] - # record = {"name" => item["metadata"]["name"], "uid" => item["metadata"]["uid"], "status" => item["status"]["phase"], "type" => notice["type"]} - # $log.info("in_kube_podinventory::watch: successfully created a record with notice") - - # # @mutex.synchronize { - # $log.info("in_kube_podinventory::watch : about to add item to noticeHash. Time: #{Time.now.utc.iso8601}") - # @noticeHash[item["metadata"]["uid"]] = record - # $log.info("in_kube_podinventory::watch : successfully added item to noticeHash. Time: #{Time.now.utc.iso8601}") - # # } - # else - # $log.info("in_kube_pod_inventory::watch: notice object was either null or empty") - # end + puts "in_kube_podinventory::watch : inside watch pods! Time: #{Time.now.utc.iso8601}" + if !notice.nil? && !notice.empty? + puts "in_kube_podinventory::watch : received a notice that is not null and not empty, type: #{notice["type"]}" + + item = notice["object"] + record = {"name" => item["metadata"]["name"], "uid" => item["metadata"]["uid"], "status" => item["status"]["phase"], "type" => notice["type"]} + + @mutex.synchronize { + @noticeHash[item["metadata"]["uid"]] = record + } + + puts "watch pods:: number of items in noticeHash = #{@noticeHash.size}" end + end rescue => exception $log.warn("in_kube_podinventory::watch : watch events session got broken and re-establishing the session.") # $log.debug_backtrace(exception.backtrace) @@ -481,6 +482,68 @@ def parse_and_emit_records(podInventory, serviceRecords, continuationToken, batc end #begin block end end + def merge_info + begin + fileContents = File.read("testing-podinventory.json") + puts "in_kube_podinventory::merge_info : file contents read" + @podHash = JSON.parse(fileContents) + puts "in_kube_podinventory::merge_info : parse successful" + rescue => error + puts "in_kube_podinventory::merge_info : something went wrong with reading file" + puts "in_kube_podinventory::merge_info : backtrace: #{error.backtrace}" + end + + puts "in_kube_podinventory::merge_info : before noticeHash loop, number of items in hash: #{@noticeHash.size()}, noticeHash: #{@noticeHash}" + + uidList = [] + + @mutex.synchronize { + @noticeHash.each do |uid, record| + puts "in_kube_podinventory::merge_info : looping through noticeHash, type of notice: #{record["type"]}" + # puts "podHash looks like: #{@podHash}" + puts "notice uid: #{uid}" + puts "notice record: #{record}" + + uidList.append(uid) + + case record["type"] + when "ADDED" + @podHash[uid] = record + puts "added" + when "MODIFIED" + puts "entered modified case as expected" + if @podHash[uid].nil? + puts "modify case where uid for add was overwritten to modify" + @podHash[uid] = record + else + puts "modify case where it is a legit modify" + val = @podHash[uid] + val["status"] = record["status"] + @podHash[uid] = val + end + puts "modified" + when "DELETED" + @podHash.delete(uid) + puts "deleted" + else + puts "something went wrong" + end + # puts "uid: #{uid} and record: #{record}" + puts "end of switch" + end + + # remove all looked at uids from the noticeHash + uidList.each do |uid| + @noticeHash.delete(uid) + end + } + + # replace entire contents of testing-podinventory.json + File.open("testing-podinventory.json", "w") do |f| + f.write JSON.pretty_generate(@podHash) + end + end + def run_periodic @mutex.lock done = @finished @@ -500,9 +563,10 @@ def run_periodic @mutex.unlock if !done begin - $log.info("in_kube_podinventory::run_periodic.enumerate.start #{Time.now.utc.iso8601}") - enumerate - $log.info("in_kube_podinventory::run_periodic.enumerate.end #{Time.now.utc.iso8601}") + $log.info("in_kube_podinventory::run_periodic.merge_info.start #{Time.now.utc.iso8601}") + # enumerate + merge_info + $log.info("in_kube_podinventory::run_periodic.merge_info.end #{Time.now.utc.iso8601}") rescue => errorStr $log.warn "in_kube_podinventory::run_periodic: enumerate Failed to retrieve pod inventory: #{errorStr}" ApplicationInsightsUtility.sendExceptionTelemetry(errorStr) From 6448767e4aa10b339a1339d161d03e293d50d9ef Mon Sep 17 00:00:00 2001 From: Khushi Chaudhari Date: Tue, 20 Jul 2021 15:19:25 -0700 Subject: [PATCH 13/76] fixed log statements - e2e should work! --- source/plugins/ruby/in_kube_podinventory.rb | 81 +++++++++++---------- 1 file changed, 42 insertions(+), 39 deletions(-) diff --git a/source/plugins/ruby/in_kube_podinventory.rb b/source/plugins/ruby/in_kube_podinventory.rb index 0041fd17c..2ee20fbe2 100644 --- a/source/plugins/ruby/in_kube_podinventory.rb +++ b/source/plugins/ruby/in_kube_podinventory.rb @@ -178,9 +178,9 @@ def watch $log.info("in_kube_pod_inventory::watch: inside infinite loop for watch pods. collection version: #{@collection_version}") begin @KubernetesWatchClient.watch_pods(resource_version: @collection_version, as: :parsed) do |notice| - puts "in_kube_podinventory::watch : inside watch pods! Time: #{Time.now.utc.iso8601}" + $log.info("in_kube_podinventory::watch : inside watch pods! Time: #{Time.now.utc.iso8601}") if !notice.nil? && !notice.empty? - puts "in_kube_podinventory::watch : received a notice that is not null and not empty, type: #{notice["type"]}" + $log.info("in_kube_podinventory::watch : received a notice that is not null and not empty, type: #{notice["type"]}") item = notice["object"] record = {"name" => item["metadata"]["name"], "uid" => item["metadata"]["uid"], "status" => item["status"]["phase"], "type" => notice["type"]} @@ -189,7 +189,7 @@ def watch @noticeHash[item["metadata"]["uid"]] = record } - puts "watch pods:: number of items in noticeHash = #{@noticeHash.size}" + $log.info("watch pods:: number of items in noticeHash = #{@noticeHash.size}") end end rescue => exception @@ -483,65 +483,68 @@ def parse_and_emit_records(podInventory, serviceRecords, continuationToken, batc end def merge_info + $log.info("merge_info:: enters this function, about to begin read file") begin fileContents = File.read("testing-podinventory.json") - puts "in_kube_podinventory::merge_info : file contents read" + $log.info("in_kube_podinventory::merge_info : file contents read") @podHash = JSON.parse(fileContents) - puts "in_kube_podinventory::merge_info : parse successful" + $log.info("in_kube_podinventory::merge_info : parse successful") rescue => error - puts "in_kube_podinventory::merge_info : something went wrong with reading file" - puts "in_kube_podinventory::merge_info : backtrace: #{error.backtrace}" + $log.info("in_kube_podinventory::merge_info : something went wrong with reading file") + $log.info("in_kube_podinventory::merge_info : backtrace: #{error.backtrace}") end - puts "in_kube_podinventory::merge_info : before noticeHash loop, number of items in hash: #{@noticeHash.size()}, noticeHash: #{@noticeHash}" + $log.info("in_kube_podinventory::merge_info : before noticeHash loop, number of items in hash: #{@noticeHash.size()}, noticeHash: #{@noticeHash}") - uidList = [] + uidList = [] - @mutex.synchronize { + @mutex.synchronize { @noticeHash.each do |uid, record| - puts "in_kube_podinventory::merge_info : looping through noticeHash, type of notice: #{record["type"]}" - # puts "podHash looks like: #{@podHash}" - puts "notice uid: #{uid}" - puts "notice record: #{record}" - - uidList.append(uid) - - case record["type"] - when "ADDED" + $log.info("in_kube_podinventory::merge_info : looping through noticeHash, type of notice: #{record["type"]}") + # $log.info("podHash looks like: #{@podHash}") + $log.info("merge_info :: notice uid: #{uid}") + $log.info("merge_info :: notice record: #{record}") + + uidList.append(uid) + + case record["type"] + when "ADDED" + @podHash[uid] = record + $log.info("merge_info :: added") + when "MODIFIED" + $log.info("merge_info :: entered modified case as expected") + if @podHash[uid].nil? + $log.info("merge_info :: modify case where uid for add was overwritten to modify" ) @podHash[uid] = record - puts "added" - when "MODIFIED" - puts "entered modified case as expected" - if @podHash[uid].nil? - puts "modify case where uid for add was overwritten to modify" - @podHash[uid] = record - else - puts "modify case where it is a legit modify" - val = @podHash[uid] - val["status"] = record["status"] - @podHash[uid] = val - end - puts "modified" - when "DELETED" - @podHash.delete(uid) - puts "deleted" else - puts "something went wrong" + $log.info("merge_info :: modify case where it is a legit modify") + val = @podHash[uid] + val["status"] = record["status"] + @podHash[uid] = val end - # puts "uid: #{uid} and record: #{record}" - puts "end of switch" + $log.info("merge_info :: modified") + when "DELETED" + @podHash.delete(uid) + $log.info("merge_info :: deleted") + else + $log.info("merge_info :: something went wrong") + end + # $log.info("uid: #{uid} and record: #{record}") + $log.info("merge_info :: end of switch") end # remove all looked at uids from the noticeHash uidList.each do |uid| - @noticeHash.delete(uid) + @noticeHash.delete(uid) end } + $log.info("in_kube_podinventory:: merge_info : about to replace entire contents of testing-podinventory.json") # replace entire contents of testing-podinventory.json File.open("testing-podinventory.json", "w") do |f| f.write JSON.pretty_generate(@podHash) end + $log.info("in_kube_podinventory:: merge_info : finished replacing contents of testing-podinventory.json") end def run_periodic From 2af17b9ba9d725f39e8e787d6f005e1ff12a1d4a Mon Sep 17 00:00:00 2001 From: Khushi Chaudhari Date: Tue, 20 Jul 2021 15:47:59 -0700 Subject: [PATCH 14/76] updated yaml --- kubernetes/omsagent.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/kubernetes/omsagent.yaml b/kubernetes/omsagent.yaml index 7b87ad7ea..f8a68fd8c 100644 --- a/kubernetes/omsagent.yaml +++ b/kubernetes/omsagent.yaml @@ -368,7 +368,7 @@ spec: value: "3" containers: - name: omsagent - image: "mcr.microsoft.com/azuremonitor/containerinsights/cidev:khushi-test-8" + image: "mcr.microsoft.com/azuremonitor/containerinsights/cidev:khushi-test-9" imagePullPolicy: Always resources: limits: @@ -446,7 +446,7 @@ spec: timeoutSeconds: 15 #Only in sidecar scraping mode - name: omsagent-prometheus - image: "mcr.microsoft.com/azuremonitor/containerinsights/cidev:khushi-test-8" + image: "mcr.microsoft.com/azuremonitor/containerinsights/cidev:khushi-test-9" imagePullPolicy: Always resources: limits: @@ -589,7 +589,7 @@ spec: serviceAccountName: omsagent containers: - name: omsagent - image: "mcr.microsoft.com/azuremonitor/containerinsights/cidev:khushi-test-8" + image: "mcr.microsoft.com/azuremonitor/containerinsights/cidev:khushi-test-9" imagePullPolicy: Always resources: limits: From 0415b843c274592d2cdf10df9c08e534ede444ff Mon Sep 17 00:00:00 2001 From: Khushi Chaudhari Date: Wed, 21 Jul 2021 09:54:55 -0700 Subject: [PATCH 15/76] updated file write --- source/plugins/ruby/in_kube_podinventory.rb | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/source/plugins/ruby/in_kube_podinventory.rb b/source/plugins/ruby/in_kube_podinventory.rb index 2ee20fbe2..550ad2cac 100644 --- a/source/plugins/ruby/in_kube_podinventory.rb +++ b/source/plugins/ruby/in_kube_podinventory.rb @@ -113,7 +113,10 @@ def shutdown def write_to_file(podInventory) $log.info("in_kube_podinventory:: write_to_file : inside write to file function") - File.write("testing-podinventory.json", JSON.pretty_generate(podInventory)) + File.open("testing-podinventory.json", "w") { |file| + file.write(JSON.pretty_generate(podInventory)) + } + # File.write("testing-podinventory.json", JSON.pretty_generate(podInventory)) $log.info("in_kube_podinventory:: write_to_file : successfully done writing to file") end @@ -487,6 +490,7 @@ def merge_info begin fileContents = File.read("testing-podinventory.json") $log.info("in_kube_podinventory::merge_info : file contents read") + $log.info("in_kube_podinventory::merge_info : fileContents: #{fileContents}") @podHash = JSON.parse(fileContents) $log.info("in_kube_podinventory::merge_info : parse successful") rescue => error @@ -541,9 +545,12 @@ def merge_info $log.info("in_kube_podinventory:: merge_info : about to replace entire contents of testing-podinventory.json") # replace entire contents of testing-podinventory.json - File.open("testing-podinventory.json", "w") do |f| - f.write JSON.pretty_generate(@podHash) - end + File.open("testing-podinventory.json", "w") { |file| + file.write(JSON.pretty_generate(podInventory)) + } + # File.open("testing-podinventory.json", "w") do |f| + # f.write JSON.pretty_generate(@podHash) + # end $log.info("in_kube_podinventory:: merge_info : finished replacing contents of testing-podinventory.json") end From 2973262f867acd5ce9ee292bbb4cd9d0935d0552 Mon Sep 17 00:00:00 2001 From: Khushi Chaudhari Date: Wed, 21 Jul 2021 09:57:39 -0700 Subject: [PATCH 16/76] updated yaml test image 10 --- kubernetes/omsagent.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/kubernetes/omsagent.yaml b/kubernetes/omsagent.yaml index f8a68fd8c..5f190e660 100644 --- a/kubernetes/omsagent.yaml +++ b/kubernetes/omsagent.yaml @@ -368,7 +368,7 @@ spec: value: "3" containers: - name: omsagent - image: "mcr.microsoft.com/azuremonitor/containerinsights/cidev:khushi-test-9" + image: "mcr.microsoft.com/azuremonitor/containerinsights/cidev:khushi-test-10" imagePullPolicy: Always resources: limits: @@ -446,7 +446,7 @@ spec: timeoutSeconds: 15 #Only in sidecar scraping mode - name: omsagent-prometheus - image: "mcr.microsoft.com/azuremonitor/containerinsights/cidev:khushi-test-9" + image: "mcr.microsoft.com/azuremonitor/containerinsights/cidev:khushi-test-10" imagePullPolicy: Always resources: limits: @@ -589,7 +589,7 @@ spec: serviceAccountName: omsagent containers: - name: omsagent - image: "mcr.microsoft.com/azuremonitor/containerinsights/cidev:khushi-test-9" + image: "mcr.microsoft.com/azuremonitor/containerinsights/cidev:khushi-test-10" imagePullPolicy: Always resources: limits: From 18d3ca36a4a3bc981e89c83d10b0985d3ee390ff Mon Sep 17 00:00:00 2001 From: Khushi Chaudhari Date: Wed, 21 Jul 2021 12:20:28 -0700 Subject: [PATCH 17/76] impl now uses getPodInventoryRecords to write to file --- kubernetes/omsagent.yaml | 6 ++-- source/plugins/ruby/in_kube_podinventory.rb | 40 ++++++++++++--------- 2 files changed, 27 insertions(+), 19 deletions(-) diff --git a/kubernetes/omsagent.yaml b/kubernetes/omsagent.yaml index 5f190e660..d1f6cca56 100644 --- a/kubernetes/omsagent.yaml +++ b/kubernetes/omsagent.yaml @@ -368,7 +368,7 @@ spec: value: "3" containers: - name: omsagent - image: "mcr.microsoft.com/azuremonitor/containerinsights/cidev:khushi-test-10" + image: "mcr.microsoft.com/azuremonitor/containerinsights/cidev:khushi-test-11" imagePullPolicy: Always resources: limits: @@ -446,7 +446,7 @@ spec: timeoutSeconds: 15 #Only in sidecar scraping mode - name: omsagent-prometheus - image: "mcr.microsoft.com/azuremonitor/containerinsights/cidev:khushi-test-10" + image: "mcr.microsoft.com/azuremonitor/containerinsights/cidev:khushi-test-11" imagePullPolicy: Always resources: limits: @@ -589,7 +589,7 @@ spec: serviceAccountName: omsagent containers: - name: omsagent - image: "mcr.microsoft.com/azuremonitor/containerinsights/cidev:khushi-test-10" + image: "mcr.microsoft.com/azuremonitor/containerinsights/cidev:khushi-test-11" imagePullPolicy: Always resources: limits: diff --git a/source/plugins/ruby/in_kube_podinventory.rb b/source/plugins/ruby/in_kube_podinventory.rb index 550ad2cac..e76d55630 100644 --- a/source/plugins/ruby/in_kube_podinventory.rb +++ b/source/plugins/ruby/in_kube_podinventory.rb @@ -113,9 +113,13 @@ def shutdown def write_to_file(podInventory) $log.info("in_kube_podinventory:: write_to_file : inside write to file function") - File.open("testing-podinventory.json", "w") { |file| - file.write(JSON.pretty_generate(podInventory)) - } + batchTime = Time.utc.iso8601 + podInventory["items"].each do |item| + podInventoryRecords = getPodInventoryRecords(item, @serviceRecords, batchTime) + File.open("testing-podinventory.json", "w") { |file| + file.write(JSON.pretty_generate(podInventoryRecords)) + } + end # File.write("testing-podinventory.json", JSON.pretty_generate(podInventory)) $log.info("in_kube_podinventory:: write_to_file : successfully done writing to file") end @@ -172,7 +176,7 @@ def initial_write def watch $log.info("in_kube_podinventory::watch : entered watch function - about to call initial write") - initial_write + enumerate # $log.info("finished initial write to pod inventory file") $log.info("in_kube_podinventory::watch : finished getting pods, about to begin infinite loop for watch") @@ -215,7 +219,7 @@ def enumerate(podList = nil) @controllerData = {} currentTime = Time.now batchTime = currentTime.utc.iso8601 - serviceRecords = [] + @serviceRecords = [] @podInventoryE2EProcessingLatencyMs = 0 podInventoryStartTime = (Time.now.to_f * 1000).to_i @@ -231,9 +235,9 @@ def enumerate(podList = nil) $log.info("in_kube_podinventory::enumerate:End:Parsing services data using yajl @ #{Time.now.utc.iso8601}") serviceInfo = nil # service inventory records much smaller and fixed size compared to serviceList - serviceRecords = KubernetesApiClient.getKubeServicesInventoryRecords(serviceList, batchTime) + @serviceRecords = KubernetesApiClient.getKubeServicesInventoryRecords(serviceList, batchTime) # updating for telemetry - @serviceCount += serviceRecords.length + @serviceCount += @serviceRecords.length serviceList = nil end @@ -245,26 +249,29 @@ def enumerate(podList = nil) $log.info("in_kube_podinventory::enumerate : Getting pods from Kube API @ #{Time.now.utc.iso8601}") continuationToken, podInventory = KubernetesApiClient.getResourcesAndContinuationToken("pods?limit=#{@PODS_CHUNK_SIZE}") @collection_version = podInventory["metadata"]["resourceVersion"] - @log.info("in_kube_podinventory::enumerate : received collection version: #{@collection_version}") + $log.info("in_kube_podinventory::enumerate : received collection version: #{@collection_version}") $log.info("in_kube_podinventory::enumerate : Done getting pods from Kube API @ #{Time.now.utc.iso8601}") podsAPIChunkEndTime = (Time.now.to_f * 1000).to_i @podsAPIE2ELatencyMs = (podsAPIChunkEndTime - podsAPIChunkStartTime) if (!podInventory.nil? && !podInventory.empty? && podInventory.key?("items") && !podInventory["items"].nil? && !podInventory["items"].empty?) $log.info("in_kube_podinventory::enumerate : number of pod items :#{podInventory["items"].length} from Kube API @ #{Time.now.utc.iso8601}") - parse_and_emit_records(podInventory, serviceRecords, continuationToken, batchTime) + write_to_file(podInventory) + # parse_and_emit_records(podInventory, @serviceRecords, continuationToken, batchTime) else $log.warn "in_kube_podinventory::enumerate:Received empty podInventory" end #If we receive a continuation token, make calls, process and flush data until we have processed all data while (!continuationToken.nil? && !continuationToken.empty?) + $log.info("in_kube_podinventory::enumerate : continuation token is not null and not empty") podsAPIChunkStartTime = (Time.now.to_f * 1000).to_i continuationToken, podInventory = KubernetesApiClient.getResourcesAndContinuationToken("pods?limit=#{@PODS_CHUNK_SIZE}&continue=#{continuationToken}") podsAPIChunkEndTime = (Time.now.to_f * 1000).to_i @podsAPIE2ELatencyMs = @podsAPIE2ELatencyMs + (podsAPIChunkEndTime - podsAPIChunkStartTime) if (!podInventory.nil? && !podInventory.empty? && podInventory.key?("items") && !podInventory["items"].nil? && !podInventory["items"].empty?) $log.info("in_kube_podinventory::enumerate : number of pod items :#{podInventory["items"].length} from Kube API @ #{Time.now.utc.iso8601}") - parse_and_emit_records(podInventory, serviceRecords, continuationToken, batchTime) + write_to_file(podInventory) + # parse_and_emit_records(podInventory, @serviceRecords, continuationToken, batchTime) else $log.warn "in_kube_podinventory::enumerate:Received empty podInventory" end @@ -273,7 +280,7 @@ def enumerate(podList = nil) @podInventoryE2EProcessingLatencyMs = ((Time.now.to_f * 1000).to_i - podInventoryStartTime) # Setting these to nil so that we dont hold memory until GC kicks in podInventory = nil - serviceRecords = nil + @serviceRecords = nil # Adding telemetry to send pod telemetry every 5 minutes timeDifference = (DateTime.now.to_time.to_i - @@podTelemetryTimeTracker).abs @@ -489,8 +496,8 @@ def merge_info $log.info("merge_info:: enters this function, about to begin read file") begin fileContents = File.read("testing-podinventory.json") - $log.info("in_kube_podinventory::merge_info : file contents read") - $log.info("in_kube_podinventory::merge_info : fileContents: #{fileContents}") + # $log.info("in_kube_podinventory::merge_info : file contents read") + $log.info("in_kube_podinventory::merge_info : file contents read, fileContents: #{fileContents}") @podHash = JSON.parse(fileContents) $log.info("in_kube_podinventory::merge_info : parse successful") rescue => error @@ -544,10 +551,11 @@ def merge_info } $log.info("in_kube_podinventory:: merge_info : about to replace entire contents of testing-podinventory.json") + write_to_file(@podHash) # replace entire contents of testing-podinventory.json - File.open("testing-podinventory.json", "w") { |file| - file.write(JSON.pretty_generate(podInventory)) - } + # File.open("testing-podinventory.json", "w") { |file| + # file.write(JSON.pretty_generate(@podHash)) + # } # File.open("testing-podinventory.json", "w") do |f| # f.write JSON.pretty_generate(@podHash) # end From 3a3d67146dcb21f7b176902b430241c3ab1573a6 Mon Sep 17 00:00:00 2001 From: Khushi Chaudhari Date: Wed, 21 Jul 2021 13:33:09 -0700 Subject: [PATCH 18/76] fixed log statements, attempting to debug why merge_info write to file doesn't work --- source/plugins/ruby/in_kube_podinventory.rb | 155 +++++++++++--------- 1 file changed, 83 insertions(+), 72 deletions(-) diff --git a/source/plugins/ruby/in_kube_podinventory.rb b/source/plugins/ruby/in_kube_podinventory.rb index e76d55630..b8a51c1ae 100644 --- a/source/plugins/ruby/in_kube_podinventory.rb +++ b/source/plugins/ruby/in_kube_podinventory.rb @@ -91,9 +91,9 @@ def start @finished = false @condition = ConditionVariable.new @mutex = Mutex.new - $log.info("in_kube:podinventory::start: about to start thread for watch") + $log.info("in_kube:podinventory::start: create thread for watch") @watchthread = Thread.new(&method(:watch)) - $log.info("in_kube:podinventory::start: about to start thread for run_periodic") + $log.info("in_kube:podinventory::start: create thread for run_periodic") @thread = Thread.new(&method(:run_periodic)) @@podTelemetryTimeTracker = DateTime.now.to_time.to_i end @@ -112,73 +112,77 @@ def shutdown end def write_to_file(podInventory) - $log.info("in_kube_podinventory:: write_to_file : inside write to file function") - batchTime = Time.utc.iso8601 - podInventory["items"].each do |item| - podInventoryRecords = getPodInventoryRecords(item, @serviceRecords, batchTime) - File.open("testing-podinventory.json", "w") { |file| - file.write(JSON.pretty_generate(podInventoryRecords)) - } + $log.info("in_kube_podinventory::write_to_file : enters write_to_file function") + batchTime = Time.now.utc.iso8601 + #TODO: check if you can pass @serviceRecords into getPodInventoryRecords rather than creating a local copy + servRecords= @serviceRecords + + begin + podInventory["items"].each do |item| + podInventoryRecords = getPodInventoryRecords(item, servRecords, batchTime) + File.open("testing-podinventory.json", "w") { |file| + file.write(JSON.pretty_generate(podInventoryRecords)) + } + end + $log.info("in_kube_podinventory:: write_to_file : successfully finished writing to file") + rescue => exception + $log.info("in_kube_podinventory::write_to_file : writing to file failed. backtrace: #{exception.backtrace}") end - # File.write("testing-podinventory.json", JSON.pretty_generate(podInventory)) - $log.info("in_kube_podinventory:: write_to_file : successfully done writing to file") end - def initial_write - $log.info("in_kube_podinventory::watch - initial_write: entered initial write function") + # def initial_write + # $log.info("in_kube_podinventory::watch - initial_write: entered initial write function") - @podsAPIE2ELatencyMs = 0 - podsAPIChunkStartTime = (Time.now.to_f * 1000).to_i + # @podsAPIE2ELatencyMs = 0 + # podsAPIChunkStartTime = (Time.now.to_f * 1000).to_i - # Initializing continuation token to nil - continuationToken = nil - $log.info("in_kube_podinventory::watch - initial_write : Getting pods from Kube API @ #{Time.now.utc.iso8601}") + # # Initializing continuation token to nil + # continuationToken = nil + # $log.info("in_kube_podinventory::watch - initial_write : Getting pods from Kube API @ #{Time.now.utc.iso8601}") - continuationToken, podInventory = KubernetesApiClient.getResourcesAndContinuationToken("pods?limit=#{@PODS_CHUNK_SIZE}") - if (continuationToken.nil? || continuationToken.empty?) - $log.info("in_kube_podinventory::watch - initial_write : continuation token is null or empty rip") - end + # continuationToken, podInventory = KubernetesApiClient.getResourcesAndContinuationToken("pods?limit=#{@PODS_CHUNK_SIZE}") + # if (continuationToken.nil? || continuationToken.empty?) + # $log.info("in_kube_podinventory::watch - initial_write : continuation token is null or empty rip") + # end - @collection_version = podInventory["metadata"]["resourceVersion"] + # @collection_version = podInventory["metadata"]["resourceVersion"] - $log.info("in_kube_podinventory::watch - initial_write : received collection version: #{@collection_version}") - $log.info("in_kube_podinventory::watch : Done getting pods from Kube API @ #{Time.now.utc.iso8601}") + # $log.info("in_kube_podinventory::watch - initial_write : received collection version: #{@collection_version}") + # $log.info("in_kube_podinventory::watch : Done getting pods from Kube API @ #{Time.now.utc.iso8601}") - podsAPIChunkEndTime = (Time.now.to_f * 1000).to_i - @podsAPIE2ELatencyMs = (podsAPIChunkEndTime - podsAPIChunkStartTime) + # podsAPIChunkEndTime = (Time.now.to_f * 1000).to_i + # @podsAPIE2ELatencyMs = (podsAPIChunkEndTime - podsAPIChunkStartTime) - if (!podInventory.nil? && !podInventory.empty? && podInventory.key?("items") && !podInventory["items"].nil? && !podInventory["items"].empty?) - $log.info("in_kube_podinventory::watch - initial_write : number of pod items :#{podInventory["items"].length} from Kube API @ #{Time.now.utc.iso8601}") - $log.info("in_kube_podinventory::watch - initial_write : time to write to a file and emit to backend - functionality later") - write_to_file(podInventory) - # parse_and_emit_records(podInventory, serviceRecords, continuationToken, batchTime) - else - $log.warn "in_kube_podinventory::watch - initial_write : Received empty podInventory" - end - - #If we receive a continuation token, make calls, process and flush data until we have processed all data - while (!continuationToken.nil? && !continuationToken.empty?) - $log.info("in_kube_podinventory::watch - initial_write : continuation token was not null or empty so round two of gettings pod inventory") - podsAPIChunkStartTime = (Time.now.to_f * 1000).to_i - continuationToken, podInventory = KubernetesApiClient.getResourcesAndContinuationToken("pods?limit=#{@PODS_CHUNK_SIZE}&continue=#{continuationToken}") - podsAPIChunkEndTime = (Time.now.to_f * 1000).to_i - @podsAPIE2ELatencyMs = @podsAPIE2ELatencyMs + (podsAPIChunkEndTime - podsAPIChunkStartTime) - if (!podInventory.nil? && !podInventory.empty? && podInventory.key?("items") && !podInventory["items"].nil? && !podInventory["items"].empty?) - $log.info("in_kube_podinventory::watch - initial_write : number of pod items :#{podInventory["items"].length} from Kube API @ #{Time.now.utc.iso8601}") - write_to_file(podInventory) - # parse_and_emit_records(podInventory, serviceRecords, continuationToken, batchTime) - else - $log.warn "in_kube_podinventory::watch - initial_write : Received empty podInventory" - end - end - end + # if (!podInventory.nil? && !podInventory.empty? && podInventory.key?("items") && !podInventory["items"].nil? && !podInventory["items"].empty?) + # $log.info("in_kube_podinventory::watch - initial_write : number of pod items :#{podInventory["items"].length} from Kube API @ #{Time.now.utc.iso8601}") + # $log.info("in_kube_podinventory::watch - initial_write : time to write to a file and emit to backend - functionality later") + # write_to_file(podInventory) + # # parse_and_emit_records(podInventory, serviceRecords, continuationToken, batchTime) + # else + # $log.warn "in_kube_podinventory::watch - initial_write : Received empty podInventory" + # end + + # #If we receive a continuation token, make calls, process and flush data until we have processed all data + # while (!continuationToken.nil? && !continuationToken.empty?) + # $log.info("in_kube_podinventory::watch - initial_write : continuation token was not null or empty so round two of gettings pod inventory") + # podsAPIChunkStartTime = (Time.now.to_f * 1000).to_i + # continuationToken, podInventory = KubernetesApiClient.getResourcesAndContinuationToken("pods?limit=#{@PODS_CHUNK_SIZE}&continue=#{continuationToken}") + # podsAPIChunkEndTime = (Time.now.to_f * 1000).to_i + # @podsAPIE2ELatencyMs = @podsAPIE2ELatencyMs + (podsAPIChunkEndTime - podsAPIChunkStartTime) + # if (!podInventory.nil? && !podInventory.empty? && podInventory.key?("items") && !podInventory["items"].nil? && !podInventory["items"].empty?) + # $log.info("in_kube_podinventory::watch - initial_write : number of pod items :#{podInventory["items"].length} from Kube API @ #{Time.now.utc.iso8601}") + # write_to_file(podInventory) + # # parse_and_emit_records(podInventory, serviceRecords, continuationToken, batchTime) + # else + # $log.warn "in_kube_podinventory::watch - initial_write : Received empty podInventory" + # end + # end + # end def watch - $log.info("in_kube_podinventory::watch : entered watch function - about to call initial write") + $log.info("in_kube_podinventory::watch : enters watch function - about to call enumerate") enumerate - - # $log.info("finished initial write to pod inventory file") $log.info("in_kube_podinventory::watch : finished getting pods, about to begin infinite loop for watch") loop do @@ -196,13 +200,13 @@ def watch @noticeHash[item["metadata"]["uid"]] = record } - $log.info("watch pods:: number of items in noticeHash = #{@noticeHash.size}") + $log.info("in_kube_podinventory::watch : watch pods:: number of items in noticeHash = #{@noticeHash.size}") end end rescue => exception $log.warn("in_kube_podinventory::watch : watch events session got broken and re-establishing the session.") # $log.debug_backtrace(exception.backtrace) - $log.info("watch events session broken backtrace: #{exception.backtrace}") + $log.info("in_kube_podinventory::watch : watch events session broken backtrace: #{exception.backtrace}") end sleep 300 end @@ -493,16 +497,15 @@ def parse_and_emit_records(podInventory, serviceRecords, continuationToken, batc end def merge_info - $log.info("merge_info:: enters this function, about to begin read file") + $log.info("in_kube_podinventory::merge_info : enters merge_info function, about to begin read file") begin fileContents = File.read("testing-podinventory.json") - # $log.info("in_kube_podinventory::merge_info : file contents read") $log.info("in_kube_podinventory::merge_info : file contents read, fileContents: #{fileContents}") @podHash = JSON.parse(fileContents) - $log.info("in_kube_podinventory::merge_info : parse successful") + $log.info("in_kube_podinventory::merge_info : parse successful, received podHash") rescue => error $log.info("in_kube_podinventory::merge_info : something went wrong with reading file") - $log.info("in_kube_podinventory::merge_info : backtrace: #{error.backtrace}") + $log.info("in_kube_podinventory::merge_info : reading file failed. backtrace: #{error.backtrace}") end $log.info("in_kube_podinventory::merge_info : before noticeHash loop, number of items in hash: #{@noticeHash.size()}, noticeHash: #{@noticeHash}") @@ -513,45 +516,53 @@ def merge_info @noticeHash.each do |uid, record| $log.info("in_kube_podinventory::merge_info : looping through noticeHash, type of notice: #{record["type"]}") # $log.info("podHash looks like: #{@podHash}") - $log.info("merge_info :: notice uid: #{uid}") - $log.info("merge_info :: notice record: #{record}") + $log.info("in_kube_podinventory::merge_info :: notice uid: #{uid}") + $log.info("in_kube_podinventory::merge_info :: notice record: #{record}") uidList.append(uid) case record["type"] when "ADDED" @podHash[uid] = record - $log.info("merge_info :: added") + $log.info("in_kube_podinventory::merge_info :: added to podhash") when "MODIFIED" - $log.info("merge_info :: entered modified case as expected") + $log.info("in_kube_podinventory::merge_info :: modify case") if @podHash[uid].nil? - $log.info("merge_info :: modify case where uid for add was overwritten to modify" ) + $log.info("in_kube_podinventory::merge_info :: modify case where uid for add was overwritten to modify" ) @podHash[uid] = record else - $log.info("merge_info :: modify case where it is a legit modify") + $log.info("in_kube_podinventory::merge_info :: modify case where it is a legit modify") val = @podHash[uid] val["status"] = record["status"] @podHash[uid] = val end - $log.info("merge_info :: modified") + $log.info("in_kube_podinventory::merge_info :: modified and changes reflected in podHash") when "DELETED" @podHash.delete(uid) - $log.info("merge_info :: deleted") + $log.info("in_kube_podinventory::merge_info :: deleted from podHash") else - $log.info("merge_info :: something went wrong") + $log.info("in_kube_podinventory::merge_info :: something went wrong and didn't enter any cases for switch") end # $log.info("uid: #{uid} and record: #{record}") - $log.info("merge_info :: end of switch") + $log.info("in_kube_podinventory::merge_info :: end of switch") end # remove all looked at uids from the noticeHash uidList.each do |uid| @noticeHash.delete(uid) end + $log.info("in_kube_podinventory::merge_info :: removed all visited uids from noticeHash") } $log.info("in_kube_podinventory:: merge_info : about to replace entire contents of testing-podinventory.json") - write_to_file(@podHash) + if (!@podHash.nil? && !@podHash.empty?) + $log.info("in_kube_podinventory:: merge_info : podHash not null and not empty, will write to file") + write_to_file(@podHash) + else + $log.info("in_kube_podinventory:: merge_info : podHash was either null or empty, so NOT writing to file - should never be in this case") + end + + # write_to_file(@podHash) # replace entire contents of testing-podinventory.json # File.open("testing-podinventory.json", "w") { |file| # file.write(JSON.pretty_generate(@podHash)) From 8a8f71039bfa78123158adfe3d44dda1c0a47a8a Mon Sep 17 00:00:00 2001 From: Khushi Chaudhari Date: Wed, 21 Jul 2021 13:34:17 -0700 Subject: [PATCH 19/76] updated yaml for test-12 --- kubernetes/omsagent.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/kubernetes/omsagent.yaml b/kubernetes/omsagent.yaml index d1f6cca56..ba44f4a79 100644 --- a/kubernetes/omsagent.yaml +++ b/kubernetes/omsagent.yaml @@ -368,7 +368,7 @@ spec: value: "3" containers: - name: omsagent - image: "mcr.microsoft.com/azuremonitor/containerinsights/cidev:khushi-test-11" + image: "mcr.microsoft.com/azuremonitor/containerinsights/cidev:khushi-test-12" imagePullPolicy: Always resources: limits: @@ -446,7 +446,7 @@ spec: timeoutSeconds: 15 #Only in sidecar scraping mode - name: omsagent-prometheus - image: "mcr.microsoft.com/azuremonitor/containerinsights/cidev:khushi-test-11" + image: "mcr.microsoft.com/azuremonitor/containerinsights/cidev:khushi-test-12" imagePullPolicy: Always resources: limits: @@ -589,7 +589,7 @@ spec: serviceAccountName: omsagent containers: - name: omsagent - image: "mcr.microsoft.com/azuremonitor/containerinsights/cidev:khushi-test-11" + image: "mcr.microsoft.com/azuremonitor/containerinsights/cidev:khushi-test-12" imagePullPolicy: Always resources: limits: From 03e9b60cfd2e3ec0222d0d0368da00f39e4e2ffa Mon Sep 17 00:00:00 2001 From: Khushi Chaudhari Date: Wed, 21 Jul 2021 23:28:22 -0700 Subject: [PATCH 20/76] added mmap2 gem and looking into merge logic by analyzing notice --- kubernetes/linux/setup.sh | 3 +- source/plugins/ruby/in_kube_podinventory.rb | 59 +++------------------ 2 files changed, 9 insertions(+), 53 deletions(-) diff --git a/kubernetes/linux/setup.sh b/kubernetes/linux/setup.sh index d140e1603..22cf50280 100644 --- a/kubernetes/linux/setup.sh +++ b/kubernetes/linux/setup.sh @@ -55,7 +55,8 @@ gem install gyoku iso8601 --no-doc # kubeclient gem sudo apt-get install libmagickwand-dev -y sudo gem install kubeclient --no-document - +# mmap2 gem +sudo gem install mmap2 rm -f $TMPDIR/docker-cimprov*.sh rm -f $TMPDIR/azure-mdsd*.deb diff --git a/source/plugins/ruby/in_kube_podinventory.rb b/source/plugins/ruby/in_kube_podinventory.rb index b8a51c1ae..b5c682b24 100644 --- a/source/plugins/ruby/in_kube_podinventory.rb +++ b/source/plugins/ruby/in_kube_podinventory.rb @@ -121,64 +121,17 @@ def write_to_file(podInventory) podInventory["items"].each do |item| podInventoryRecords = getPodInventoryRecords(item, servRecords, batchTime) File.open("testing-podinventory.json", "w") { |file| + $log.info("write_to_file:: makes it inside the file open just about to write pod inventory records") file.write(JSON.pretty_generate(podInventoryRecords)) } end $log.info("in_kube_podinventory:: write_to_file : successfully finished writing to file") rescue => exception $log.info("in_kube_podinventory::write_to_file : writing to file failed. backtrace: #{exception.backtrace}") + $log.info("write_to_file:: failed. podInventory: #{podInventory}") + $log.info("write_to_file:: failed. podInventory items: #{podInventory["items"]}") end - end - - # def initial_write - # $log.info("in_kube_podinventory::watch - initial_write: entered initial write function") - - # @podsAPIE2ELatencyMs = 0 - # podsAPIChunkStartTime = (Time.now.to_f * 1000).to_i - - # # Initializing continuation token to nil - # continuationToken = nil - # $log.info("in_kube_podinventory::watch - initial_write : Getting pods from Kube API @ #{Time.now.utc.iso8601}") - - # continuationToken, podInventory = KubernetesApiClient.getResourcesAndContinuationToken("pods?limit=#{@PODS_CHUNK_SIZE}") - # if (continuationToken.nil? || continuationToken.empty?) - # $log.info("in_kube_podinventory::watch - initial_write : continuation token is null or empty rip") - # end - - # @collection_version = podInventory["metadata"]["resourceVersion"] - - # $log.info("in_kube_podinventory::watch - initial_write : received collection version: #{@collection_version}") - # $log.info("in_kube_podinventory::watch : Done getting pods from Kube API @ #{Time.now.utc.iso8601}") - - # podsAPIChunkEndTime = (Time.now.to_f * 1000).to_i - # @podsAPIE2ELatencyMs = (podsAPIChunkEndTime - podsAPIChunkStartTime) - - # if (!podInventory.nil? && !podInventory.empty? && podInventory.key?("items") && !podInventory["items"].nil? && !podInventory["items"].empty?) - # $log.info("in_kube_podinventory::watch - initial_write : number of pod items :#{podInventory["items"].length} from Kube API @ #{Time.now.utc.iso8601}") - # $log.info("in_kube_podinventory::watch - initial_write : time to write to a file and emit to backend - functionality later") - # write_to_file(podInventory) - # # parse_and_emit_records(podInventory, serviceRecords, continuationToken, batchTime) - # else - # $log.warn "in_kube_podinventory::watch - initial_write : Received empty podInventory" - # end - - # #If we receive a continuation token, make calls, process and flush data until we have processed all data - # while (!continuationToken.nil? && !continuationToken.empty?) - # $log.info("in_kube_podinventory::watch - initial_write : continuation token was not null or empty so round two of gettings pod inventory") - # podsAPIChunkStartTime = (Time.now.to_f * 1000).to_i - # continuationToken, podInventory = KubernetesApiClient.getResourcesAndContinuationToken("pods?limit=#{@PODS_CHUNK_SIZE}&continue=#{continuationToken}") - # podsAPIChunkEndTime = (Time.now.to_f * 1000).to_i - # @podsAPIE2ELatencyMs = @podsAPIE2ELatencyMs + (podsAPIChunkEndTime - podsAPIChunkStartTime) - # if (!podInventory.nil? && !podInventory.empty? && podInventory.key?("items") && !podInventory["items"].nil? && !podInventory["items"].empty?) - # $log.info("in_kube_podinventory::watch - initial_write : number of pod items :#{podInventory["items"].length} from Kube API @ #{Time.now.utc.iso8601}") - # write_to_file(podInventory) - # # parse_and_emit_records(podInventory, serviceRecords, continuationToken, batchTime) - # else - # $log.warn "in_kube_podinventory::watch - initial_write : Received empty podInventory" - # end - # end - # end - + end def watch $log.info("in_kube_podinventory::watch : enters watch function - about to call enumerate") @@ -190,6 +143,7 @@ def watch begin @KubernetesWatchClient.watch_pods(resource_version: @collection_version, as: :parsed) do |notice| $log.info("in_kube_podinventory::watch : inside watch pods! Time: #{Time.now.utc.iso8601}") + $log.info("in_kube_podinventory::watch : notice looks like: #{notice}") if !notice.nil? && !notice.empty? $log.info("in_kube_podinventory::watch : received a notice that is not null and not empty, type: #{notice["type"]}") @@ -503,6 +457,7 @@ def merge_info $log.info("in_kube_podinventory::merge_info : file contents read, fileContents: #{fileContents}") @podHash = JSON.parse(fileContents) $log.info("in_kube_podinventory::merge_info : parse successful, received podHash") + $log.info("in_kube_podinventory::merge_info : podHash: #{podHash}") rescue => error $log.info("in_kube_podinventory::merge_info : something went wrong with reading file") $log.info("in_kube_podinventory::merge_info : reading file failed. backtrace: #{error.backtrace}") @@ -557,7 +512,7 @@ def merge_info $log.info("in_kube_podinventory:: merge_info : about to replace entire contents of testing-podinventory.json") if (!@podHash.nil? && !@podHash.empty?) $log.info("in_kube_podinventory:: merge_info : podHash not null and not empty, will write to file") - write_to_file(@podHash) + # write_to_file(@podHash) else $log.info("in_kube_podinventory:: merge_info : podHash was either null or empty, so NOT writing to file - should never be in this case") end From cb6601e99c6343516055b0fd6de4e7eb12e0bd08 Mon Sep 17 00:00:00 2001 From: Khushi Chaudhari Date: Wed, 21 Jul 2021 23:29:23 -0700 Subject: [PATCH 21/76] updated yaml test-13 --- kubernetes/omsagent.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/kubernetes/omsagent.yaml b/kubernetes/omsagent.yaml index ba44f4a79..3ce5e517e 100644 --- a/kubernetes/omsagent.yaml +++ b/kubernetes/omsagent.yaml @@ -368,7 +368,7 @@ spec: value: "3" containers: - name: omsagent - image: "mcr.microsoft.com/azuremonitor/containerinsights/cidev:khushi-test-12" + image: "mcr.microsoft.com/azuremonitor/containerinsights/cidev:khushi-test-13" imagePullPolicy: Always resources: limits: @@ -446,7 +446,7 @@ spec: timeoutSeconds: 15 #Only in sidecar scraping mode - name: omsagent-prometheus - image: "mcr.microsoft.com/azuremonitor/containerinsights/cidev:khushi-test-12" + image: "mcr.microsoft.com/azuremonitor/containerinsights/cidev:khushi-test-13" imagePullPolicy: Always resources: limits: @@ -589,7 +589,7 @@ spec: serviceAccountName: omsagent containers: - name: omsagent - image: "mcr.microsoft.com/azuremonitor/containerinsights/cidev:khushi-test-12" + image: "mcr.microsoft.com/azuremonitor/containerinsights/cidev:khushi-test-13" imagePullPolicy: Always resources: limits: From 6e9b6ecb834ec4c241353021d97a01d9d104c4f6 Mon Sep 17 00:00:00 2001 From: Khushi Chaudhari Date: Thu, 22 Jul 2021 14:09:49 -0700 Subject: [PATCH 22/76] added notice record creation for merging logic impl --- kubernetes/omsagent.yaml | 6 +- source/plugins/ruby/in_kube_podinventory.rb | 129 +++++++++++++++++--- 2 files changed, 116 insertions(+), 19 deletions(-) diff --git a/kubernetes/omsagent.yaml b/kubernetes/omsagent.yaml index 3ce5e517e..ac86187eb 100644 --- a/kubernetes/omsagent.yaml +++ b/kubernetes/omsagent.yaml @@ -368,7 +368,7 @@ spec: value: "3" containers: - name: omsagent - image: "mcr.microsoft.com/azuremonitor/containerinsights/cidev:khushi-test-13" + image: "mcr.microsoft.com/azuremonitor/containerinsights/cidev:khushi-test-14" imagePullPolicy: Always resources: limits: @@ -446,7 +446,7 @@ spec: timeoutSeconds: 15 #Only in sidecar scraping mode - name: omsagent-prometheus - image: "mcr.microsoft.com/azuremonitor/containerinsights/cidev:khushi-test-13" + image: "mcr.microsoft.com/azuremonitor/containerinsights/cidev:khushi-test-14" imagePullPolicy: Always resources: limits: @@ -589,7 +589,7 @@ spec: serviceAccountName: omsagent containers: - name: omsagent - image: "mcr.microsoft.com/azuremonitor/containerinsights/cidev:khushi-test-13" + image: "mcr.microsoft.com/azuremonitor/containerinsights/cidev:khushi-test-14" imagePullPolicy: Always resources: limits: diff --git a/source/plugins/ruby/in_kube_podinventory.rb b/source/plugins/ruby/in_kube_podinventory.rb index b5c682b24..072ae8665 100644 --- a/source/plugins/ruby/in_kube_podinventory.rb +++ b/source/plugins/ruby/in_kube_podinventory.rb @@ -120,9 +120,15 @@ def write_to_file(podInventory) begin podInventory["items"].each do |item| podInventoryRecords = getPodInventoryRecords(item, servRecords, batchTime) + podInventoryHash = {} + podInventoryRecords.each { |record| + uid = record["PodUid"] + podInventoryHash[uid] = record + } + $log.info("write_to_file:: pod inventory hash: #{podInventoryHash}") File.open("testing-podinventory.json", "w") { |file| $log.info("write_to_file:: makes it inside the file open just about to write pod inventory records") - file.write(JSON.pretty_generate(podInventoryRecords)) + file.write(JSON.pretty_generate(podInventoryHash)) } end $log.info("in_kube_podinventory:: write_to_file : successfully finished writing to file") @@ -133,6 +139,103 @@ def write_to_file(podInventory) end end + def getNoticeRecord(notice) + record = {} + item = notice["object"] + #TODO: check assumption that batch time can be current time (CollectionTime) + batchTime = Time.now.utc.iso8601 + + begin + record["CollectionTime"] = batchTime + record["Name"] = item["metadata"]["name"] + podNameSpace = item["metadata"]["namespace"] + #TODO: change uid later, handle case of horizontal scaling of pods but no controller (explained in getPodUid in KubenetesApiClient) + podUid = item["metadata"]["uid"] + + nodeName = "" + # For unscheduled (non-started) pods nodeName does NOT exist + if !item["spec"]["nodeName"].nil? + nodeName = item["spec"]["nodeName"] + end + + record["PodUid"] = podUid + record["PodLabel"] = [item["metadata"]["labels"]] + record["Namespace"] = podNameSpace + record["PodCreationTimeStamp"] = item["metadata"]["creationTimestamp"] + + if !item["status"]["startTime"].nil? + record["PodStartTime"] = item["status"]["startTime"] + else + record["PodStartTime"] = "" + end + + #podStatus + # NodeLost scenario -- pod(s) in the lost node is still being reported as running + podReadyCondition = true + if !item["status"]["reason"].nil? && item["status"]["reason"] == "NodeLost" && !item["status"]["conditions"].nil? + item["status"]["conditions"].each do |condition| + if condition["type"] == "Ready" && condition["status"] == "False" + podReadyCondition = false + break + end + end + end + if podReadyCondition == false + record["PodStatus"] = "Unknown" + elsif !item["metadata"]["deletionTimestamp"].nil? && !item["metadata"]["deletionTimestamp"].empty? + record["PodStatus"] = Constants::POD_STATUS_TERMINATING + else + record["PodStatus"] = item["status"]["phase"] + end + + # For unscheduled (non-started) pods podIP does NOT exist + if !item["status"]["podIP"].nil? + record["PodIp"] = item["status"]["podIP"] + else + record["PodIp"] = "" + end + + record["Computer"] = nodeName + #TODO: replace w KubernetesApiClient.getClusterId in agent code + record["ClusterId"] = "" + #TODO: replace w KubernetesApiClient.getClusterName in agent code + record["ClusterName"] = "" + #TODO: make a call to getServiceNameFromLabels -- need to pass in serviceRecords for this + record["ServiceName"] = "" + + if !item["metadata"]["ownerReferences"].nil? + record["ControllerKind"] = item["metadata"]["ownerReferences"][0]["kind"] + record["ControllerName"] = item["metadata"]["ownerReferences"][0]["name"] + # @controllerSet.add(record["ControllerKind"] + record["ControllerName"]) + # Adding controller kind to telemetry ro information about customer workload + # if (@controllerData[record["ControllerKind"]].nil?) + # @controllerData[record["ControllerKind"]] = 1 + # else + # controllerValue = @controllerData[record["ControllerKind"]] + # @controllerData[record["ControllerKind"]] += 1 + # end + end + + podRestartCount = 0 + record["PodRestartCount"] = 0 + + #TODO: popular real values for container fields + record["ContainerID"] = "" + record["ContainerName"] = "" + record["ContainerRestartCount"] = 0 + record["ContainerRestartReason"] = "" + record["ContainerStatus"] = "" + record["ContainerCreationTimeStamp"] = Time.now.utc.iso8601 + record["ContainerLastStatus"] = Hash.new + + record["NoticeType"] = notice["type"] + + rescue => exception + puts "getNoticeRecord failed: #{exception.backtrace}" + end + return record + end + def watch $log.info("in_kube_podinventory::watch : enters watch function - about to call enumerate") enumerate @@ -148,7 +251,9 @@ def watch $log.info("in_kube_podinventory::watch : received a notice that is not null and not empty, type: #{notice["type"]}") item = notice["object"] - record = {"name" => item["metadata"]["name"], "uid" => item["metadata"]["uid"], "status" => item["status"]["phase"], "type" => notice["type"]} + record = getNoticeRecord(notice) + # record = {"name" => item["metadata"]["name"], "uid" => item["metadata"]["uid"], "status" => item["status"]["phase"], "type" => notice["type"] + $log.info("in_kube_podinventory::watch : constructed record: #{record}") @mutex.synchronize { @noticeHash[item["metadata"]["uid"]] = record @@ -457,7 +562,7 @@ def merge_info $log.info("in_kube_podinventory::merge_info : file contents read, fileContents: #{fileContents}") @podHash = JSON.parse(fileContents) $log.info("in_kube_podinventory::merge_info : parse successful, received podHash") - $log.info("in_kube_podinventory::merge_info : podHash: #{podHash}") + $log.info("in_kube_podinventory::merge_info : podHash: #{@podHash}") rescue => error $log.info("in_kube_podinventory::merge_info : something went wrong with reading file") $log.info("in_kube_podinventory::merge_info : reading file failed. backtrace: #{error.backtrace}") @@ -469,14 +574,14 @@ def merge_info @mutex.synchronize { @noticeHash.each do |uid, record| - $log.info("in_kube_podinventory::merge_info : looping through noticeHash, type of notice: #{record["type"]}") + $log.info("in_kube_podinventory::merge_info : looping through noticeHash, type of notice: #{record["NoticeType"]}") # $log.info("podHash looks like: #{@podHash}") $log.info("in_kube_podinventory::merge_info :: notice uid: #{uid}") $log.info("in_kube_podinventory::merge_info :: notice record: #{record}") uidList.append(uid) - case record["type"] + case record["NoticeType"] when "ADDED" @podHash[uid] = record $log.info("in_kube_podinventory::merge_info :: added to podhash") @@ -486,9 +591,9 @@ def merge_info $log.info("in_kube_podinventory::merge_info :: modify case where uid for add was overwritten to modify" ) @podHash[uid] = record else - $log.info("in_kube_podinventory::merge_info :: modify case where it is a legit modify") + $log.info("in_kube_podinventory::merge_info :: modify case where it is a legit modify. new status is #{record["PodStatus"]}") val = @podHash[uid] - val["status"] = record["status"] + val["PodStatus"] = record["PodStatus"] @podHash[uid] = val end $log.info("in_kube_podinventory::merge_info :: modified and changes reflected in podHash") @@ -512,19 +617,11 @@ def merge_info $log.info("in_kube_podinventory:: merge_info : about to replace entire contents of testing-podinventory.json") if (!@podHash.nil? && !@podHash.empty?) $log.info("in_kube_podinventory:: merge_info : podHash not null and not empty, will write to file") - # write_to_file(@podHash) + write_to_file(@podHash) else $log.info("in_kube_podinventory:: merge_info : podHash was either null or empty, so NOT writing to file - should never be in this case") end - # write_to_file(@podHash) - # replace entire contents of testing-podinventory.json - # File.open("testing-podinventory.json", "w") { |file| - # file.write(JSON.pretty_generate(@podHash)) - # } - # File.open("testing-podinventory.json", "w") do |f| - # f.write JSON.pretty_generate(@podHash) - # end $log.info("in_kube_podinventory:: merge_info : finished replacing contents of testing-podinventory.json") end From 9be1c64f321dd2e14a33404ef28827b7093a043c Mon Sep 17 00:00:00 2001 From: Khushi Chaudhari Date: Thu, 22 Jul 2021 15:21:20 -0700 Subject: [PATCH 23/76] added flag for mmap and mmap file impl --- kubernetes/omsagent.yaml | 2 ++ source/plugins/ruby/in_kube_podinventory.rb | 34 +++++++++++++++++---- 2 files changed, 30 insertions(+), 6 deletions(-) diff --git a/kubernetes/omsagent.yaml b/kubernetes/omsagent.yaml index ac86187eb..2404968ae 100644 --- a/kubernetes/omsagent.yaml +++ b/kubernetes/omsagent.yaml @@ -400,6 +400,8 @@ spec: value: "" - name: AZMON_CONTAINERLOGS_ONEAGENT_REGIONS value: "koreacentral,norwayeast,eastus2" + - name: USEMMAP + value: false securityContext: privileged: true ports: diff --git a/source/plugins/ruby/in_kube_podinventory.rb b/source/plugins/ruby/in_kube_podinventory.rb index 072ae8665..4b3f7d83a 100644 --- a/source/plugins/ruby/in_kube_podinventory.rb +++ b/source/plugins/ruby/in_kube_podinventory.rb @@ -22,6 +22,7 @@ def initialize require "set" require "time" require "kubeclient" + require "mmap/mmap" require_relative "kubernetes_container_inventory" require_relative "KubernetesApiClient" @@ -44,6 +45,7 @@ def initialize @podsAPIE2ELatencyMs = 0 @noticeHash = {} + @useMmap = false @kubeperfTag = "oneagent.containerInsights.LINUX_PERF_BLOB" @kubeservicesTag = "oneagent.containerInsights.KUBE_SERVICES_BLOB" @@ -78,6 +80,13 @@ def start $log.warn("in_kube_podinventory::start: setting to default value since got PODS_EMIT_STREAM_BATCH_SIZE nil or empty") @PODS_EMIT_STREAM_BATCH_SIZE = 200 end + + if ENV["USEMMAP"] + @useMmap = true + end + + $log.info("in_kube_podinventory::start: use mmap is: #{@useMmap}") + # create kubernetes watch client ssl_options = { ca_file: "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt", @@ -125,11 +134,19 @@ def write_to_file(podInventory) uid = record["PodUid"] podInventoryHash[uid] = record } - $log.info("write_to_file:: pod inventory hash: #{podInventoryHash}") - File.open("testing-podinventory.json", "w") { |file| - $log.info("write_to_file:: makes it inside the file open just about to write pod inventory records") - file.write(JSON.pretty_generate(podInventoryHash)) - } + + if @useMmap + $log.info("mmap file write_to_file:: pod inventory hash: #{podInventoryHash}") + File.new("testing-podinventory.json", "w") + @mmap = Mmap.new("testing-podinventory.json", "rw") + @mmap << JSON.pretty_generate(podInventoryHash) + else + $log.info("regular file write_to_file:: pod inventory hash: #{podInventoryHash}") + File.open("testing-podinventory.json", "w") { |file| + $log.info("write_to_file:: makes it inside the file open just about to write pod inventory records") + file.write(JSON.pretty_generate(podInventoryHash)) + } + end end $log.info("in_kube_podinventory:: write_to_file : successfully finished writing to file") rescue => exception @@ -558,7 +575,12 @@ def parse_and_emit_records(podInventory, serviceRecords, continuationToken, batc def merge_info $log.info("in_kube_podinventory::merge_info : enters merge_info function, about to begin read file") begin - fileContents = File.read("testing-podinventory.json") + fileContents = "" + if @useMmap + fileContents << @mmap + else + fileContents = File.read("testing-podinventory.json") + end $log.info("in_kube_podinventory::merge_info : file contents read, fileContents: #{fileContents}") @podHash = JSON.parse(fileContents) $log.info("in_kube_podinventory::merge_info : parse successful, received podHash") From 9d1c9078a7e9fc18764adb06c58fd61a82e25a45 Mon Sep 17 00:00:00 2001 From: Khushi Chaudhari Date: Thu, 22 Jul 2021 15:22:40 -0700 Subject: [PATCH 24/76] updated yaml test-15 --- kubernetes/omsagent.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/kubernetes/omsagent.yaml b/kubernetes/omsagent.yaml index 2404968ae..56391f619 100644 --- a/kubernetes/omsagent.yaml +++ b/kubernetes/omsagent.yaml @@ -368,7 +368,7 @@ spec: value: "3" containers: - name: omsagent - image: "mcr.microsoft.com/azuremonitor/containerinsights/cidev:khushi-test-14" + image: "mcr.microsoft.com/azuremonitor/containerinsights/cidev:khushi-test-15" imagePullPolicy: Always resources: limits: @@ -448,7 +448,7 @@ spec: timeoutSeconds: 15 #Only in sidecar scraping mode - name: omsagent-prometheus - image: "mcr.microsoft.com/azuremonitor/containerinsights/cidev:khushi-test-14" + image: "mcr.microsoft.com/azuremonitor/containerinsights/cidev:khushi-test-15" imagePullPolicy: Always resources: limits: @@ -591,7 +591,7 @@ spec: serviceAccountName: omsagent containers: - name: omsagent - image: "mcr.microsoft.com/azuremonitor/containerinsights/cidev:khushi-test-14" + image: "mcr.microsoft.com/azuremonitor/containerinsights/cidev:khushi-test-15" imagePullPolicy: Always resources: limits: From 3dfca9f74e0580ad90db1cbb3b94982caaecb3b5 Mon Sep 17 00:00:00 2001 From: Khushi Chaudhari Date: Thu, 22 Jul 2021 18:44:04 -0700 Subject: [PATCH 25/76] changes to mmap usage + removed some files --- .../linux/dockerbuild/khushi-watch-test | 20 - kubernetes/omsagent.yaml | 911 ------------------ source/plugins/ruby/in_kube_podinventory.rb | 2 +- 3 files changed, 1 insertion(+), 932 deletions(-) delete mode 100644 kubernetes/linux/dockerbuild/khushi-watch-test delete mode 100644 kubernetes/omsagent.yaml diff --git a/kubernetes/linux/dockerbuild/khushi-watch-test b/kubernetes/linux/dockerbuild/khushi-watch-test deleted file mode 100644 index 69729d1ec..000000000 --- a/kubernetes/linux/dockerbuild/khushi-watch-test +++ /dev/null @@ -1,20 +0,0 @@ -apiVersion: v1 -clusters: -- cluster: - certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUU2RENDQXRDZ0F3SUJBZ0lRWDdub2hxOWthLzY4OHBzSHh0b1RHekFOQmdrcWhraUc5dzBCQVFzRkFEQU4KTVFzd0NRWURWUVFERXdKallUQWdGdzB5TVRBM01UWXlNekl4TVRoYUdBOHlNRFV4TURjeE5qSXpNekV4T0ZvdwpEVEVMTUFrR0ExVUVBeE1DWTJFd2dnSWlNQTBHQ1NxR1NJYjNEUUVCQVFVQUE0SUNEd0F3Z2dJS0FvSUNBUUMvCmR1aE1wZkxhWURuTHoyUjNrVW55M2hwbHREdGQ4cmlvakE1ZUZteXUveUJZcUl5Qi9LUDNMQWd4ajVwLzRtYlQKT2RqT0pseElKcW8vRlVGdU51b1E3YkRpSlVpZ0I3UVVQVVl0dVZCTUtFUHpuZk8yUFB1VXY3UVloZjNXMmpQegp1cUNGWkFRVnZUeDZmVTZLRjNhNkt1aC9pSkZLY2c1YnZGUVpnOWVNSy91QjlBdXZUSEx0aS9VRUJjWmM5UXBICm9hbklsUGo1N3dVV01zV25veWxRTndwSk1WczE4aDhaMStYS1pYakkvN0dVRTZhaEZwTVdmaU5GWDgybGNlbEQKQ0NwdzZSL3NiRyt1dXRuRURXQVpPRGFSOERpdFRBc3hyUHJMNHdDWXI5d05tY0tSQ3JwUXFsb3p6WFNGZGN5bApwbXZDMExYOVUwSGdZOURMMTZHaDUxbHRrSnN0dS9velhCd1BSM1JrdVVJZ3Q3WmpkOERSTmtlbVVvdGlyZHkvCm16VTJmb2hpSWE0a3VKR1JjVnR0RHZ0eVFSYTNPMzNHMzNQRVJTbkxCblpHSzRHMkVMbTBxTDh1WVFOZ1IveHoKeWlTdmozSlUrOVZoRFplaFlyOWNMWXFuSXRLci84anhVSFozU3RkNjhpSkFvQUJwZzFFWFRDc3pxamdxckJnagpLaG5YNHFCTlpQVmVUZjNrQjVkZ0VsZlJtMXIvcGsxblpFQmMrQllkcjBoSmNKTGgzSEYyR2hCM1BLc1NwUXE0CmVibVhQalZmanNxeFhSYVN2MTdzUTloSmhHRXN6WlBHU29SUjA1b3UwaU04ZHF3M0hjTnI4NzJuYW5uK2pXK24KTS9EaWxrSnJRY25WUVJLc1IxVkJ2ZnhKNjJzOFA2Ynl6V2FWL3NMNm1RSURBUUFCbzBJd1FEQU9CZ05WSFE4QgpBZjhFQkFNQ0FxUXdEd1lEVlIwVEFRSC9CQVV3QXdFQi96QWRCZ05WSFE0RUZnUVVCRG1ORE9VczRrVE9OLzlVClI3bkQ0MGNnR1VVd0RRWUpLb1pJaHZjTkFRRUxCUUFEZ2dJQkFEdGZ6M2pjV2lNeE1zaDIyT3I1RWV2R1pTRU0Kd0JDZmQ0QWMwUXlENkRmVkdaUFlwZmU3aDIzV1dKUWs5T05ncUszdFJhY1FUTTA0N2R3WUJGL3FTOEU2M3pKSQpINmZxL1lwQW9adHpZRkdaZTU1bUFaWWFQTlZWeDdvaHdoMzV2RTdFY0JaZ3JuS1pUN0EzOXlOWHRka2F2cThYCmhHQzdMV1lIMUxjQlhxckxHZlkvL1FpNnYraXdzZWlFdDkvUnRrSjhJK0VFd2JTWU9EazY5Z1lhSUtyK2RJaDgKbXdZM0d3VTZJUXl6MkFqb3pCaEcxbzQ1dzh1K1dKOTVyZEtrS0lEVG0ySTBNMEQ3dWhnZ0ZDT2srY25sOWNtSQozNkdqM2hIMlNFWC9Ea0tKN0orZ1o4bG1rc01HbGh3bGZ3K3FqRXVkVEQ4bCs3a1QrWWpMbEl3dTZRVkhvOGxWCkwyZ3diTG9tTzNJdU9IdHo4VUkxNVRJYSt4QmNsNDVSek1ZWWo0NHcvY2JiZStxUDE5THFQSUZXaXl1OG51UE4KUVhtcUtVR2wyOE83RmtteWFOckpuQURKNmZ5YU5nY21Bb1RrSU1oSk5IMHd2UkNjcTBGaXdJNkFYSGhEczJXYQpDNUkvS0c4NDg3dnlIRUJON3RhcldLN2RMMEhjcWt5cGl4cVkyNERRSnZLVUlIdUlHVk5sMGtza1Z4OWRKekxOCnkrd25SWndUTEVxRkNLSlMzU1F4eFl5Z0xLd09uZGVGTG1TSG1vbGJYM3IzYllDdkZLQ3BlNFA2V1Iwb0M1b20KOGZvMUpwLzI4SnF3K091RTJQZkFhRGFBaFNtK2tmb3V3WnIyWUNHSm50RjFTOEdlSFQxanFFRkJSWnp1UWd3eQp3R3cySnB0cjFEdmJvWmhaCi0tLS0tRU5EIENFUlRJRklDQVRFLS0tLS0K - server: https://khushi-watch-test-dns-9d093a4f.hcp.westus2.azmk8s.io:443 - name: khushi-watch-test -contexts: -- context: - cluster: khushi-watch-test - user: clusterUser_khushi-watch-test_khushi-watch-test - name: khushi-watch-test -current-context: khushi-watch-test -kind: Config -preferences: {} -users: -- name: clusterUser_khushi-watch-test_khushi-watch-test - user: - client-certificate-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUZIakNDQXdhZ0F3SUJBZ0lSQVArdktyZDVaRms2ajFycDVCS01adjh3RFFZSktvWklodmNOQVFFTEJRQXcKRFRFTE1Ba0dBMVVFQXhNQ1kyRXdIaGNOTWpFd056RTJNak15TVRFNFdoY05Nak13TnpFMk1qTXpNVEU0V2pBdwpNUmN3RlFZRFZRUUtFdzV6ZVhOMFpXMDZiV0Z6ZEdWeWN6RVZNQk1HQTFVRUF4TU1iV0Z6ZEdWeVkyeHBaVzUwCk1JSUNJakFOQmdrcWhraUc5dzBCQVFFRkFBT0NBZzhBTUlJQ0NnS0NBZ0VBcDhoV0piY3F4emFrejkrZjdRdG0KV0xvVzB3RFo0czlacmlDM25xZ2ZzaFBiYkMwSWlCdGxtQ05lYU1pRTVXeDBnby90VXQ5VUtvUlAwbzQ0Rlh2SApEZGVhU1J1OEllZnh2K1VUOXpkR0VwZDc4ejFBdlF3djhKYysyUjk5b2RnbWtYS0ZXeVhZb0UxRXpSMk9mbVVJCnVOVnNlNldoUHdCbFhNWXZmL0IwYU0rcHg0QzNZOGxnT2p4QTdybFRYYW1pNUlIemxZclQ0aVdtbHI1c05LczAKbkJkVWpicGtsdngxZk5DalR4S0tRaGFNRGdMc1dUUXBaVjI1b0FzYXhIcUx0Z0g3TytZaGZ2bnBJNUxLV3JNego1by9TQ3Vya0phK1RyWnVTbnJKWmgvR21NK09wczhXT1NJYVJpcEtLZlVtajNONzlUeEhkWTJVdThpNnZiTS9MCjhHMVZVYThRV3BnZ0wrbmtSWWRZQmdjSENKVjJZRUovSnlmSm9oK3c0RFF1aUV5RmZyNndmWlpKZGNnRlVRcTgKQllJZ29HNnZGQ3hMNEZwRW5ueWJHcFBLemczVUtmVmxCMGJ0Zkw0TjRZM3ZPYzhPeDgrWHZBWFMzSDdXV1I2TQpUeTFvMGxua1RRYW1uUVo1U2Y4TFU0OERHU2R5NFIzUTdiNW9UTU5TYmVHc0t0WXlxYURPQjE4WW0zTE9CQ0dvCnU4VE4vTFlsaFRHY0NSVklTOFE2SEoyYUZ0L1Z6b04zbXZlYitsUDF3LzNNaTltYjIyOTFsQ25hUG9pcE1memUKdFpoVUNOUURmUGpVRlFyS0dKODNhVzZkTWRRTVY0dXNzWG1Dbno5NXlWeEx6eEtRSVEzM0RKYWhVNlQ1VUtxRQpqY3ZqY09MWmFTdDE5Z21DMXh5QlF4OENBd0VBQWFOV01GUXdEZ1lEVlIwUEFRSC9CQVFEQWdXZ01CTUdBMVVkCkpRUU1NQW9HQ0NzR0FRVUZCd01DTUF3R0ExVWRFd0VCL3dRQ01BQXdId1lEVlIwakJCZ3dGb0FVQkRtTkRPVXMKNGtUT04vOVVSN25ENDBjZ0dVVXdEUVlKS29aSWh2Y05BUUVMQlFBRGdnSUJBRS9XNDdtU21PS0ZSYmJQV01aVQppTmROdmpzOVQ5QTVSc25WYk9uKzN5WFdGSk9pM0JucFBlOHNRc094MTlva0c4K2pIN2lpZWFmOW1SNEtUKzFJCjNMSy9uMWRNTVUzZXRTdE9zcmpob2ZYS3dNbVB3YklDYXAwaGs1T2w5Wk0xaFVIWEtGWGVMME9qQ2piVG5wa0gKYzdTd0JEd1NLcS83bDhYaWdYN29wclFxRTZUNFRLMWI5SDFQQ1JjZWw0UFJZZ0lwTXdHTWZOckFuK2F3dkJpVApZTmpudXljdEpHdXlXT2xGYVl6V281Skh3b2JSQUpJaG9sN1RFckM5Z28yUlpTL1dwSXNYbHhLREFaSnNmUGFkCmsrSmQzNlV6eU1QRnNvUHZqUWhlajlIVnl0Wk8rY1piYVlYbkd1eWx0U3Y2dVErUTd2cEtJVmoxTEN5cVJUL0kKWkxjM2ovMTNUNGFwLzExKytOaHRYNmtRM0JJRDdQS2VXd2YxeFF2NVRPTExIbG5VMTQ0WXI3b1NqK1QzWEdCSAp0bENqdHF5clFxMERZaitGaVhyb3JuUGJRMFFybG0zVE5lTVRLbktUSzdWMDl4a2FMRW8xalgrOHkxMzkwUzVMCk03bWRVTmlzS1A4T09EUTNMSVVETWdYRWEwWFFxdnJmMnJlVG9pVFhwZlJaK3JZU0RMVitsT3VoYzhlSFN6aVQKWXE0dDhJMlplMHdUUzdrTkhiMzlPYVVHYTc1Y2ZEdHZBcWxNWUk3UXhkU3hVYXNFd2Y0NytwUXdMZEl0UWFXcgp3cFZ2dk5nNnBIZEdVbG1VQVJ0dEhNMnR5YWhRNERXVmFoM0lubFlDYzRqVEtuNGc1aldxZkhBNWFsOEtNTWhQCmpNVWFVRGMzVkN5RGFacHRDeWRHQUgrLwotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0tCg== - client-key-data: LS0tLS1CRUdJTiBSU0EgUFJJVkFURSBLRVktLS0tLQpNSUlKS0FJQkFBS0NBZ0VBcDhoV0piY3F4emFrejkrZjdRdG1XTG9XMHdEWjRzOVpyaUMzbnFnZnNoUGJiQzBJCmlCdGxtQ05lYU1pRTVXeDBnby90VXQ5VUtvUlAwbzQ0Rlh2SERkZWFTUnU4SWVmeHYrVVQ5emRHRXBkNzh6MUEKdlF3djhKYysyUjk5b2RnbWtYS0ZXeVhZb0UxRXpSMk9mbVVJdU5Wc2U2V2hQd0JsWE1ZdmYvQjBhTStweDRDMwpZOGxnT2p4QTdybFRYYW1pNUlIemxZclQ0aVdtbHI1c05LczBuQmRVamJwa2x2eDFmTkNqVHhLS1FoYU1EZ0xzCldUUXBaVjI1b0FzYXhIcUx0Z0g3TytZaGZ2bnBJNUxLV3JNejVvL1NDdXJrSmErVHJadVNuckpaaC9HbU0rT3AKczhXT1NJYVJpcEtLZlVtajNONzlUeEhkWTJVdThpNnZiTS9MOEcxVlVhOFFXcGdnTCtua1JZZFlCZ2NIQ0pWMgpZRUovSnlmSm9oK3c0RFF1aUV5RmZyNndmWlpKZGNnRlVRcThCWUlnb0c2dkZDeEw0RnBFbm55YkdwUEt6ZzNVCktmVmxCMGJ0Zkw0TjRZM3ZPYzhPeDgrWHZBWFMzSDdXV1I2TVR5MW8wbG5rVFFhbW5RWjVTZjhMVTQ4REdTZHkKNFIzUTdiNW9UTU5TYmVHc0t0WXlxYURPQjE4WW0zTE9CQ0dvdThUTi9MWWxoVEdjQ1JWSVM4UTZISjJhRnQvVgp6b04zbXZlYitsUDF3LzNNaTltYjIyOTFsQ25hUG9pcE1memV0WmhVQ05RRGZQalVGUXJLR0o4M2FXNmRNZFFNClY0dXNzWG1Dbno5NXlWeEx6eEtRSVEzM0RKYWhVNlQ1VUtxRWpjdmpjT0xaYVN0MTlnbUMxeHlCUXg4Q0F3RUEKQVFLQ0FnQUhyaXdEWGZjZlYya0Qrd0NmSVQ1MklVNmFLaGZQUWg2ZzBlNlYzS3hXd29IdzJiN2lQQjdTY0F3SwpUK09GZlFsNFVJNVVsTlNOZmJFSnVtam0wdHV4em9USmcvT0F1ZFZmSzJWV2s3a3BjTFhEMUxINTlXemNYcEFKCjhGOFg0WVVpYzFPWGNJd1NDbmR6ekQ2Um1wNWpsNkYzcDRWU0ZQcU8zS09mLzZuVWdtMExMT0U0T0NlbmdzcVcKSXZXbCsvWHc3K0h1bm9SRWZlUzZVYzB5UEFRVWdSemx2L3FLenRPeCt2cit5Nko0ay8rbFJJejlLRjRjdmNXWgpoZWlieGVCUFhKZmJqaFZLY3JZeFlxN1FxQk5nSW9WQzZxMjI2K3FlcjlodVcwdXQ1V251UmlXbWpReU9WVnFrCk5VRVJxeVZOSHdnSmJvL3IxTjNwU0NuUG1WT0Vnd0t2Qk1iN24zZFE2ME1TZHIxVkNKOVl5ZkI3TGlVODVOdHYKY01CWWt3Y1FocWxPR1o0WlZJa2VjaWxvZnBmY2lJVWlzWmJpMTlsUnB4SHBoRzUvR0N3K3p4VVJ2U0xud0Rqcgo2Zy81ZCthWjQ0bFlSSzIwS09OQ003VnA1dHRnZHJwcmtnK211cldBZ0tIcThJQTB6KzhzZlVYWDNXUFoyelVTCnppb2NGcmlmdXhENm1aMEFHZEhRaFdqK1dMSEpYdnJtaENtYmFsUEk5NllVVWRxZlo0RHdkL1ZZcXlLOEI2TjkKM0RFVnI4SlhKU1piUFV1MEZDcS80amV2R29JekJPVHBrTVU1RGtSUlA4ODgwY1ZLMGoxNmwxYXhiajNYb3hVVQpuWmQ1UDhSMWx6dTNONzdYUHNydXBPRDloUmJXRDFiQnVKN0E4aER0bWlQZ3hwTUxRUUtDQVFFQXkrWmZHMWJqClpnNUZqcWhOclR2dDVENmlmWmorUkkrL0YyYlpLdjdsdVRRRkgwemkvazhMSk44VFQ2b3ZYQTFHM29vQ1ZNZnAKNUZEYmphL3l1azJKejFwcCtiZ1pYeHdwRUdpSm0vRjZRVGQvM2Q0RzhGL1hnQnZIb3FHSXZCTzRFRlBCcTFlTgppRGNCMEUzdnZTWEM5bTFsWGgvSEVHczYySVg2bG5Nd2tYbUcrRTZMT0NTRTNQcTFUMDQxaWdUOTNYTmIzQnZxCm1YWWNBeGlzZVJMUGY3aExpNVVaNTJ1ekJLUDJ1OHg0R2lEOVlPQm5oc1g2dnBIUVExOWozUjlLMm52U3ppRjMKMWlTWVVLbWhIU0xwY3RHY0ZzYTdRYWJvVklpdGYzOWEzRHl4dHZXNndPOEx1TWVGRnMzdytqR3JrbTVIZ1BjZgpMY1A0T203T2xOa2JmUUtDQVFFQTBxZHZqUkYrMWxlRHZJUFhuSy9wTk5kTUNza21DTko1b25MTllKTWxNMVJTCk4velBZbjg3bXRsbnBzb3QrR0ZXS0pEaHkwTHlhUk4zZERacHJvRWxzOFVnNUt5NFZiZTV3OFloKzR5K0FrMFQKM3lwSTVDdUllNFpaTndpbEF5RkVWTkNhTXJaU3Y1eGpaR0pZQU1aa05mRE5XU3NSbFowUEkycFhYRDFCVnA5UQo4djVDWG5UTVZrTSs4NVZVN3dKMHl3QU1IRDdqVGM1RGVnaGZJT3NlZzVIc3RBTTJSbXc3dHU5YVVlL0t5eGVDCjZPY2NOMGhtT3VuZ2lPakE5MXYxWm5nV0lxaUZMZFVUUG9YS0JsWlRwbSt5QUJCTC92M0dWYm5NcUdzcnFlV3kKaS9ZQ1ZoSlVXcGZPa2t2djVUTTJXU0hKTGZYejU3Q25oL1Jodi80RHl3S0NBUUVBbGVlYWlJa0t5ZDg0T0RkNApWQ0k1TlpMdTk1UGhiQnFhak9QcVNYZjVBSDVFUGN6VEhkQ1RDdHFPWWdWbXFEQ0NwOTJpOVIyODBVUzVCYVFUCnVmQ0RudFNFRVRuT1BXU0F0RFdHNWdWVXNsblJRaGFYMTJVL0ZFcFlMVExCU2pUZEgvUFQ4TnAvaldPVGk4ZWcKdDlqcFN5OEdWWHJiYVRETXBKOXJxZXlxQ21ua1Z1MjM0T0RJWllaVWdpZW5xUDhlZlE3d3ZCUXlGLzBEZnl0RwpzZ1NvVk9memNuMTkrK3ZzUXo4Z2lOVThmMGs5djFsOEExUE9rd2kwcXJPWXdkcmR0MTlOam9xQWhnbUZpZGdNClFWV0hlQWl1enZmd2Q4WDBEU0luSEJOUGc1ZUR1RlZVcGIrWlNKSTNRSnJMemNWeXRFY3JmcDh3WnY4cW9oc1EKK2RIY0tRS0NBUUIydGF5RFRzeXZkWG5qamxpL0Q5TFgyRXdkOStUYksrRW56cEkyVGpXMGkxd1orUG5WZytYUApDWjBEdlFQUzZPZG96Tnl4Y3ZTd2lpdlM3YWI0bEFidkc3UEJxaVBuQ2paQlFUSjlVMzd3UlFkaHg2NTBCcVJQCkdCTEdsTFNJNHdKaTJYdE1BTHI1QitScStaQ21QeWJSenZXcHZqK1dsSCtuY1pIeGhRT0JFUjdKRU1mTit2djcKME1GMCt6NTF3bWlXelZ3RnZ0clJTZDkwc2FzS0hmV0FKNGZBbWQ3SUtNNHQ5UXh0Q0RpNzRtLy9WOTNxdVg0Qwp6VEdmZGFyb3VvVzhUUWdNVi9OVk5MSTNsYVdYbEtabk1LS2FycFJsQ2hYdTBWbW52MTFIelEybEdlbVBINTB4CndFTEU1czQxMTNwVkdGa0s4WU9WbVZPUHBnUS84YUdwQW9JQkFFa256SzNMeElSRmowVHB4TlJaSEp1NVNVdHkKTGRGaDQwd3kyUjNJWS9SSGpBbmx1TTBmUUQydjN1ZTY1VHhaenY4Y2tiZ1pKTkE0eGxTd2M0a3U5M3pkSkxYZwpLZzZPTU52cWZBbW9nMzkvekV3cEZ2MEZsdXpYTWdEdjVaSk5EYm0yMjhMaFJaQ1E3TFdKNWZMZXREY1BVaFRVCmd0UW9WVGZ6Zm5jTVMwa3IzQzdFSnQxam91UXMvNFhrVlEvZTkweTVkNTkwRGV5YmRSNnlDMWt5bU9sWVZJWDgKK0drdkxuZnFpMnZRZzdVR1hNTEY1LzRRNjZYRWl3NzVxcEdDTUFTdnNSa216L2FWYkpFNzNvR2dnZ1BGc3ZOOQpVcEFuR1NqaUFpd2Z5RmJ5d0NRV2hnUG1SbDhXV2xjTStZM0QxL1lLTElTVU5xQURnMGZXVXdkYVhFMD0KLS0tLS1FTkQgUlNBIFBSSVZBVEUgS0VZLS0tLS0K - token: 71b6ec454624539f8cf4a07c38d4c0efb015da0d5a9720766ab1e1d7c68c5920906350d2f4745a395a55abc5620c5f0002124c2f870358b61670078f137b5aa9 diff --git a/kubernetes/omsagent.yaml b/kubernetes/omsagent.yaml deleted file mode 100644 index 56391f619..000000000 --- a/kubernetes/omsagent.yaml +++ /dev/null @@ -1,911 +0,0 @@ -apiVersion: v1 -kind: ServiceAccount -metadata: - name: omsagent - namespace: kube-system ---- -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: omsagent-reader -rules: - - apiGroups: [""] - resources: - [ - "pods", - "events", - "nodes", - "nodes/stats", - "nodes/metrics", - "nodes/spec", - "nodes/proxy", - "namespaces", - "services", - "persistentvolumes" - ] - verbs: ["list", "get", "watch"] - - apiGroups: ["apps", "extensions", "autoscaling"] - resources: ["replicasets", "deployments", "horizontalpodautoscalers"] - verbs: ["list"] - - apiGroups: ["azmon.container.insights"] - resources: ["healthstates"] - verbs: ["get", "create", "patch"] - - nonResourceURLs: ["/metrics"] - verbs: ["get"] ---- -kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: omsagentclusterrolebinding -subjects: - - kind: ServiceAccount - name: omsagent - namespace: kube-system -roleRef: - kind: ClusterRole - name: omsagent-reader - apiGroup: rbac.authorization.k8s.io ---- -kind: ConfigMap -apiVersion: v1 -data: - kube.conf: |- - # Fluentd config file for OMS Docker - cluster components (kubeAPI) - #fluent forward plugin - - type forward - port "#{ENV['HEALTHMODEL_REPLICASET_SERVICE_SERVICE_PORT']}" - bind 0.0.0.0 - chunk_size_limit 4m - - - #Kubernetes pod inventory - - type kubepodinventory - tag oms.containerinsights.KubePodInventory - run_interval 60 - log_level debug - - - #Kubernetes Persistent Volume inventory - - type kubepvinventory - tag oms.containerinsights.KubePVInventory - run_interval 60 - log_level debug - - - #Kubernetes events - - type kubeevents - tag oms.containerinsights.KubeEvents - run_interval 60 - log_level debug - - - #Kubernetes Nodes - - type kubenodeinventory - tag oms.containerinsights.KubeNodeInventory - run_interval 60 - log_level debug - - - #Kubernetes health - - type kubehealth - tag kubehealth.ReplicaSet - run_interval 60 - log_level debug - - - #cadvisor perf- Windows nodes - - type wincadvisorperf - tag oms.api.wincadvisorperf - run_interval 60 - log_level debug - - - #Kubernetes object state - deployments - - type kubestatedeployments - tag oms.containerinsights.KubeStateDeployments - run_interval 60 - log_level debug - - - #Kubernetes object state - HPA - - type kubestatehpa - tag oms.containerinsights.KubeStateHpa - run_interval 60 - log_level debug - - - - type filter_inventory2mdm - log_level info - - - #custom_metrics_mdm filter plugin for perf data from windows nodes - - type filter_cadvisor2mdm - metrics_to_collect cpuUsageNanoCores,memoryWorkingSetBytes,pvUsedBytes - log_level info - - - #health model aggregation filter - - type filter_health_model_builder - - - - type out_oms - log_level debug - num_threads 2 - buffer_chunk_limit 4m - buffer_type file - buffer_path %STATE_DIR_WS%/out_oms_kubepods*.buffer - buffer_queue_limit 20 - buffer_queue_full_action drop_oldest_chunk - flush_interval 20s - retry_limit 10 - retry_wait 5s - max_retry_wait 5m - - - - type out_oms - log_level debug - num_threads 5 - buffer_chunk_limit 4m - buffer_type file - buffer_path %STATE_DIR_WS%/state/out_oms_kubepv*.buffer - buffer_queue_limit 20 - buffer_queue_full_action drop_oldest_chunk - flush_interval 20s - retry_limit 10 - retry_wait 5s - max_retry_wait 5m - - - - type out_oms - log_level debug - num_threads 2 - buffer_chunk_limit 4m - buffer_type file - buffer_path %STATE_DIR_WS%/out_oms_kubeevents*.buffer - buffer_queue_limit 20 - buffer_queue_full_action drop_oldest_chunk - flush_interval 20s - retry_limit 10 - retry_wait 5s - max_retry_wait 5m - - - - type out_oms - log_level debug - num_threads 2 - buffer_chunk_limit 4m - buffer_type file - buffer_path %STATE_DIR_WS%/out_oms_kubeservices*.buffer - buffer_queue_limit 20 - buffer_queue_full_action drop_oldest_chunk - flush_interval 20s - retry_limit 10 - retry_wait 5s - max_retry_wait 5m - - - - type out_oms - log_level debug - num_threads 2 - buffer_chunk_limit 4m - buffer_type file - buffer_path %STATE_DIR_WS%/state/out_oms_kubenodes*.buffer - buffer_queue_limit 20 - buffer_queue_full_action drop_oldest_chunk - flush_interval 20s - retry_limit 10 - retry_wait 5s - max_retry_wait 5m - - - - type out_oms - log_level debug - num_threads 3 - buffer_chunk_limit 4m - buffer_type file - buffer_path %STATE_DIR_WS%/out_oms_containernodeinventory*.buffer - buffer_queue_limit 20 - flush_interval 20s - retry_limit 10 - retry_wait 5s - max_retry_wait 5m - - - - type out_oms - log_level debug - num_threads 2 - buffer_chunk_limit 4m - buffer_type file - buffer_path %STATE_DIR_WS%/out_oms_kubeperf*.buffer - buffer_queue_limit 20 - buffer_queue_full_action drop_oldest_chunk - flush_interval 20s - retry_limit 10 - retry_wait 5s - max_retry_wait 5m - - - - type out_mdm - log_level debug - num_threads 5 - buffer_chunk_limit 4m - buffer_type file - buffer_path %STATE_DIR_WS%/out_mdm_*.buffer - buffer_queue_limit 20 - buffer_queue_full_action drop_oldest_chunk - flush_interval 20s - retry_limit 10 - retry_wait 5s - max_retry_wait 5m - retry_mdm_post_wait_minutes 30 - - - - type out_oms - log_level debug - num_threads 5 - buffer_chunk_limit 4m - buffer_type file - buffer_path %STATE_DIR_WS%/out_oms_api_wincadvisorperf*.buffer - buffer_queue_limit 20 - buffer_queue_full_action drop_oldest_chunk - flush_interval 20s - retry_limit 10 - retry_wait 5s - max_retry_wait 5m - - - - type out_mdm - log_level debug - num_threads 5 - buffer_chunk_limit 4m - buffer_type file - buffer_path %STATE_DIR_WS%/out_mdm_cdvisorperf*.buffer - buffer_queue_limit 20 - buffer_queue_full_action drop_oldest_chunk - flush_interval 20s - retry_limit 10 - retry_wait 5s - max_retry_wait 5m - retry_mdm_post_wait_minutes 30 - - - - type out_oms - log_level debug - num_threads 5 - buffer_chunk_limit 4m - buffer_type file - buffer_path %STATE_DIR_WS%/out_oms_kubehealth*.buffer - buffer_queue_limit 20 - buffer_queue_full_action drop_oldest_chunk - flush_interval 20s - retry_limit 10 - retry_wait 5s - max_retry_wait 5m - - - - type out_oms - log_level debug - num_threads 5 - buffer_chunk_limit 4m - buffer_type file - buffer_path %STATE_DIR_WS%/out_oms_insightsmetrics*.buffer - buffer_queue_limit 20 - buffer_queue_full_action drop_oldest_chunk - flush_interval 20s - retry_limit 10 - retry_wait 5s - max_retry_wait 5m - - -metadata: - name: omsagent-rs-config - namespace: kube-system ---- -apiVersion: v1 -kind: Secret -metadata: - name: omsagent-secret - namespace: kube-system -type: Opaque -data: - #BASE64 ENCODED (Both WSID & KEY) INSIDE DOUBLE QUOTE ("") - WSID: "ZTNkYmI1OTItNDhhNS00N2IwLWE2MTAtMTUwYzU1ZmM1Y2ZmCg==" - KEY: "U3htc215eXY3UXYrZzk3QXZVdlZzd3FlSW1ENHoyRlpjdHJGamtHUUl5NUx5bDhYUEc1RmxuQU5wRkhFLzNJczBWSjd2elFuS3FIa2NDOVhwWU5UR3c9PQo=" ---- -apiVersion: apps/v1 -kind: DaemonSet -metadata: - name: omsagent - namespace: kube-system - labels: - component: oms-agent - tier: node -spec: - updateStrategy: - type: RollingUpdate - selector: - matchLabels: - component: oms-agent - tier: node - template: - metadata: - labels: - component: oms-agent - tier: node - annotations: - agentVersion: "1.10.0.1" - dockerProviderVersion: "16.0.0-0" - schema-versions: "v1" - spec: - serviceAccountName: omsagent - dnsConfig: - options: - - name: ndots - value: "3" - containers: - - name: omsagent - image: "mcr.microsoft.com/azuremonitor/containerinsights/cidev:khushi-test-15" - imagePullPolicy: Always - resources: - limits: - cpu: 500m - memory: 600Mi - requests: - cpu: 75m - memory: 225Mi - env: - # azure devops pipeline uses AKS_RESOURCE_ID and AKS_REGION hence ensure to uncomment these - - name: AKS_RESOURCE_ID - value: "/subscriptions/b4b0cbab-64dc-42bc-9d95-ea7c8d0cf188/resourcegroups/khushi-watch-test/providers/Microsoft.ContainerService/managedClusters/khushi-watch-test" - - name: AKS_REGION - value: "West US 2" - # this used for e2e test and setting this just emits some additional log statements which used for the e2e tests - - name: ISTEST - value: "true" - #Uncomment below two lines for ACS clusters and set the cluster names manually. Also comment out the above two lines for ACS clusters - #- name: ACS_RESOURCE_NAME - # value: "my_acs_cluster_name" - - name: CONTROLLER_TYPE - value: "DaemonSet" - - name: NODE_IP - valueFrom: - fieldRef: - fieldPath: status.hostIP - # Update this with the user assigned msi client id for omsagent - - name: USER_ASSIGNED_IDENTITY_CLIENT_ID - value: "" - - name: AZMON_CONTAINERLOGS_ONEAGENT_REGIONS - value: "koreacentral,norwayeast,eastus2" - - name: USEMMAP - value: false - securityContext: - privileged: true - ports: - - containerPort: 25225 - protocol: TCP - - containerPort: 25224 - protocol: UDP - volumeMounts: - - mountPath: /hostfs - name: host-root - readOnly: true - - mountPath: /var/run/host - name: docker-sock - - mountPath: /var/log - name: host-log - - mountPath: /var/lib/docker/containers - name: containerlog-path - readOnly: true - - mountPath: /mnt/docker - name: containerlog-path-2 - readOnly: true - - mountPath: /mnt/containers - name: containerlog-path-3 - readOnly: true - - mountPath: /etc/kubernetes/host - name: azure-json-path - - mountPath: /etc/omsagent-secret - name: omsagent-secret - readOnly: true - - mountPath: /etc/config/settings - name: settings-vol-config - readOnly: true - - mountPath: /etc/config/settings/adx - name: omsagent-adx-secret - readOnly: true - livenessProbe: - exec: - command: - - /bin/bash - - -c - - /opt/livenessprobe.sh - initialDelaySeconds: 60 - periodSeconds: 60 - timeoutSeconds: 15 -#Only in sidecar scraping mode - - name: omsagent-prometheus - image: "mcr.microsoft.com/azuremonitor/containerinsights/cidev:khushi-test-15" - imagePullPolicy: Always - resources: - limits: - cpu: 500m - memory: 1Gi - requests: - cpu: 75m - memory: 225Mi - env: - # azure devops pipeline uses AKS_RESOURCE_ID and AKS_REGION hence ensure to uncomment these - - name: AKS_RESOURCE_ID - value: "/subscriptions/b4b0cbab-64dc-42bc-9d95-ea7c8d0cf188/resourcegroups/khushi-watch-test/providers/Microsoft.ContainerService/managedClusters/khushi-watch-test" - - name: AKS_REGION - value: "West US 2" - #Uncomment below two lines for ACS clusters and set the cluster names manually. Also comment out the above two lines for ACS clusters - #- name: ACS_RESOURCE_NAME - # value: "my_acs_cluster_name" - - name: CONTAINER_TYPE - value: "PrometheusSidecar" - - name: CONTROLLER_TYPE - value: "DaemonSet" - - name: NODE_IP - valueFrom: - fieldRef: - fieldPath: status.hostIP - # Update this with the user assigned msi client id for omsagent - - name: USER_ASSIGNED_IDENTITY_CLIENT_ID - value: "" - securityContext: - privileged: true - volumeMounts: - - mountPath: /etc/kubernetes/host - name: azure-json-path - - mountPath: /etc/omsagent-secret - name: omsagent-secret - readOnly: true - - mountPath: /etc/config/settings - name: settings-vol-config - readOnly: true - - mountPath: /etc/config/osm-settings - name: osm-settings-vol-config - readOnly: true - livenessProbe: - exec: - command: - - /bin/bash - - -c - - /opt/livenessprobe.sh - initialDelaySeconds: 60 - periodSeconds: 60 - timeoutSeconds: 15 - affinity: - nodeAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - - labelSelector: - matchExpressions: - # kubernetes.io/os label doesnt exist in k8s versions < 1.14 so make sure to choose label based on k8s version in aks yaml - - key: kubernetes.io/os - operator: In - values: - - linux - - key: type - operator: NotIn - values: - - virtual-kubelet - # Tolerate a NoSchedule taint on master that ACS Engine sets. - tolerations: - - operator: "Exists" - effect: "NoSchedule" - - operator: "Exists" - effect: "NoExecute" - - operator: "Exists" - effect: "PreferNoSchedule" - volumes: - - name: host-root - hostPath: - path: / - - name: docker-sock - hostPath: - path: /var/run - - name: container-hostname - hostPath: - path: /etc/hostname - - name: host-log - hostPath: - path: /var/log - - name: containerlog-path - hostPath: - path: /var/lib/docker/containers - - name: containerlog-path-2 - hostPath: - path: /mnt/docker - - name: containerlog-path-3 - hostPath: - path: /mnt/containers - - name: azure-json-path - hostPath: - path: /etc/kubernetes - - name: omsagent-secret - secret: - secretName: omsagent-secret - - name: settings-vol-config - configMap: - name: container-azm-ms-agentconfig - optional: true - - name: omsagent-adx-secret - secret: - secretName: omsagent-adx-secret - optional: true - - name: osm-settings-vol-config - configMap: - name: container-azm-ms-osmconfig - optional: true ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: omsagent-rs - namespace: kube-system - labels: - component: oms-agent - tier: node -spec: - replicas: 1 - selector: - matchLabels: - rsName: "omsagent-rs" - strategy: - type: RollingUpdate - template: - metadata: - labels: - rsName: "omsagent-rs" - annotations: - agentVersion: "1.10.0.1" - dockerProviderVersion: "16.0.0-0" - schema-versions: "v1" - spec: - serviceAccountName: omsagent - containers: - - name: omsagent - image: "mcr.microsoft.com/azuremonitor/containerinsights/cidev:khushi-test-15" - imagePullPolicy: Always - resources: - limits: - cpu: 1 - memory: 1Gi - requests: - cpu: 150m - memory: 250Mi - env: - - name: AKS_RESOURCE_ID - value: "/subscriptions/b4b0cbab-64dc-42bc-9d95-ea7c8d0cf188/resourcegroups/khushi-watch-test/providers/Microsoft.ContainerService/managedClusters/khushi-watch-test" - - name: AKS_REGION - value: "West US 2" - # this used for e2e test and setting this just emits some additional log statements which used for the e2e tests - - name: ISTEST - value: "true" - # Uncomment below two lines for ACS clusters and set the cluster names manually. Also comment out the above two lines for ACS clusters - #- name: ACS_RESOURCE_NAME - # value: "my_acs_cluster_name" - - name: CONTROLLER_TYPE - value: "ReplicaSet" - - name: NODE_IP - valueFrom: - fieldRef: - fieldPath: status.hostIP - # Update this with the user assigned msi client id for omsagent - - name: USER_ASSIGNED_IDENTITY_CLIENT_ID - value: "" - # Add the below environment variable to true only in sidecar enabled regions, else set it to false - - name: SIDECAR_SCRAPING_ENABLED - value: "true" - securityContext: - privileged: true - ports: - - containerPort: 25225 - protocol: TCP - - containerPort: 25224 - protocol: UDP - - containerPort: 25227 - protocol: TCP - name: in-rs-tcp - volumeMounts: - - mountPath: /var/run/host - name: docker-sock - - mountPath: /var/log - name: host-log - - mountPath: /etc/kubernetes/host - name: azure-json-path - - mountPath: /etc/omsagent-secret - name: omsagent-secret - readOnly: true - - mountPath: /etc/config - name: omsagent-rs-config - - mountPath: /etc/config/settings - name: settings-vol-config - readOnly: true - - mountPath: /etc/config/settings/adx - name: omsagent-adx-secret - - mountPath: /etc/config/osm-settings - name: osm-settings-vol-config - readOnly: true - # livenessProbe: - # exec: - # command: - # - /bin/bash - # - -c - # - /opt/livenessprobe.sh - # initialDelaySeconds: 60 - # periodSeconds: 60 - # timeoutSeconds: 15 - affinity: - nodeAffinity: - # affinity to schedule on to ephemeral os node if its available - preferredDuringSchedulingIgnoredDuringExecution: - - weight: 1 - preference: - matchExpressions: - - key: storageprofile - operator: NotIn - values: - - managed - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - - labelSelector: - matchExpressions: - - key: beta.kubernetes.io/os - operator: In - values: - - linux - - key: type - operator: NotIn - values: - - virtual-kubelet - # The following label selector is removed for AKS, this is only required for non AKS - - key: kubernetes.io/role - operator: NotIn - values: - - master - # The following tolerations are removed for AKS, this is only required for non AKS - tolerations: - - operator: "Exists" - effect: "NoSchedule" - - operator: "Exists" - effect: "NoExecute" - - operator: "Exists" - effect: "PreferNoSchedule" - volumes: - - name: docker-sock - hostPath: - path: /var/run - - name: container-hostname - hostPath: - path: /etc/hostname - - name: host-log - hostPath: - path: /var/log - - name: azure-json-path - hostPath: - path: /etc/kubernetes - - name: omsagent-secret - secret: - secretName: omsagent-secret - - name: omsagent-rs-config - configMap: - name: omsagent-rs-config - - name: settings-vol-config - configMap: - name: container-azm-ms-agentconfig - optional: true - - name: omsagent-adx-secret - secret: - secretName: omsagent-adx-secret - optional: true - - name: osm-settings-vol-config - configMap: - name: container-azm-ms-osmconfig - optional: true ---- - apiVersion: apps/v1 - kind: DaemonSet - metadata: - name: omsagent-win - namespace: kube-system - labels: - component: oms-agent-win - tier: node-win - spec: - updateStrategy: - type: RollingUpdate - selector: - matchLabels: - component: oms-agent-win - tier: node-win - template: - metadata: - labels: - component: oms-agent-win - tier: node-win - annotations: - agentVersion: "1.10.0.1" - dockerProviderVersion: "16.0.0-0" - schema-versions: "v1" - spec: - serviceAccountName: omsagent - dnsConfig: - options: - - name: ndots - value: "3" - containers: - - name: omsagent-win - image: "mcr.microsoft.com/azuremonitor/containerinsights/ciprod:win-ciprod06112021" - imagePullPolicy: IfNotPresent - resources: - limits: - cpu: 200m - memory: 600Mi - env: - # azure devops pipeline uses AKS_RESOURCE_ID and AKS_REGION hence ensure to uncomment these - - name: AKS_RESOURCE_ID - value: "/subscriptions/b4b0cbab-64dc-42bc-9d95-ea7c8d0cf188/resourcegroups/khushi-watch-test/providers/Microsoft.ContainerService/managedClusters/khushi-watch-test" - - name: AKS_REGION - value: "West US 2" - #- name: ACS_RESOURCE_NAME - # value: "my_acs_cluster_name" - - name: CONTROLLER_TYPE - value: "DaemonSet" - - name: HOSTNAME - valueFrom: - fieldRef: - fieldPath: spec.nodeName - - name: PODNAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: NODE_IP - valueFrom: - fieldRef: - fieldPath: status.hostIP - - name: SIDECAR_SCRAPING_ENABLED - value: "true" - # Update this with the user assigned msi client id for omsagent - - name: USER_ASSIGNED_IDENTITY_CLIENT_ID - value: "" - # Add this only for clouds that require cert bootstrapping - - name: REQUIRES_CERT_BOOTSTRAP - value: "true" - volumeMounts: - - mountPath: C:\ProgramData\docker\containers - name: docker-windows-containers - readOnly: true - - mountPath: C:\var #Read + Write access on this for position file - name: docker-windows-kuberenetes-container-logs - - mountPath: C:\etc\config\settings - name: settings-vol-config - readOnly: true - - mountPath: C:\etc\omsagent-secret - name: omsagent-secret - readOnly: true - - mountPath: C:\etc\config\adx - name: omsagent-adx-secret - readOnly: true - # Need to mount this only for airgapped clouds - Commenting this since it wont exist in non airgapped clouds - # - mountPath: C:\ca - # name: ca-certs - # readOnly: true - - mountPath: C:\etc\kubernetes\host - name: azure-json-path - readOnly: true - livenessProbe: - exec: - command: - - cmd - - /c - - C:\opt\omsagentwindows\scripts\cmd\livenessProbe.cmd - periodSeconds: 60 - initialDelaySeconds: 180 - timeoutSeconds: 15 - affinity: - nodeAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - - matchExpressions: - - key: beta.kubernetes.io/os - operator: In - values: - - windows - tolerations: - - key: "CriticalAddonsOnly" - operator: "Exists" - - operator: "Exists" - effect: NoExecute - - operator: "Exists" - effect: NoSchedule - - operator: "Exists" - effect: PreferNoSchedule - volumes: - - name: docker-windows-kuberenetes-container-logs - hostPath: - path: C:\var - - name: azure-json-path - hostPath: - path: C:\k - # Need to mount this only for airgapped clouds - Commenting this since it wont exist in non airgapped clouds - #- name: ca-certs - # hostPath: - # path: C:\ca - - name: docker-windows-containers - hostPath: - path: C:\ProgramData\docker\containers - type: DirectoryOrCreate - - name: settings-vol-config - configMap: - name: container-azm-ms-agentconfig - optional: true - - name: omsagent-secret - secret: - secretName: omsagent-secret - - name: omsagent-adx-secret - secret: - secretName: omsagent-adx-secret - optional: true ---- -kind: Service -apiVersion: v1 -metadata: - name: healthmodel-replicaset-service - namespace: kube-system -spec: - selector: - rsName: "omsagent-rs" - ports: - - protocol: TCP - port: 25227 - targetPort: in-rs-tcp ---- -# this is for versions >=1.19, for versions <1.19 we continue to use v1beta1 -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - name: healthstates.azmon.container.insights - namespace: kube-system -spec: - group: azmon.container.insights - versions: - - name: v1 - served: true - storage: true - schema: - openAPIV3Schema: - type: object - properties: - state: - type: string - scope: Namespaced - names: - plural: healthstates - kind: HealthState diff --git a/source/plugins/ruby/in_kube_podinventory.rb b/source/plugins/ruby/in_kube_podinventory.rb index 4b3f7d83a..ad9235462 100644 --- a/source/plugins/ruby/in_kube_podinventory.rb +++ b/source/plugins/ruby/in_kube_podinventory.rb @@ -81,7 +81,7 @@ def start @PODS_EMIT_STREAM_BATCH_SIZE = 200 end - if ENV["USEMMAP"] + if (!ENV["USEMMAP"].nil? && !ENV["USEMMAP"].empty? && ENV["USEMMAP"].casecmp("true") == 0) @useMmap = true end From 3de0a395396caa12d3ca47ab3b4d7bebfa5f7cbb Mon Sep 17 00:00:00 2001 From: Khushi Chaudhari Date: Fri, 23 Jul 2021 13:53:20 -0700 Subject: [PATCH 26/76] added parse_and_emit_records for merge updates --- source/plugins/ruby/in_kube_podinventory.rb | 224 +++++++++++++++----- 1 file changed, 168 insertions(+), 56 deletions(-) diff --git a/source/plugins/ruby/in_kube_podinventory.rb b/source/plugins/ruby/in_kube_podinventory.rb index ad9235462..8ceb54f75 100644 --- a/source/plugins/ruby/in_kube_podinventory.rb +++ b/source/plugins/ruby/in_kube_podinventory.rb @@ -121,13 +121,13 @@ def shutdown end def write_to_file(podInventory) - $log.info("in_kube_podinventory::write_to_file : enters write_to_file function") batchTime = Time.now.utc.iso8601 #TODO: check if you can pass @serviceRecords into getPodInventoryRecords rather than creating a local copy servRecords= @serviceRecords begin podInventory["items"].each do |item| + # Extract needed fields using getPodInventoryRecords and create a hash mapping uid -> record podInventoryRecords = getPodInventoryRecords(item, servRecords, batchTime) podInventoryHash = {} podInventoryRecords.each { |record| @@ -135,20 +135,20 @@ def write_to_file(podInventory) podInventoryHash[uid] = record } + # Write to mmap or regular file based on value of @useMmap flag if @useMmap - $log.info("mmap file write_to_file:: pod inventory hash: #{podInventoryHash}") + $log.info("in_kube_podinventory::write_to_file : writing to mmap file case") File.new("testing-podinventory.json", "w") @mmap = Mmap.new("testing-podinventory.json", "rw") @mmap << JSON.pretty_generate(podInventoryHash) else - $log.info("regular file write_to_file:: pod inventory hash: #{podInventoryHash}") + $log.info("in_kube_podinventory::write_to_file : writing to regular file case") File.open("testing-podinventory.json", "w") { |file| - $log.info("write_to_file:: makes it inside the file open just about to write pod inventory records") file.write(JSON.pretty_generate(podInventoryHash)) } end end - $log.info("in_kube_podinventory:: write_to_file : successfully finished writing to file") + # $log.info("in_kube_podinventory::write_to_file : successfully finished writing to file") rescue => exception $log.info("in_kube_podinventory::write_to_file : writing to file failed. backtrace: #{exception.backtrace}") $log.info("write_to_file:: failed. podInventory: #{podInventory}") @@ -157,6 +157,7 @@ def write_to_file(podInventory) end def getNoticeRecord(notice) + # Helper function that extracts necessary fields from notice JSON record = {} item = notice["object"] #TODO: check assumption that batch time can be current time (CollectionTime) @@ -254,36 +255,34 @@ def getNoticeRecord(notice) end def watch - $log.info("in_kube_podinventory::watch : enters watch function - about to call enumerate") + $log.info("in_kube_podinventory::watch : calling enumerate to make API server call and populate file with initial pod inventory data.") enumerate - $log.info("in_kube_podinventory::watch : finished getting pods, about to begin infinite loop for watch") loop do - $log.info("in_kube_pod_inventory::watch: inside infinite loop for watch pods. collection version: #{@collection_version}") + #TODO: check if collection_version is correct when continuation token is not null and collection_version changes + $log.info("in_kube_pod_inventory::watch : inside infinite loop for watch pods. collection version: #{@collection_version}") begin @KubernetesWatchClient.watch_pods(resource_version: @collection_version, as: :parsed) do |notice| - $log.info("in_kube_podinventory::watch : inside watch pods! Time: #{Time.now.utc.iso8601}") - $log.info("in_kube_podinventory::watch : notice looks like: #{notice}") + $log.info("in_kube_podinventory::watch : inside watch pods! current time: #{Time.now.utc.iso8601}. notice: #{JSON.pretty_generate(notice)}") if !notice.nil? && !notice.empty? - $log.info("in_kube_podinventory::watch : received a notice that is not null and not empty, type: #{notice["type"]}") + $log.info("in_kube_podinventory::watch : received a notice that is not null and not empty. notice type: #{notice["type"]}") item = notice["object"] + # Construct record with necessary fields (same fields as getPodInventoryRecords) record = getNoticeRecord(notice) - # record = {"name" => item["metadata"]["name"], "uid" => item["metadata"]["uid"], "status" => item["status"]["phase"], "type" => notice["type"] - $log.info("in_kube_podinventory::watch : constructed record: #{record}") @mutex.synchronize { @noticeHash[item["metadata"]["uid"]] = record } - $log.info("in_kube_podinventory::watch : watch pods:: number of items in noticeHash = #{@noticeHash.size}") + $log.info("in_kube_podinventory::watch : number of items in noticeHash = #{@noticeHash.size}") end end rescue => exception - $log.warn("in_kube_podinventory::watch : watch events session got broken and re-establishing the session.") + $log.warn("in_kube_podinventory::watch : watch events session got broken and re-establishing the session. backtrace: #{exception.backtrace}") # $log.debug_backtrace(exception.backtrace) - $log.info("in_kube_podinventory::watch : watch events session broken backtrace: #{exception.backtrace}") end + #TODO: check if 300 is the correct number to use here sleep 300 end end @@ -336,7 +335,7 @@ def enumerate(podList = nil) if (!podInventory.nil? && !podInventory.empty? && podInventory.key?("items") && !podInventory["items"].nil? && !podInventory["items"].empty?) $log.info("in_kube_podinventory::enumerate : number of pod items :#{podInventory["items"].length} from Kube API @ #{Time.now.utc.iso8601}") write_to_file(podInventory) - # parse_and_emit_records(podInventory, @serviceRecords, continuationToken, batchTime) + parse_and_emit_records(podInventory, @serviceRecords, continuationToken, batchTime) else $log.warn "in_kube_podinventory::enumerate:Received empty podInventory" end @@ -351,7 +350,7 @@ def enumerate(podList = nil) if (!podInventory.nil? && !podInventory.empty? && podInventory.key?("items") && !podInventory["items"].nil? && !podInventory["items"].empty?) $log.info("in_kube_podinventory::enumerate : number of pod items :#{podInventory["items"].length} from Kube API @ #{Time.now.utc.iso8601}") write_to_file(podInventory) - # parse_and_emit_records(podInventory, @serviceRecords, continuationToken, batchTime) + parse_and_emit_records(podInventory, @serviceRecords, continuationToken, batchTime) else $log.warn "in_kube_podinventory::enumerate:Received empty podInventory" end @@ -375,6 +374,7 @@ def enumerate(podList = nil) telemetryProperties["Computer"] = @@hostName telemetryProperties["PODS_CHUNK_SIZE"] = @PODS_CHUNK_SIZE telemetryProperties["PODS_EMIT_STREAM_BATCH_SIZE"] = @PODS_EMIT_STREAM_BATCH_SIZE + telemetryProperties["USE_MMAP"] = @useMmap ApplicationInsightsUtility.sendCustomEvent("KubePodInventoryHeartBeatEvent", telemetryProperties) ApplicationInsightsUtility.sendMetricTelemetry("PodCount", @podCount, {}) ApplicationInsightsUtility.sendMetricTelemetry("ServiceCount", @serviceCount, {}) @@ -572,79 +572,191 @@ def parse_and_emit_records(podInventory, serviceRecords, continuationToken, batc end #begin block end end - def merge_info - $log.info("in_kube_podinventory::merge_info : enters merge_info function, about to begin read file") + def parse_and_emit_merge_updates(podInventoryRecords) + currentTime = Time.now + emitTime = Fluent::Engine.now + batchTime = currentTime.utc.iso8601 + eventStream = Fluent::MultiEventStream.new + containerInventoryStream = Fluent::MultiEventStream.new + kubePerfEventStream = Fluent::MultiEventStream.new + insightsMetricsEventStream = Fluent::MultiEventStream.new + @@istestvar = ENV["ISTEST"] + + continuationToken = nil + + begin #begin block start + # Getting windows nodes from kubeapi + winNodes = KubernetesApiClient.getWindowsNodesArray + + podInventoryRecords.each do |uid, record| + if !record.nil? + eventStream.add(emitTime, record) if record + @inventoryToMdmConvertor.process_pod_inventory_record(record) + end + end + + if @PODS_EMIT_STREAM_BATCH_SIZE > 0 && eventStream.count >= @PODS_EMIT_STREAM_BATCH_SIZE + $log.info("in_kube_podinventory::parse_and_emit_merge_updates: number of pod inventory records emitted #{@PODS_EMIT_STREAM_BATCH_SIZE} @ #{Time.now.utc.iso8601}") + if (!@@istestvar.nil? && !@@istestvar.empty? && @@istestvar.casecmp("true") == 0) + $log.info("kubePodInventoryEmitStreamSuccess @ #{Time.now.utc.iso8601}") + end + router.emit_stream(@tag, eventStream) if eventStream + eventStream = Fluent::MultiEventStream.new + end + + if eventStream.count > 0 + $log.info("in_kube_podinventory::parse_and_emit_merge_updates: number of pod inventory records emitted #{eventStream.count} @ #{Time.now.utc.iso8601}") + router.emit_stream(@tag, eventStream) if eventStream + if (!@@istestvar.nil? && !@@istestvar.empty? && @@istestvar.casecmp("true") == 0) + $log.info("kubePodInventoryEmitStreamSuccess @ #{Time.now.utc.iso8601}") + end + eventStream = nil + end + + if containerInventoryStream.count > 0 + $log.info("in_kube_podinventory::parse_and_emit_merge_updates: number of windows container inventory records emitted #{containerInventoryStream.count} @ #{Time.now.utc.iso8601}") + router.emit_stream(@containerInventoryTag, containerInventoryStream) if containerInventoryStream + if (!@@istestvar.nil? && !@@istestvar.empty? && @@istestvar.casecmp("true") == 0) + $log.info("kubeWindowsContainerInventoryEmitStreamSuccess @ #{Time.now.utc.iso8601}") + end + containerInventoryStream = nil + end + + if kubePerfEventStream.count > 0 + $log.info("in_kube_podinventory::parse_and_emit_merge_updates: number of perf records emitted #{kubePerfEventStream.count} @ #{Time.now.utc.iso8601}") + router.emit_stream(@kubeperfTag, kubePerfEventStream) if kubePerfEventStream + kubePerfEventStream = nil + if (!@@istestvar.nil? && !@@istestvar.empty? && @@istestvar.casecmp("true") == 0) + $log.info("kubeContainerPerfEventEmitStreamSuccess @ #{Time.now.utc.iso8601}") + end + end + + if insightsMetricsEventStream.count > 0 + $log.info("in_kube_podinventory::parse_and_emit_merge_updates: number of insights metrics records emitted #{insightsMetricsEventStream.count} @ #{Time.now.utc.iso8601}") + router.emit_stream(@insightsMetricsTag, insightsMetricsEventStream) if insightsMetricsEventStream + if (!@@istestvar.nil? && !@@istestvar.empty? && @@istestvar.casecmp("true") == 0) + $log.info("kubePodInsightsMetricsEmitStreamSuccess @ #{Time.now.utc.iso8601}") + end + insightsMetricsEventStream = nil + end + + if continuationToken.nil? #no more chunks in this batch to be sent, get all mdm pod inventory records to send + @log.info "Sending pod inventory mdm records to out_mdm" + pod_inventory_mdm_records = @inventoryToMdmConvertor.get_pod_inventory_mdm_records(batchTime) + @log.info "pod_inventory_mdm_records.size #{pod_inventory_mdm_records.size}" + mdm_pod_inventory_es = Fluent::MultiEventStream.new + pod_inventory_mdm_records.each { |pod_inventory_mdm_record| + mdm_pod_inventory_es.add(batchTime, pod_inventory_mdm_record) if pod_inventory_mdm_record + } if pod_inventory_mdm_records + router.emit_stream(@@MDMKubePodInventoryTag, mdm_pod_inventory_es) if mdm_pod_inventory_es + end + + if continuationToken.nil? # sending kube services inventory records + kubeServicesEventStream = Fluent::MultiEventStream.new + # serviceRecords.each do |kubeServiceRecord| + # if !kubeServiceRecord.nil? + # # adding before emit to reduce memory foot print + # kubeServiceRecord["ClusterId"] = KubernetesApiClient.getClusterId + # kubeServiceRecord["ClusterName"] = KubernetesApiClient.getClusterName + # kubeServicesEventStream.add(emitTime, kubeServiceRecord) if kubeServiceRecord + # if @PODS_EMIT_STREAM_BATCH_SIZE > 0 && kubeServicesEventStream.count >= @PODS_EMIT_STREAM_BATCH_SIZE + # $log.info("in_kube_podinventory::parse_and_emit_merge_updates: number of service records emitted #{@PODS_EMIT_STREAM_BATCH_SIZE} @ #{Time.now.utc.iso8601}") + # router.emit_stream(@kubeservicesTag, kubeServicesEventStream) if kubeServicesEventStream + # kubeServicesEventStream = Fluent::MultiEventStream.new + # if (!@@istestvar.nil? && !@@istestvar.empty? && @@istestvar.casecmp("true") == 0) + # $log.info("kubeServicesEventEmitStreamSuccess @ #{Time.now.utc.iso8601}") + # end + # end + # end + # end + + if kubeServicesEventStream.count > 0 + $log.info("in_kube_podinventory::parse_and_emit_merge_updates : number of service records emitted #{kubeServicesEventStream.count} @ #{Time.now.utc.iso8601}") + router.emit_stream(@kubeservicesTag, kubeServicesEventStream) if kubeServicesEventStream + if (!@@istestvar.nil? && !@@istestvar.empty? && @@istestvar.casecmp("true") == 0) + $log.info("kubeServicesEventEmitStreamSuccess @ #{Time.now.utc.iso8601}") + end + end + kubeServicesEventStream = nil + end + + #Updating value for AppInsights telemetry + @podCount += podInventory["items"].length + rescue => errorStr + $log.warn "Failed in parse_and_emit_merge_updates pod inventory: #{errorStr}" + $log.debug_backtrace(errorStr.backtrace) + ApplicationInsightsUtility.sendExceptionTelemetry(errorStr) + end #begin block end + end + + def merge_updates begin fileContents = "" + # Read file if @useMmap fileContents << @mmap else fileContents = File.read("testing-podinventory.json") end - $log.info("in_kube_podinventory::merge_info : file contents read, fileContents: #{fileContents}") - @podHash = JSON.parse(fileContents) - $log.info("in_kube_podinventory::merge_info : parse successful, received podHash") - $log.info("in_kube_podinventory::merge_info : podHash: #{@podHash}") + # $log.info("in_kube_podinventory::merge_updates : file contents read, fileContents: #{fileContents}") + @podInventoryHash = JSON.parse(fileContents) + $log.info("in_kube_podinventory::merge_updates : parse successful, received podInventoryHash: #{podInventoryHash}") rescue => error - $log.info("in_kube_podinventory::merge_info : something went wrong with reading file") - $log.info("in_kube_podinventory::merge_info : reading file failed. backtrace: #{error.backtrace}") + $log.info("in_kube_podinventory::merge_updates : something went wrong with reading file. #{error.backtrace}") end - $log.info("in_kube_podinventory::merge_info : before noticeHash loop, number of items in hash: #{@noticeHash.size()}, noticeHash: #{@noticeHash}") + $log.info("in_kube_podinventory::merge_updates : before noticeHash loop, number of items in hash: #{@noticeHash.size()}, noticeHash: #{@noticeHash}") uidList = [] @mutex.synchronize { @noticeHash.each do |uid, record| - $log.info("in_kube_podinventory::merge_info : looping through noticeHash, type of notice: #{record["NoticeType"]}") - # $log.info("podHash looks like: #{@podHash}") - $log.info("in_kube_podinventory::merge_info :: notice uid: #{uid}") - $log.info("in_kube_podinventory::merge_info :: notice record: #{record}") + $log.info("in_kube_podinventory::merge_updates : looping through noticeHash, type of notice: #{record["NoticeType"]}. notice uid: #{uid}. notice record: #{record}") uidList.append(uid) case record["NoticeType"] when "ADDED" - @podHash[uid] = record - $log.info("in_kube_podinventory::merge_info :: added to podhash") + @podInventoryHash[uid] = record + $log.info("in_kube_podinventory::merge_updates : added new record to podInventoryHash") when "MODIFIED" - $log.info("in_kube_podinventory::merge_info :: modify case") - if @podHash[uid].nil? - $log.info("in_kube_podinventory::merge_info :: modify case where uid for add was overwritten to modify" ) - @podHash[uid] = record + if @podInventoryHash[uid].nil? + $log.info("in_kube_podinventory::merge_updates : modify case where uid for add was overwritten to modify within same minute") + @podInventoryHash[uid] = record else - $log.info("in_kube_podinventory::merge_info :: modify case where it is a legit modify. new status is #{record["PodStatus"]}") - val = @podHash[uid] + $log.info("in_kube_podinventory::merge_updates : modify case where it is only a modification. old status: #{@podInventoryHash[uid]["PodStatus"]}. new status: #{record["PodStatus"]}") + val = @podInventoryHash[uid] val["PodStatus"] = record["PodStatus"] - @podHash[uid] = val + @podInventoryHash[uid] = val end - $log.info("in_kube_podinventory::merge_info :: modified and changes reflected in podHash") + $log.info("in_kube_podinventory::merge_updates :: modified and changes reflected in podInventoryHash") when "DELETED" - @podHash.delete(uid) - $log.info("in_kube_podinventory::merge_info :: deleted from podHash") + @podInventoryHash.delete(uid) + $log.info("in_kube_podinventory::merge_updates :: deleted from podInventoryHash") else - $log.info("in_kube_podinventory::merge_info :: something went wrong and didn't enter any cases for switch") + $log.info("in_kube_podinventory::merge_updates :: something went wrong and didn't enter any cases for switch") end - # $log.info("uid: #{uid} and record: #{record}") - $log.info("in_kube_podinventory::merge_info :: end of switch") + # $log.info("in_kube_podinventory::merge_updates :: end of switch") end # remove all looked at uids from the noticeHash uidList.each do |uid| @noticeHash.delete(uid) end - $log.info("in_kube_podinventory::merge_info :: removed all visited uids from noticeHash") + $log.info("in_kube_podinventory::merge_updates :: removed all visited uids from noticeHash") } - $log.info("in_kube_podinventory:: merge_info : about to replace entire contents of testing-podinventory.json") - if (!@podHash.nil? && !@podHash.empty?) - $log.info("in_kube_podinventory:: merge_info : podHash not null and not empty, will write to file") - write_to_file(@podHash) + #TODO: Look for a way to replace only necessary contents, rather than everything + $log.info("in_kube_podinventory:: merge_updates : about to replace entire contents of testing-podinventory.json") + if (!@podInventoryHash.nil? && !@podInventoryHash.empty?) + $log.info("in_kube_podinventory:: merge_updates : podInventoryHash not null and not empty, will write to file") + write_to_file(@podInventoryHash) + parse_and_emit_merge_updates(@podInventoryHash) else - $log.info("in_kube_podinventory:: merge_info : podHash was either null or empty, so NOT writing to file - should never be in this case") + $log.info("in_kube_podinventory:: merge_updates : podInventoryHash was either null or empty, so NOT writing to file - should never be in this case") end - $log.info("in_kube_podinventory:: merge_info : finished replacing contents of testing-podinventory.json") + $log.info("in_kube_podinventory:: merge_updates : finished replacing contents of testing-podinventory.json") end def run_periodic @@ -666,10 +778,10 @@ def run_periodic @mutex.unlock if !done begin - $log.info("in_kube_podinventory::run_periodic.merge_info.start #{Time.now.utc.iso8601}") + $log.info("in_kube_podinventory::run_periodic.merge_updates.start #{Time.now.utc.iso8601}") # enumerate - merge_info - $log.info("in_kube_podinventory::run_periodic.merge_info.end #{Time.now.utc.iso8601}") + merge_updates + $log.info("in_kube_podinventory::run_periodic.merge_updates.end #{Time.now.utc.iso8601}") rescue => errorStr $log.warn "in_kube_podinventory::run_periodic: enumerate Failed to retrieve pod inventory: #{errorStr}" ApplicationInsightsUtility.sendExceptionTelemetry(errorStr) From 1eb6d297a7c098163b11073595c4cda87904b763 Mon Sep 17 00:00:00 2001 From: Khushi Chaudhari Date: Sat, 24 Jul 2021 10:50:25 -0700 Subject: [PATCH 27/76] fixed write to file bug when coming from merge_updates --- source/plugins/ruby/in_kube_podinventory.rb | 50 ++++++++++++--------- 1 file changed, 28 insertions(+), 22 deletions(-) diff --git a/source/plugins/ruby/in_kube_podinventory.rb b/source/plugins/ruby/in_kube_podinventory.rb index 8ceb54f75..7dd107e89 100644 --- a/source/plugins/ruby/in_kube_podinventory.rb +++ b/source/plugins/ruby/in_kube_podinventory.rb @@ -125,28 +125,34 @@ def write_to_file(podInventory) #TODO: check if you can pass @serviceRecords into getPodInventoryRecords rather than creating a local copy servRecords= @serviceRecords + podInventoryHash = {} + begin - podInventory["items"].each do |item| - # Extract needed fields using getPodInventoryRecords and create a hash mapping uid -> record - podInventoryRecords = getPodInventoryRecords(item, servRecords, batchTime) - podInventoryHash = {} - podInventoryRecords.each { |record| - uid = record["PodUid"] - podInventoryHash[uid] = record - } - - # Write to mmap or regular file based on value of @useMmap flag - if @useMmap - $log.info("in_kube_podinventory::write_to_file : writing to mmap file case") - File.new("testing-podinventory.json", "w") - @mmap = Mmap.new("testing-podinventory.json", "rw") - @mmap << JSON.pretty_generate(podInventoryHash) - else - $log.info("in_kube_podinventory::write_to_file : writing to regular file case") - File.open("testing-podinventory.json", "w") { |file| - file.write(JSON.pretty_generate(podInventoryHash)) + if !podInventory["items"].nil? && !podInventory["items"].empty? + podInventory["items"].each do |item| + # Extract needed fields using getPodInventoryRecords and create a hash mapping uid -> record + podInventoryRecords = getPodInventoryRecords(item, servRecords, batchTime) + podInventoryRecords.each { |record| + uid = record["PodUid"] + podInventoryHash[uid] = record } end + else + podInventoryHash = podInventory + end + + + # Write to mmap or regular file based on value of @useMmap flag + if @useMmap + $log.info("in_kube_podinventory::write_to_file : writing to mmap file case") + File.new("testing-podinventory.json", "w") + @mmap = Mmap.new("testing-podinventory.json", "rw") + @mmap << JSON.pretty_generate(podInventoryHash) + else + $log.info("in_kube_podinventory::write_to_file : writing to regular file case") + File.open("testing-podinventory.json", "w") { |file| + file.write(JSON.pretty_generate(podInventoryHash)) + } end # $log.info("in_kube_podinventory::write_to_file : successfully finished writing to file") rescue => exception @@ -215,9 +221,9 @@ def getNoticeRecord(notice) record["Computer"] = nodeName #TODO: replace w KubernetesApiClient.getClusterId in agent code - record["ClusterId"] = "" + record["ClusterId"] = KubernetesApiClient.getClusterId #TODO: replace w KubernetesApiClient.getClusterName in agent code - record["ClusterName"] = "" + record["ClusterName"] = KubernetesApiClient.getClusterName #TODO: make a call to getServiceNameFromLabels -- need to pass in serviceRecords for this record["ServiceName"] = "" @@ -681,7 +687,7 @@ def parse_and_emit_merge_updates(podInventoryRecords) end #Updating value for AppInsights telemetry - @podCount += podInventory["items"].length + # @podCount += podInventory["items"].length rescue => errorStr $log.warn "Failed in parse_and_emit_merge_updates pod inventory: #{errorStr}" $log.debug_backtrace(errorStr.backtrace) From 3635321288df545fe7605431cd645bf5138d154a Mon Sep 17 00:00:00 2001 From: Khushi Chaudhari Date: Sat, 24 Jul 2021 12:02:30 -0700 Subject: [PATCH 28/76] fixed a log statement bug --- source/plugins/ruby/in_kube_podinventory.rb | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/source/plugins/ruby/in_kube_podinventory.rb b/source/plugins/ruby/in_kube_podinventory.rb index 7dd107e89..e8b10b194 100644 --- a/source/plugins/ruby/in_kube_podinventory.rb +++ b/source/plugins/ruby/in_kube_podinventory.rb @@ -126,7 +126,7 @@ def write_to_file(podInventory) servRecords= @serviceRecords podInventoryHash = {} - + begin if !podInventory["items"].nil? && !podInventory["items"].empty? podInventory["items"].each do |item| @@ -154,7 +154,7 @@ def write_to_file(podInventory) file.write(JSON.pretty_generate(podInventoryHash)) } end - # $log.info("in_kube_podinventory::write_to_file : successfully finished writing to file") + $log.info("in_kube_podinventory::write_to_file : successfully finished writing to file") rescue => exception $log.info("in_kube_podinventory::write_to_file : writing to file failed. backtrace: #{exception.backtrace}") $log.info("write_to_file:: failed. podInventory: #{podInventory}") @@ -269,7 +269,7 @@ def watch $log.info("in_kube_pod_inventory::watch : inside infinite loop for watch pods. collection version: #{@collection_version}") begin @KubernetesWatchClient.watch_pods(resource_version: @collection_version, as: :parsed) do |notice| - $log.info("in_kube_podinventory::watch : inside watch pods! current time: #{Time.now.utc.iso8601}. notice: #{JSON.pretty_generate(notice)}") + $log.info("in_kube_podinventory::watch : inside watch pods! current time: #{Time.now.utc.iso8601}.") if !notice.nil? && !notice.empty? $log.info("in_kube_podinventory::watch : received a notice that is not null and not empty. notice type: #{notice["type"]}") @@ -704,20 +704,23 @@ def merge_updates else fileContents = File.read("testing-podinventory.json") end + $log.info("in_kube_podinventory::merge_updates : file contents read") # $log.info("in_kube_podinventory::merge_updates : file contents read, fileContents: #{fileContents}") @podInventoryHash = JSON.parse(fileContents) - $log.info("in_kube_podinventory::merge_updates : parse successful, received podInventoryHash: #{podInventoryHash}") + $log.info("in_kube_podinventory::merge_updates : parse successful") + # $log.info("in_kube_podinventory::merge_updates : parse successful, received podInventoryHash: #{@podInventoryHash}") rescue => error $log.info("in_kube_podinventory::merge_updates : something went wrong with reading file. #{error.backtrace}") + $log.info("in_kube_podinventory::merge_updates : Reading error: podInventoryHash: #{@podInventoryHash}") end - $log.info("in_kube_podinventory::merge_updates : before noticeHash loop, number of items in hash: #{@noticeHash.size()}, noticeHash: #{@noticeHash}") + $log.info("in_kube_podinventory::merge_updates : before noticeHash loop, number of items in hash: #{@noticeHash.size()}") uidList = [] @mutex.synchronize { @noticeHash.each do |uid, record| - $log.info("in_kube_podinventory::merge_updates : looping through noticeHash, type of notice: #{record["NoticeType"]}. notice uid: #{uid}. notice record: #{record}") + $log.info("in_kube_podinventory::merge_updates : looping through noticeHash, type of notice: #{record["NoticeType"]}") uidList.append(uid) From 10c538d59b3010677b285c34ab0b8b452c0b78bb Mon Sep 17 00:00:00 2001 From: Khushi Chaudhari Date: Sat, 24 Jul 2021 22:30:38 -0700 Subject: [PATCH 29/76] podInventoryHash is an instance variable + added logging for mmap sanity check --- source/plugins/ruby/in_kube_podinventory.rb | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/source/plugins/ruby/in_kube_podinventory.rb b/source/plugins/ruby/in_kube_podinventory.rb index e8b10b194..7bf97e7dd 100644 --- a/source/plugins/ruby/in_kube_podinventory.rb +++ b/source/plugins/ruby/in_kube_podinventory.rb @@ -46,6 +46,7 @@ def initialize @noticeHash = {} @useMmap = false + @podInventoryHash = {} @kubeperfTag = "oneagent.containerInsights.LINUX_PERF_BLOB" @kubeservicesTag = "oneagent.containerInsights.KUBE_SERVICES_BLOB" @@ -124,8 +125,6 @@ def write_to_file(podInventory) batchTime = Time.now.utc.iso8601 #TODO: check if you can pass @serviceRecords into getPodInventoryRecords rather than creating a local copy servRecords= @serviceRecords - - podInventoryHash = {} begin if !podInventory["items"].nil? && !podInventory["items"].empty? @@ -134,11 +133,11 @@ def write_to_file(podInventory) podInventoryRecords = getPodInventoryRecords(item, servRecords, batchTime) podInventoryRecords.each { |record| uid = record["PodUid"] - podInventoryHash[uid] = record + @podInventoryHash[uid] = record } end else - podInventoryHash = podInventory + @podInventoryHash = podInventory end @@ -147,11 +146,15 @@ def write_to_file(podInventory) $log.info("in_kube_podinventory::write_to_file : writing to mmap file case") File.new("testing-podinventory.json", "w") @mmap = Mmap.new("testing-podinventory.json", "rw") - @mmap << JSON.pretty_generate(podInventoryHash) + @mmap << JSON.pretty_generate(@podInventoryHash) + + sanityCheck = "" + sanityCheck << @mmap + $log.info("write_to_file:: sanity check: #{sanityCheck}") else $log.info("in_kube_podinventory::write_to_file : writing to regular file case") File.open("testing-podinventory.json", "w") { |file| - file.write(JSON.pretty_generate(podInventoryHash)) + file.write(JSON.pretty_generate(@podInventoryHash)) } end $log.info("in_kube_podinventory::write_to_file : successfully finished writing to file") @@ -701,6 +704,7 @@ def merge_updates # Read file if @useMmap fileContents << @mmap + $log.info("merge_updates : sanity check : fileContents = #{fileContents}") else fileContents = File.read("testing-podinventory.json") end From 8503ecf06fced07fdd80ee8edf67e610f6cd8336 Mon Sep 17 00:00:00 2001 From: Khushi Chaudhari Date: Sun, 25 Jul 2021 11:36:57 -0700 Subject: [PATCH 30/76] checking if changing rw permissions on file.new fixes reading error --- source/plugins/ruby/in_kube_podinventory.rb | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/source/plugins/ruby/in_kube_podinventory.rb b/source/plugins/ruby/in_kube_podinventory.rb index 7bf97e7dd..67eac786f 100644 --- a/source/plugins/ruby/in_kube_podinventory.rb +++ b/source/plugins/ruby/in_kube_podinventory.rb @@ -144,13 +144,19 @@ def write_to_file(podInventory) # Write to mmap or regular file based on value of @useMmap flag if @useMmap $log.info("in_kube_podinventory::write_to_file : writing to mmap file case") - File.new("testing-podinventory.json", "w") + File.new("testing-podinventory.json", "rw") @mmap = Mmap.new("testing-podinventory.json", "rw") @mmap << JSON.pretty_generate(@podInventoryHash) sanityCheck = "" sanityCheck << @mmap - $log.info("write_to_file:: sanity check: #{sanityCheck}") + + if sanityCheck.empty? + $log.info("write_to_file:: sanity check was empty") + else + $log.info("write_to_file:: sanity check was NOT empty -- good") + end + # $log.info("write_to_file:: sanity check: #{sanityCheck}") else $log.info("in_kube_podinventory::write_to_file : writing to regular file case") File.open("testing-podinventory.json", "w") { |file| From 0e8ae15bff64cf89a846fec8ffecde70596651b0 Mon Sep 17 00:00:00 2001 From: Khushi Chaudhari Date: Sun, 25 Jul 2021 21:49:22 -0700 Subject: [PATCH 31/76] read write error potential fix with moving creation of file to start --- source/plugins/ruby/in_kube_podinventory.rb | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/source/plugins/ruby/in_kube_podinventory.rb b/source/plugins/ruby/in_kube_podinventory.rb index 67eac786f..a73e466dc 100644 --- a/source/plugins/ruby/in_kube_podinventory.rb +++ b/source/plugins/ruby/in_kube_podinventory.rb @@ -45,8 +45,8 @@ def initialize @podsAPIE2ELatencyMs = 0 @noticeHash = {} - @useMmap = false @podInventoryHash = {} + @useMmap = false @kubeperfTag = "oneagent.containerInsights.LINUX_PERF_BLOB" @kubeservicesTag = "oneagent.containerInsights.KUBE_SERVICES_BLOB" @@ -86,6 +86,12 @@ def start @useMmap = true end + #TODO: assumption made regarding mode of new file - might mess up regular file case + File.new("testing-podinventory.json", "w") + if @useMmap + @mmap = Mmap.new("testing-podinventory.json", "rw") + end + $log.info("in_kube_podinventory::start: use mmap is: #{@useMmap}") # create kubernetes watch client @@ -101,6 +107,7 @@ def start @finished = false @condition = ConditionVariable.new @mutex = Mutex.new + $log.info("in_kube:podinventory::start: create thread for watch") @watchthread = Thread.new(&method(:watch)) $log.info("in_kube:podinventory::start: create thread for run_periodic") @@ -144,8 +151,8 @@ def write_to_file(podInventory) # Write to mmap or regular file based on value of @useMmap flag if @useMmap $log.info("in_kube_podinventory::write_to_file : writing to mmap file case") - File.new("testing-podinventory.json", "rw") - @mmap = Mmap.new("testing-podinventory.json", "rw") + # File.new("testing-podinventory.json", "rw") + # @mmap = Mmap.new("testing-podinventory.json", "rw") @mmap << JSON.pretty_generate(@podInventoryHash) sanityCheck = "" @@ -166,8 +173,8 @@ def write_to_file(podInventory) $log.info("in_kube_podinventory::write_to_file : successfully finished writing to file") rescue => exception $log.info("in_kube_podinventory::write_to_file : writing to file failed. backtrace: #{exception.backtrace}") - $log.info("write_to_file:: failed. podInventory: #{podInventory}") - $log.info("write_to_file:: failed. podInventory items: #{podInventory["items"]}") + # $log.info("write_to_file:: failed. podInventory: #{podInventory}") + # $log.info("write_to_file:: failed. podInventory items: #{podInventory["items"]}") end end @@ -710,7 +717,7 @@ def merge_updates # Read file if @useMmap fileContents << @mmap - $log.info("merge_updates : sanity check : fileContents = #{fileContents}") + # $log.info("merge_updates : sanity check : fileContents = #{fileContents}") else fileContents = File.read("testing-podinventory.json") end @@ -721,7 +728,7 @@ def merge_updates # $log.info("in_kube_podinventory::merge_updates : parse successful, received podInventoryHash: #{@podInventoryHash}") rescue => error $log.info("in_kube_podinventory::merge_updates : something went wrong with reading file. #{error.backtrace}") - $log.info("in_kube_podinventory::merge_updates : Reading error: podInventoryHash: #{@podInventoryHash}") + # $log.info("in_kube_podinventory::merge_updates : Reading error: podInventoryHash: #{@podInventoryHash}") end $log.info("in_kube_podinventory::merge_updates : before noticeHash loop, number of items in hash: #{@noticeHash.size()}") From 23af2bbef2d536475ee838a51fcc6610cf2883e7 Mon Sep 17 00:00:00 2001 From: Khushi Chaudhari Date: Sun, 25 Jul 2021 23:43:59 -0700 Subject: [PATCH 32/76] removing pretty gen and converting hash to string when writing to mmap --- source/plugins/ruby/in_kube_podinventory.rb | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/source/plugins/ruby/in_kube_podinventory.rb b/source/plugins/ruby/in_kube_podinventory.rb index a73e466dc..852639279 100644 --- a/source/plugins/ruby/in_kube_podinventory.rb +++ b/source/plugins/ruby/in_kube_podinventory.rb @@ -153,7 +153,8 @@ def write_to_file(podInventory) $log.info("in_kube_podinventory::write_to_file : writing to mmap file case") # File.new("testing-podinventory.json", "rw") # @mmap = Mmap.new("testing-podinventory.json", "rw") - @mmap << JSON.pretty_generate(@podInventoryHash) + # @mmap << JSON.pretty_generate(@podInventoryHash).to_s + @mmap << @podInventoryHash.to_s sanityCheck = "" sanityCheck << @mmap From 78ac1c59cbdc5a66f3c16766ef1443e24cda86d0 Mon Sep 17 00:00:00 2001 From: Khushi Chaudhari Date: Mon, 26 Jul 2021 09:14:25 -0700 Subject: [PATCH 33/76] string might be frozen --- source/plugins/ruby/in_kube_podinventory.rb | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/source/plugins/ruby/in_kube_podinventory.rb b/source/plugins/ruby/in_kube_podinventory.rb index 852639279..35aa06eeb 100644 --- a/source/plugins/ruby/in_kube_podinventory.rb +++ b/source/plugins/ruby/in_kube_podinventory.rb @@ -156,7 +156,12 @@ def write_to_file(podInventory) # @mmap << JSON.pretty_generate(@podInventoryHash).to_s @mmap << @podInventoryHash.to_s + $log.info("write_to_file:: trying to use sanity check to read files") sanityCheck = "" + if @mmap.empty? + $log.info("write_to_file :: sanity check - mmap seems to be empty for some reason") + end + sanityCheck = sanityCheck.dup? if sanityCheck.frozen? sanityCheck << @mmap if sanityCheck.empty? @@ -174,6 +179,7 @@ def write_to_file(podInventory) $log.info("in_kube_podinventory::write_to_file : successfully finished writing to file") rescue => exception $log.info("in_kube_podinventory::write_to_file : writing to file failed. backtrace: #{exception.backtrace}") + $log.info("write_to_file:: writing failed: #{exception}") # $log.info("write_to_file:: failed. podInventory: #{podInventory}") # $log.info("write_to_file:: failed. podInventory items: #{podInventory["items"]}") end @@ -717,6 +723,7 @@ def merge_updates fileContents = "" # Read file if @useMmap + fileContents = fileContents.dup if fileContents.frozen? fileContents << @mmap # $log.info("merge_updates : sanity check : fileContents = #{fileContents}") else @@ -728,7 +735,7 @@ def merge_updates $log.info("in_kube_podinventory::merge_updates : parse successful") # $log.info("in_kube_podinventory::merge_updates : parse successful, received podInventoryHash: #{@podInventoryHash}") rescue => error - $log.info("in_kube_podinventory::merge_updates : something went wrong with reading file. #{error.backtrace}") + $log.info("in_kube_podinventory::merge_updates : something went wrong with reading file. #{error}: #{error.backtrace}") # $log.info("in_kube_podinventory::merge_updates : Reading error: podInventoryHash: #{@podInventoryHash}") end From f45ecfbcc936e2e620c0e2a4205c1f239e13e2ef Mon Sep 17 00:00:00 2001 From: Khushi Chaudhari Date: Mon, 26 Jul 2021 09:42:27 -0700 Subject: [PATCH 34/76] podinvhash to json and removed dup syntax error --- source/plugins/ruby/in_kube_podinventory.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/plugins/ruby/in_kube_podinventory.rb b/source/plugins/ruby/in_kube_podinventory.rb index 35aa06eeb..d2ebb1426 100644 --- a/source/plugins/ruby/in_kube_podinventory.rb +++ b/source/plugins/ruby/in_kube_podinventory.rb @@ -154,14 +154,14 @@ def write_to_file(podInventory) # File.new("testing-podinventory.json", "rw") # @mmap = Mmap.new("testing-podinventory.json", "rw") # @mmap << JSON.pretty_generate(@podInventoryHash).to_s - @mmap << @podInventoryHash.to_s + @mmap << @podInventoryHash.to_json $log.info("write_to_file:: trying to use sanity check to read files") sanityCheck = "" if @mmap.empty? $log.info("write_to_file :: sanity check - mmap seems to be empty for some reason") end - sanityCheck = sanityCheck.dup? if sanityCheck.frozen? + sanityCheck = sanityCheck.dup if sanityCheck.frozen? sanityCheck << @mmap if sanityCheck.empty? From f83f319cacebbefd70f28150fc7f61c3851be56a Mon Sep 17 00:00:00 2001 From: Khushi Chaudhari Date: Mon, 26 Jul 2021 12:04:37 -0700 Subject: [PATCH 35/76] using yajl parser --- source/plugins/ruby/in_kube_podinventory.rb | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/source/plugins/ruby/in_kube_podinventory.rb b/source/plugins/ruby/in_kube_podinventory.rb index d2ebb1426..0401a851a 100644 --- a/source/plugins/ruby/in_kube_podinventory.rb +++ b/source/plugins/ruby/in_kube_podinventory.rb @@ -157,7 +157,7 @@ def write_to_file(podInventory) @mmap << @podInventoryHash.to_json $log.info("write_to_file:: trying to use sanity check to read files") - sanityCheck = "" + sanityCheck = "" if @mmap.empty? $log.info("write_to_file :: sanity check - mmap seems to be empty for some reason") end @@ -731,7 +731,10 @@ def merge_updates end $log.info("in_kube_podinventory::merge_updates : file contents read") # $log.info("in_kube_podinventory::merge_updates : file contents read, fileContents: #{fileContents}") - @podInventoryHash = JSON.parse(fileContents) + # use Yajl Parser + $log.info("merge_updates:: using yajl parser") + @podInventoryHash = Yajl::Parser.parse(fileContents) + # @podInventoryHash = JSON.parse(fileContents) $log.info("in_kube_podinventory::merge_updates : parse successful") # $log.info("in_kube_podinventory::merge_updates : parse successful, received podInventoryHash: #{@podInventoryHash}") rescue => error From 83c804a0a472baaebe02e13cdc271b6d81120f3a Mon Sep 17 00:00:00 2001 From: Khushi Chaudhari Date: Mon, 26 Jul 2021 15:50:04 -0700 Subject: [PATCH 36/76] added logging for size of file --- source/plugins/ruby/in_kube_podinventory.rb | 34 +++------------------ 1 file changed, 4 insertions(+), 30 deletions(-) diff --git a/source/plugins/ruby/in_kube_podinventory.rb b/source/plugins/ruby/in_kube_podinventory.rb index 0401a851a..fa3665b2d 100644 --- a/source/plugins/ruby/in_kube_podinventory.rb +++ b/source/plugins/ruby/in_kube_podinventory.rb @@ -147,41 +147,21 @@ def write_to_file(podInventory) @podInventoryHash = podInventory end - # Write to mmap or regular file based on value of @useMmap flag if @useMmap $log.info("in_kube_podinventory::write_to_file : writing to mmap file case") - # File.new("testing-podinventory.json", "rw") - # @mmap = Mmap.new("testing-podinventory.json", "rw") - # @mmap << JSON.pretty_generate(@podInventoryHash).to_s - @mmap << @podInventoryHash.to_json - - $log.info("write_to_file:: trying to use sanity check to read files") - sanityCheck = "" - if @mmap.empty? - $log.info("write_to_file :: sanity check - mmap seems to be empty for some reason") - end - sanityCheck = sanityCheck.dup if sanityCheck.frozen? - sanityCheck << @mmap - - if sanityCheck.empty? - $log.info("write_to_file:: sanity check was empty") - else - $log.info("write_to_file:: sanity check was NOT empty -- good") - end - # $log.info("write_to_file:: sanity check: #{sanityCheck}") + @mmap << JSON.pretty_generate(@podInventoryHash).to_s else $log.info("in_kube_podinventory::write_to_file : writing to regular file case") File.open("testing-podinventory.json", "w") { |file| file.write(JSON.pretty_generate(@podInventoryHash)) } end + $log.info("in_kube_podinventory::write_to_file : successfully finished writing to file") + $log.info("in_kube_podinventory::write_to_file : size of written file = #{File.size("testing-podinventory.json") / 1000000.0}") rescue => exception - $log.info("in_kube_podinventory::write_to_file : writing to file failed. backtrace: #{exception.backtrace}") - $log.info("write_to_file:: writing failed: #{exception}") - # $log.info("write_to_file:: failed. podInventory: #{podInventory}") - # $log.info("write_to_file:: failed. podInventory items: #{podInventory["items"]}") + $log.info("in_kube_podinventory::write_to_file : writing to file failed. exception: #{exception} backtrace: #{exception.backtrace}") end end @@ -725,21 +705,15 @@ def merge_updates if @useMmap fileContents = fileContents.dup if fileContents.frozen? fileContents << @mmap - # $log.info("merge_updates : sanity check : fileContents = #{fileContents}") else fileContents = File.read("testing-podinventory.json") end $log.info("in_kube_podinventory::merge_updates : file contents read") - # $log.info("in_kube_podinventory::merge_updates : file contents read, fileContents: #{fileContents}") - # use Yajl Parser - $log.info("merge_updates:: using yajl parser") @podInventoryHash = Yajl::Parser.parse(fileContents) # @podInventoryHash = JSON.parse(fileContents) $log.info("in_kube_podinventory::merge_updates : parse successful") - # $log.info("in_kube_podinventory::merge_updates : parse successful, received podInventoryHash: #{@podInventoryHash}") rescue => error $log.info("in_kube_podinventory::merge_updates : something went wrong with reading file. #{error}: #{error.backtrace}") - # $log.info("in_kube_podinventory::merge_updates : Reading error: podInventoryHash: #{@podInventoryHash}") end $log.info("in_kube_podinventory::merge_updates : before noticeHash loop, number of items in hash: #{@noticeHash.size()}") From cc9dc1b287f5b1946f67f16c3b28dd3015065834 Mon Sep 17 00:00:00 2001 From: Khushi Chaudhari Date: Mon, 26 Jul 2021 17:54:06 -0700 Subject: [PATCH 37/76] current version --- source/plugins/ruby/in_kube_podinventory.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/plugins/ruby/in_kube_podinventory.rb b/source/plugins/ruby/in_kube_podinventory.rb index fa3665b2d..355522fbb 100644 --- a/source/plugins/ruby/in_kube_podinventory.rb +++ b/source/plugins/ruby/in_kube_podinventory.rb @@ -745,7 +745,7 @@ def merge_updates @podInventoryHash.delete(uid) $log.info("in_kube_podinventory::merge_updates :: deleted from podInventoryHash") else - $log.info("in_kube_podinventory::merge_updates :: something went wrong and didn't enter any cases for switch") + $log.info("in_kube_podinventory::merge_updates :: something went wrong and didn't enter any cases for switch, notice type was #{record["NoticeType"]}") end # $log.info("in_kube_podinventory::merge_updates :: end of switch") end From 915c7ebb921fa92f157531fa4915dd8bd27859e6 Mon Sep 17 00:00:00 2001 From: Khushi Chaudhari Date: Tue, 27 Jul 2021 11:36:54 -0700 Subject: [PATCH 38/76] changed workspace and location of json file --- .../linux/dockerbuild/existingClusterParam.json | 2 +- source/plugins/ruby/in_kube_podinventory.rb | 12 +++++++----- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/kubernetes/linux/dockerbuild/existingClusterParam.json b/kubernetes/linux/dockerbuild/existingClusterParam.json index 409e82c1e..abf0e0a47 100644 --- a/kubernetes/linux/dockerbuild/existingClusterParam.json +++ b/kubernetes/linux/dockerbuild/existingClusterParam.json @@ -9,7 +9,7 @@ "value": "West US 2" }, "workspaceResourceId": { - "value": "/subscriptions/b4b0cbab-64dc-42bc-9d95-ea7c8d0cf188/resourcegroups/defaultresourcegroup-eus/providers/microsoft.operationalinsights/workspaces/defaultworkspace-b4b0cbab-64dc-42bc-9d95-ea7c8d0cf188-eus" + "value": "/subscriptions/b4b0cbab-64dc-42bc-9d95-ea7c8d0cf188/resourcegroups/defaultresourcegroup-suk/providers/microsoft.operationalinsights/workspaces/defaultworkspace-b4b0cbab-64dc-42bc-9d95-ea7c8d0cf188-suk" } } } \ No newline at end of file diff --git a/source/plugins/ruby/in_kube_podinventory.rb b/source/plugins/ruby/in_kube_podinventory.rb index 355522fbb..a1e9e6ca7 100644 --- a/source/plugins/ruby/in_kube_podinventory.rb +++ b/source/plugins/ruby/in_kube_podinventory.rb @@ -87,9 +87,9 @@ def start end #TODO: assumption made regarding mode of new file - might mess up regular file case - File.new("testing-podinventory.json", "w") + # File.new("testing-podinventory.json", "w") if @useMmap - @mmap = Mmap.new("testing-podinventory.json", "rw") + @mmap = Mmap.new("/var/opt/microsoft/docker-cimprov/log/testing-podinventory.json", "rw") end $log.info("in_kube_podinventory::start: use mmap is: #{@useMmap}") @@ -153,13 +153,13 @@ def write_to_file(podInventory) @mmap << JSON.pretty_generate(@podInventoryHash).to_s else $log.info("in_kube_podinventory::write_to_file : writing to regular file case") - File.open("testing-podinventory.json", "w") { |file| + File.open("/var/opt/microsoft/docker-cimprov/log/testing-podinventory.json", "w") { |file| file.write(JSON.pretty_generate(@podInventoryHash)) } end $log.info("in_kube_podinventory::write_to_file : successfully finished writing to file") - $log.info("in_kube_podinventory::write_to_file : size of written file = #{File.size("testing-podinventory.json") / 1000000.0}") + $log.info("in_kube_podinventory::write_to_file : size of written file = #{File.size("/var/opt/microsoft/docker-cimprov/log/testing-podinventory.json") / 1000000.0}") rescue => exception $log.info("in_kube_podinventory::write_to_file : writing to file failed. exception: #{exception} backtrace: #{exception.backtrace}") end @@ -706,7 +706,7 @@ def merge_updates fileContents = fileContents.dup if fileContents.frozen? fileContents << @mmap else - fileContents = File.read("testing-podinventory.json") + fileContents = File.read("/var/opt/microsoft/docker-cimprov/log/testing-podinventory.json") end $log.info("in_kube_podinventory::merge_updates : file contents read") @podInventoryHash = Yajl::Parser.parse(fileContents) @@ -761,7 +761,9 @@ def merge_updates $log.info("in_kube_podinventory:: merge_updates : about to replace entire contents of testing-podinventory.json") if (!@podInventoryHash.nil? && !@podInventoryHash.empty?) $log.info("in_kube_podinventory:: merge_updates : podInventoryHash not null and not empty, will write to file") + # only write if there is a change write_to_file(@podInventoryHash) + parse_and_emit_merge_updates(@podInventoryHash) else $log.info("in_kube_podinventory:: merge_updates : podInventoryHash was either null or empty, so NOT writing to file - should never be in this case") From 256e15de3cb1ba007ad2467247f91f1dfbf87ac4 Mon Sep 17 00:00:00 2001 From: Khushi Chaudhari Date: Tue, 27 Jul 2021 14:04:59 -0700 Subject: [PATCH 39/76] writing to file now json rather than hash --- source/plugins/ruby/in_kube_podinventory.rb | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/source/plugins/ruby/in_kube_podinventory.rb b/source/plugins/ruby/in_kube_podinventory.rb index a1e9e6ca7..6a656246c 100644 --- a/source/plugins/ruby/in_kube_podinventory.rb +++ b/source/plugins/ruby/in_kube_podinventory.rb @@ -150,11 +150,11 @@ def write_to_file(podInventory) # Write to mmap or regular file based on value of @useMmap flag if @useMmap $log.info("in_kube_podinventory::write_to_file : writing to mmap file case") - @mmap << JSON.pretty_generate(@podInventoryHash).to_s + @mmap << JSON.pretty_generate(@podInventoryHash.to_json).to_s else $log.info("in_kube_podinventory::write_to_file : writing to regular file case") File.open("/var/opt/microsoft/docker-cimprov/log/testing-podinventory.json", "w") { |file| - file.write(JSON.pretty_generate(@podInventoryHash)) + file.write(JSON.pretty_generate(@podInventoryHash.to_json)) } end @@ -405,6 +405,7 @@ def enumerate(podList = nil) end def parse_and_emit_records(podInventory, serviceRecords, continuationToken, batchTime = Time.utc.iso8601) + $log.info("parse_and_emit_records:: podInventory: #{podInventory}") currentTime = Time.now emitTime = Fluent::Engine.now #batchTime = currentTime.utc.iso8601 @@ -582,6 +583,8 @@ def parse_and_emit_records(podInventory, serviceRecords, continuationToken, batc end def parse_and_emit_merge_updates(podInventoryRecords) + $log.info("parse_and_emit_merge_updates:: podInventory: #{podInventoryRecords}") + currentTime = Time.now emitTime = Fluent::Engine.now batchTime = currentTime.utc.iso8601 From 3e346d7804954dd696c548c69bef5b35e9f80de7 Mon Sep 17 00:00:00 2001 From: Khushi Chaudhari Date: Tue, 27 Jul 2021 17:09:59 -0700 Subject: [PATCH 40/76] commented out merge logic to check if emitting works --- source/plugins/ruby/in_kube_podinventory.rb | 157 ++++++-------------- 1 file changed, 46 insertions(+), 111 deletions(-) diff --git a/source/plugins/ruby/in_kube_podinventory.rb b/source/plugins/ruby/in_kube_podinventory.rb index 6a656246c..2ad6e4ed1 100644 --- a/source/plugins/ruby/in_kube_podinventory.rb +++ b/source/plugins/ruby/in_kube_podinventory.rb @@ -405,7 +405,7 @@ def enumerate(podList = nil) end def parse_and_emit_records(podInventory, serviceRecords, continuationToken, batchTime = Time.utc.iso8601) - $log.info("parse_and_emit_records:: podInventory: #{podInventory}") + # $log.info("parse_and_emit_records:: podInventory: #{podInventory}") currentTime = Time.now emitTime = Fluent::Engine.now #batchTime = currentTime.utc.iso8601 @@ -583,7 +583,7 @@ def parse_and_emit_records(podInventory, serviceRecords, continuationToken, batc end def parse_and_emit_merge_updates(podInventoryRecords) - $log.info("parse_and_emit_merge_updates:: podInventory: #{podInventoryRecords}") + # $log.info("parse_and_emit_merge_updates:: podInventory: #{podInventoryRecords}") currentTime = Time.now emitTime = Fluent::Engine.now @@ -624,76 +624,6 @@ def parse_and_emit_merge_updates(podInventoryRecords) end eventStream = nil end - - if containerInventoryStream.count > 0 - $log.info("in_kube_podinventory::parse_and_emit_merge_updates: number of windows container inventory records emitted #{containerInventoryStream.count} @ #{Time.now.utc.iso8601}") - router.emit_stream(@containerInventoryTag, containerInventoryStream) if containerInventoryStream - if (!@@istestvar.nil? && !@@istestvar.empty? && @@istestvar.casecmp("true") == 0) - $log.info("kubeWindowsContainerInventoryEmitStreamSuccess @ #{Time.now.utc.iso8601}") - end - containerInventoryStream = nil - end - - if kubePerfEventStream.count > 0 - $log.info("in_kube_podinventory::parse_and_emit_merge_updates: number of perf records emitted #{kubePerfEventStream.count} @ #{Time.now.utc.iso8601}") - router.emit_stream(@kubeperfTag, kubePerfEventStream) if kubePerfEventStream - kubePerfEventStream = nil - if (!@@istestvar.nil? && !@@istestvar.empty? && @@istestvar.casecmp("true") == 0) - $log.info("kubeContainerPerfEventEmitStreamSuccess @ #{Time.now.utc.iso8601}") - end - end - - if insightsMetricsEventStream.count > 0 - $log.info("in_kube_podinventory::parse_and_emit_merge_updates: number of insights metrics records emitted #{insightsMetricsEventStream.count} @ #{Time.now.utc.iso8601}") - router.emit_stream(@insightsMetricsTag, insightsMetricsEventStream) if insightsMetricsEventStream - if (!@@istestvar.nil? && !@@istestvar.empty? && @@istestvar.casecmp("true") == 0) - $log.info("kubePodInsightsMetricsEmitStreamSuccess @ #{Time.now.utc.iso8601}") - end - insightsMetricsEventStream = nil - end - - if continuationToken.nil? #no more chunks in this batch to be sent, get all mdm pod inventory records to send - @log.info "Sending pod inventory mdm records to out_mdm" - pod_inventory_mdm_records = @inventoryToMdmConvertor.get_pod_inventory_mdm_records(batchTime) - @log.info "pod_inventory_mdm_records.size #{pod_inventory_mdm_records.size}" - mdm_pod_inventory_es = Fluent::MultiEventStream.new - pod_inventory_mdm_records.each { |pod_inventory_mdm_record| - mdm_pod_inventory_es.add(batchTime, pod_inventory_mdm_record) if pod_inventory_mdm_record - } if pod_inventory_mdm_records - router.emit_stream(@@MDMKubePodInventoryTag, mdm_pod_inventory_es) if mdm_pod_inventory_es - end - - if continuationToken.nil? # sending kube services inventory records - kubeServicesEventStream = Fluent::MultiEventStream.new - # serviceRecords.each do |kubeServiceRecord| - # if !kubeServiceRecord.nil? - # # adding before emit to reduce memory foot print - # kubeServiceRecord["ClusterId"] = KubernetesApiClient.getClusterId - # kubeServiceRecord["ClusterName"] = KubernetesApiClient.getClusterName - # kubeServicesEventStream.add(emitTime, kubeServiceRecord) if kubeServiceRecord - # if @PODS_EMIT_STREAM_BATCH_SIZE > 0 && kubeServicesEventStream.count >= @PODS_EMIT_STREAM_BATCH_SIZE - # $log.info("in_kube_podinventory::parse_and_emit_merge_updates: number of service records emitted #{@PODS_EMIT_STREAM_BATCH_SIZE} @ #{Time.now.utc.iso8601}") - # router.emit_stream(@kubeservicesTag, kubeServicesEventStream) if kubeServicesEventStream - # kubeServicesEventStream = Fluent::MultiEventStream.new - # if (!@@istestvar.nil? && !@@istestvar.empty? && @@istestvar.casecmp("true") == 0) - # $log.info("kubeServicesEventEmitStreamSuccess @ #{Time.now.utc.iso8601}") - # end - # end - # end - # end - - if kubeServicesEventStream.count > 0 - $log.info("in_kube_podinventory::parse_and_emit_merge_updates : number of service records emitted #{kubeServicesEventStream.count} @ #{Time.now.utc.iso8601}") - router.emit_stream(@kubeservicesTag, kubeServicesEventStream) if kubeServicesEventStream - if (!@@istestvar.nil? && !@@istestvar.empty? && @@istestvar.casecmp("true") == 0) - $log.info("kubeServicesEventEmitStreamSuccess @ #{Time.now.utc.iso8601}") - end - end - kubeServicesEventStream = nil - end - - #Updating value for AppInsights telemetry - # @podCount += podInventory["items"].length rescue => errorStr $log.warn "Failed in parse_and_emit_merge_updates pod inventory: #{errorStr}" $log.debug_backtrace(errorStr.backtrace) @@ -712,9 +642,10 @@ def merge_updates fileContents = File.read("/var/opt/microsoft/docker-cimprov/log/testing-podinventory.json") end $log.info("in_kube_podinventory::merge_updates : file contents read") - @podInventoryHash = Yajl::Parser.parse(fileContents) - # @podInventoryHash = JSON.parse(fileContents) - $log.info("in_kube_podinventory::merge_updates : parse successful") + if !fileContents.empty? + @podInventoryHash = Yajl::Parser.parse(fileContents).to_hash + $log.info("in_kube_podinventory::merge_updates : parse successful.") + end rescue => error $log.info("in_kube_podinventory::merge_updates : something went wrong with reading file. #{error}: #{error.backtrace}") end @@ -723,42 +654,44 @@ def merge_updates uidList = [] - @mutex.synchronize { - @noticeHash.each do |uid, record| - $log.info("in_kube_podinventory::merge_updates : looping through noticeHash, type of notice: #{record["NoticeType"]}") - - uidList.append(uid) - - case record["NoticeType"] - when "ADDED" - @podInventoryHash[uid] = record - $log.info("in_kube_podinventory::merge_updates : added new record to podInventoryHash") - when "MODIFIED" - if @podInventoryHash[uid].nil? - $log.info("in_kube_podinventory::merge_updates : modify case where uid for add was overwritten to modify within same minute") - @podInventoryHash[uid] = record - else - $log.info("in_kube_podinventory::merge_updates : modify case where it is only a modification. old status: #{@podInventoryHash[uid]["PodStatus"]}. new status: #{record["PodStatus"]}") - val = @podInventoryHash[uid] - val["PodStatus"] = record["PodStatus"] - @podInventoryHash[uid] = val - end - $log.info("in_kube_podinventory::merge_updates :: modified and changes reflected in podInventoryHash") - when "DELETED" - @podInventoryHash.delete(uid) - $log.info("in_kube_podinventory::merge_updates :: deleted from podInventoryHash") - else - $log.info("in_kube_podinventory::merge_updates :: something went wrong and didn't enter any cases for switch, notice type was #{record["NoticeType"]}") - end - # $log.info("in_kube_podinventory::merge_updates :: end of switch") - end - - # remove all looked at uids from the noticeHash - uidList.each do |uid| - @noticeHash.delete(uid) - end - $log.info("in_kube_podinventory::merge_updates :: removed all visited uids from noticeHash") - } + # @mutex.synchronize { + # @noticeHash.each do |uid, record| + # $log.info("in_kube_podinventory::merge_updates : looping through noticeHash, type of notice: #{record["NoticeType"]}") + + # uidList.append(uid) + + # case record["NoticeType"] + # when "ADDED" + # @podInventoryHash[uid] = record + # $log.info("in_kube_podinventory::merge_updates : added new record to podInventoryHash") + # when "MODIFIED" + # if @podInventoryHash[uid].nil? + # $log.info("in_kube_podinventory::merge_updates : modify case where uid for add was overwritten to modify within same minute") + # @podInventoryHash[uid] = record + # else + # $log.info("in_kube_podinventory::merge_updates : modify case where it is only a modification. old status: #{@podInventoryHash[uid]["PodStatus"]}. new status: #{record["PodStatus"]}") + # val = @podInventoryHash[uid] + # val["PodStatus"] = record["PodStatus"] + # @podInventoryHash[uid] = val + # end + # $log.info("in_kube_podinventory::merge_updates :: modified and changes reflected in podInventoryHash") + # when "DELETED" + # @podInventoryHash.delete(uid) + # $log.info("in_kube_podinventory::merge_updates :: deleted from podInventoryHash") + # else + # $log.info("in_kube_podinventory::merge_updates :: something went wrong and didn't enter any cases for switch, notice type was #{record["NoticeType"]}") + # end + # # $log.info("in_kube_podinventory::merge_updates :: end of switch") + # end + + # # remove all looked at uids from the noticeHash + # uidList.each do |uid| + # @noticeHash.delete(uid) + # end + # # copy noticeHash to tempHash and use tempHash to loop through so we dont lock on it for a long time + + # $log.info("in_kube_podinventory::merge_updates :: removed all visited uids from noticeHash") + # } #TODO: Look for a way to replace only necessary contents, rather than everything $log.info("in_kube_podinventory:: merge_updates : about to replace entire contents of testing-podinventory.json") @@ -766,8 +699,10 @@ def merge_updates $log.info("in_kube_podinventory:: merge_updates : podInventoryHash not null and not empty, will write to file") # only write if there is a change write_to_file(@podInventoryHash) + $log.info("merge_updates:: number of items in podInventoryHash: #{@podInventoryHash.length}") parse_and_emit_merge_updates(@podInventoryHash) + @podInventoryHash = {} else $log.info("in_kube_podinventory:: merge_updates : podInventoryHash was either null or empty, so NOT writing to file - should never be in this case") end From 5515822725bc116eee5f7bbe0d23828aa34246f5 Mon Sep 17 00:00:00 2001 From: Khushi Chaudhari Date: Tue, 27 Jul 2021 18:12:54 -0700 Subject: [PATCH 41/76] removed to_hash function --- source/plugins/ruby/in_kube_podinventory.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/plugins/ruby/in_kube_podinventory.rb b/source/plugins/ruby/in_kube_podinventory.rb index 2ad6e4ed1..c10cc489c 100644 --- a/source/plugins/ruby/in_kube_podinventory.rb +++ b/source/plugins/ruby/in_kube_podinventory.rb @@ -643,7 +643,7 @@ def merge_updates end $log.info("in_kube_podinventory::merge_updates : file contents read") if !fileContents.empty? - @podInventoryHash = Yajl::Parser.parse(fileContents).to_hash + @podInventoryHash = Yajl::Parser.parse(fileContents) $log.info("in_kube_podinventory::merge_updates : parse successful.") end rescue => error From a04d4d2ca4d1b2b08c637e2092633e8e201a94d5 Mon Sep 17 00:00:00 2001 From: Khushi Chaudhari Date: Tue, 27 Jul 2021 21:37:51 -0700 Subject: [PATCH 42/76] debugging parse and emit merge updates for failing each on string --- source/plugins/ruby/in_kube_podinventory.rb | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/source/plugins/ruby/in_kube_podinventory.rb b/source/plugins/ruby/in_kube_podinventory.rb index c10cc489c..ebb817f73 100644 --- a/source/plugins/ruby/in_kube_podinventory.rb +++ b/source/plugins/ruby/in_kube_podinventory.rb @@ -150,16 +150,16 @@ def write_to_file(podInventory) # Write to mmap or regular file based on value of @useMmap flag if @useMmap $log.info("in_kube_podinventory::write_to_file : writing to mmap file case") - @mmap << JSON.pretty_generate(@podInventoryHash.to_json).to_s + @mmap << JSON.pretty_generate(@podInventoryHash).to_s else $log.info("in_kube_podinventory::write_to_file : writing to regular file case") File.open("/var/opt/microsoft/docker-cimprov/log/testing-podinventory.json", "w") { |file| - file.write(JSON.pretty_generate(@podInventoryHash.to_json)) + file.write(JSON.pretty_generate(@podInventoryHash)) } end $log.info("in_kube_podinventory::write_to_file : successfully finished writing to file") - $log.info("in_kube_podinventory::write_to_file : size of written file = #{File.size("/var/opt/microsoft/docker-cimprov/log/testing-podinventory.json") / 1000000.0}") + $log.info("in_kube_podinventory::write_to_file : size of written file = #{File.size("/var/opt/microsoft/docker-cimprov/log/testing-podinventory.json") / 1000000.0} MB") rescue => exception $log.info("in_kube_podinventory::write_to_file : writing to file failed. exception: #{exception} backtrace: #{exception.backtrace}") end @@ -600,8 +600,11 @@ def parse_and_emit_merge_updates(podInventoryRecords) # Getting windows nodes from kubeapi winNodes = KubernetesApiClient.getWindowsNodesArray + $log.info("parse_and_emit_merge_updates:: podInventoryRecords: #{podInventoryRecords}") podInventoryRecords.each do |uid, record| + $log.info("parse_and_emit_merge_updates:: inside each method. uid: #{uid}. record: #{record}") if !record.nil? + $log.info("parse_and_emit_merge_updates:: record is not null.") eventStream.add(emitTime, record) if record @inventoryToMdmConvertor.process_pod_inventory_record(record) end @@ -625,7 +628,7 @@ def parse_and_emit_merge_updates(podInventoryRecords) eventStream = nil end rescue => errorStr - $log.warn "Failed in parse_and_emit_merge_updates pod inventory: #{errorStr}" + $log.warn "Failed in parse_and_emit_merge_updates pod inventory: #{errorStr}. backtrace: #{errorStr.backtrace}" $log.debug_backtrace(errorStr.backtrace) ApplicationInsightsUtility.sendExceptionTelemetry(errorStr) end #begin block end @@ -644,7 +647,7 @@ def merge_updates $log.info("in_kube_podinventory::merge_updates : file contents read") if !fileContents.empty? @podInventoryHash = Yajl::Parser.parse(fileContents) - $log.info("in_kube_podinventory::merge_updates : parse successful.") + $log.info("in_kube_podinventory::merge_updates : parse successful. size of hash: #{@podInventoryHash.size()} podInventoryHash: #{@podInventoryHash}") end rescue => error $log.info("in_kube_podinventory::merge_updates : something went wrong with reading file. #{error}: #{error.backtrace}") @@ -699,8 +702,7 @@ def merge_updates $log.info("in_kube_podinventory:: merge_updates : podInventoryHash not null and not empty, will write to file") # only write if there is a change write_to_file(@podInventoryHash) - $log.info("merge_updates:: number of items in podInventoryHash: #{@podInventoryHash.length}") - + $log.info("merge_updates:: number of items in podInventoryHash: #{@podInventoryHash.length}. podInventoryHash: #{@podInventoryHash}") parse_and_emit_merge_updates(@podInventoryHash) @podInventoryHash = {} else From 96c360decf98c1e2ea6556db9e272ec5a32cfed6 Mon Sep 17 00:00:00 2001 From: Khushi Chaudhari Date: Tue, 27 Jul 2021 23:00:11 -0700 Subject: [PATCH 43/76] adding back merging logic w mutex --- source/plugins/ruby/in_kube_podinventory.rb | 82 +++++++++++---------- 1 file changed, 42 insertions(+), 40 deletions(-) diff --git a/source/plugins/ruby/in_kube_podinventory.rb b/source/plugins/ruby/in_kube_podinventory.rb index ebb817f73..e13912f03 100644 --- a/source/plugins/ruby/in_kube_podinventory.rb +++ b/source/plugins/ruby/in_kube_podinventory.rb @@ -144,7 +144,9 @@ def write_to_file(podInventory) } end else + $log.info("write_to_file:: else case coming from merge. podInventory: #{podInventory}") @podInventoryHash = podInventory + $log.info("write_to_file:: else case: size: #{podInventory.size()}") end # Write to mmap or regular file based on value of @useMmap flag @@ -602,7 +604,7 @@ def parse_and_emit_merge_updates(podInventoryRecords) $log.info("parse_and_emit_merge_updates:: podInventoryRecords: #{podInventoryRecords}") podInventoryRecords.each do |uid, record| - $log.info("parse_and_emit_merge_updates:: inside each method. uid: #{uid}. record: #{record}") + # $log.info("parse_and_emit_merge_updates:: inside each method. uid: #{uid}. record: #{record}") if !record.nil? $log.info("parse_and_emit_merge_updates:: record is not null.") eventStream.add(emitTime, record) if record @@ -657,44 +659,44 @@ def merge_updates uidList = [] - # @mutex.synchronize { - # @noticeHash.each do |uid, record| - # $log.info("in_kube_podinventory::merge_updates : looping through noticeHash, type of notice: #{record["NoticeType"]}") - - # uidList.append(uid) - - # case record["NoticeType"] - # when "ADDED" - # @podInventoryHash[uid] = record - # $log.info("in_kube_podinventory::merge_updates : added new record to podInventoryHash") - # when "MODIFIED" - # if @podInventoryHash[uid].nil? - # $log.info("in_kube_podinventory::merge_updates : modify case where uid for add was overwritten to modify within same minute") - # @podInventoryHash[uid] = record - # else - # $log.info("in_kube_podinventory::merge_updates : modify case where it is only a modification. old status: #{@podInventoryHash[uid]["PodStatus"]}. new status: #{record["PodStatus"]}") - # val = @podInventoryHash[uid] - # val["PodStatus"] = record["PodStatus"] - # @podInventoryHash[uid] = val - # end - # $log.info("in_kube_podinventory::merge_updates :: modified and changes reflected in podInventoryHash") - # when "DELETED" - # @podInventoryHash.delete(uid) - # $log.info("in_kube_podinventory::merge_updates :: deleted from podInventoryHash") - # else - # $log.info("in_kube_podinventory::merge_updates :: something went wrong and didn't enter any cases for switch, notice type was #{record["NoticeType"]}") - # end - # # $log.info("in_kube_podinventory::merge_updates :: end of switch") - # end - - # # remove all looked at uids from the noticeHash - # uidList.each do |uid| - # @noticeHash.delete(uid) - # end - # # copy noticeHash to tempHash and use tempHash to loop through so we dont lock on it for a long time - - # $log.info("in_kube_podinventory::merge_updates :: removed all visited uids from noticeHash") - # } + @mutex.synchronize { + @noticeHash.each do |uid, record| + $log.info("in_kube_podinventory::merge_updates : looping through noticeHash, type of notice: #{record["NoticeType"]}") + + uidList.append(uid) + + case record["NoticeType"] + when "ADDED" + @podInventoryHash[uid] = record + $log.info("in_kube_podinventory::merge_updates : added new record to podInventoryHash") + when "MODIFIED" + if @podInventoryHash[uid].nil? + $log.info("in_kube_podinventory::merge_updates : modify case where uid for add was overwritten to modify within same minute") + @podInventoryHash[uid] = record + else + $log.info("in_kube_podinventory::merge_updates : modify case where it is only a modification. old status: #{@podInventoryHash[uid]["PodStatus"]}. new status: #{record["PodStatus"]}") + val = @podInventoryHash[uid] + val["PodStatus"] = record["PodStatus"] + @podInventoryHash[uid] = val + end + $log.info("in_kube_podinventory::merge_updates :: modified and changes reflected in podInventoryHash") + when "DELETED" + @podInventoryHash.delete(uid) + $log.info("in_kube_podinventory::merge_updates :: deleted from podInventoryHash") + else + $log.info("in_kube_podinventory::merge_updates :: something went wrong and didn't enter any cases for switch, notice type was #{record["NoticeType"]}") + end + # $log.info("in_kube_podinventory::merge_updates :: end of switch") + end + + # remove all looked at uids from the noticeHash + uidList.each do |uid| + @noticeHash.delete(uid) + end + # copy noticeHash to tempHash and use tempHash to loop through so we dont lock on it for a long time + + $log.info("in_kube_podinventory::merge_updates :: removed all visited uids from noticeHash") + } #TODO: Look for a way to replace only necessary contents, rather than everything $log.info("in_kube_podinventory:: merge_updates : about to replace entire contents of testing-podinventory.json") @@ -704,7 +706,7 @@ def merge_updates write_to_file(@podInventoryHash) $log.info("merge_updates:: number of items in podInventoryHash: #{@podInventoryHash.length}. podInventoryHash: #{@podInventoryHash}") parse_and_emit_merge_updates(@podInventoryHash) - @podInventoryHash = {} + @podInventoryHash.clear else $log.info("in_kube_podinventory:: merge_updates : podInventoryHash was either null or empty, so NOT writing to file - should never be in this case") end From 2e900a84698d6e0374075d8d34d20110488a9c85 Mon Sep 17 00:00:00 2001 From: Khushi Chaudhari Date: Wed, 28 Jul 2021 01:29:50 -0700 Subject: [PATCH 44/76] added fix for multiple json object error with mmap read --- source/plugins/ruby/in_kube_podinventory.rb | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/source/plugins/ruby/in_kube_podinventory.rb b/source/plugins/ruby/in_kube_podinventory.rb index e13912f03..09dd9b8dc 100644 --- a/source/plugins/ruby/in_kube_podinventory.rb +++ b/source/plugins/ruby/in_kube_podinventory.rb @@ -88,9 +88,9 @@ def start #TODO: assumption made regarding mode of new file - might mess up regular file case # File.new("testing-podinventory.json", "w") - if @useMmap - @mmap = Mmap.new("/var/opt/microsoft/docker-cimprov/log/testing-podinventory.json", "rw") - end + # if @useMmap + # @mmap = Mmap.new("/var/opt/microsoft/docker-cimprov/log/testing-podinventory.json", "rw") + # end $log.info("in_kube_podinventory::start: use mmap is: #{@useMmap}") @@ -151,6 +151,10 @@ def write_to_file(podInventory) # Write to mmap or regular file based on value of @useMmap flag if @useMmap + $log.info("in_kube_podinventory::write_to_file : creating new mmap file.") + File.new("/var/opt/microsoft/docker-cimprov/log/testing-podinventory.json") + @mmap = Mmap.new("/var/opt/microsoft/docker-cimprov/log/testing-podinventory.json", "rw") + $log.info("in_kube_podinventory::write_to_file : writing to mmap file case") @mmap << JSON.pretty_generate(@podInventoryHash).to_s else From 0ae8a984547654be246360fb9ecb23f7042d6ae2 Mon Sep 17 00:00:00 2001 From: Khushi Chaudhari Date: Wed, 28 Jul 2021 02:08:13 -0700 Subject: [PATCH 45/76] debugging why mmap error still occurs --- source/plugins/ruby/in_kube_podinventory.rb | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/source/plugins/ruby/in_kube_podinventory.rb b/source/plugins/ruby/in_kube_podinventory.rb index 09dd9b8dc..381184f8c 100644 --- a/source/plugins/ruby/in_kube_podinventory.rb +++ b/source/plugins/ruby/in_kube_podinventory.rb @@ -156,7 +156,14 @@ def write_to_file(podInventory) @mmap = Mmap.new("/var/opt/microsoft/docker-cimprov/log/testing-podinventory.json", "rw") $log.info("in_kube_podinventory::write_to_file : writing to mmap file case") + $log.info("write_to_file:: podInventoryHash: #{@podInventoryHash}") @mmap << JSON.pretty_generate(@podInventoryHash).to_s + + sanityCheck = "" + sanityCheck = sanityCheck.dup if sanityCheck.frozen? + sanityCheck << @mmap + + $log.info("write_to_file:: sanity check: #{sanityCheck}") else $log.info("in_kube_podinventory::write_to_file : writing to regular file case") File.open("/var/opt/microsoft/docker-cimprov/log/testing-podinventory.json", "w") { |file| @@ -652,6 +659,7 @@ def merge_updates end $log.info("in_kube_podinventory::merge_updates : file contents read") if !fileContents.empty? + $log.info("merge_updates:: file contents before parsing: #{fileContents}") @podInventoryHash = Yajl::Parser.parse(fileContents) $log.info("in_kube_podinventory::merge_updates : parse successful. size of hash: #{@podInventoryHash.size()} podInventoryHash: #{@podInventoryHash}") end From 4cf149d641ba779f7dd9df0bd54f65baaf292ac0 Mon Sep 17 00:00:00 2001 From: Khushi Chaudhari Date: Wed, 28 Jul 2021 09:27:14 -0700 Subject: [PATCH 46/76] fixed mmap issue by changing file.new to file.open --- source/plugins/ruby/in_kube_podinventory.rb | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/source/plugins/ruby/in_kube_podinventory.rb b/source/plugins/ruby/in_kube_podinventory.rb index 381184f8c..17bcfc8ab 100644 --- a/source/plugins/ruby/in_kube_podinventory.rb +++ b/source/plugins/ruby/in_kube_podinventory.rb @@ -152,7 +152,8 @@ def write_to_file(podInventory) # Write to mmap or regular file based on value of @useMmap flag if @useMmap $log.info("in_kube_podinventory::write_to_file : creating new mmap file.") - File.new("/var/opt/microsoft/docker-cimprov/log/testing-podinventory.json") + # this is to ensure that we clear file contents before writing to file, check if there is a better way to do this + File.open("/var/opt/microsoft/docker-cimprov/log/testing-podinventory.json", "w") @mmap = Mmap.new("/var/opt/microsoft/docker-cimprov/log/testing-podinventory.json", "rw") $log.info("in_kube_podinventory::write_to_file : writing to mmap file case") From dd177c16580513a2e00ce06e0cfe7b124ad83271 Mon Sep 17 00:00:00 2001 From: Khushi Chaudhari Date: Wed, 28 Jul 2021 11:22:52 -0700 Subject: [PATCH 47/76] update record collection time with batch time --- source/plugins/ruby/in_kube_podinventory.rb | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/source/plugins/ruby/in_kube_podinventory.rb b/source/plugins/ruby/in_kube_podinventory.rb index 17bcfc8ab..211162eb6 100644 --- a/source/plugins/ruby/in_kube_podinventory.rb +++ b/source/plugins/ruby/in_kube_podinventory.rb @@ -136,7 +136,7 @@ def write_to_file(podInventory) begin if !podInventory["items"].nil? && !podInventory["items"].empty? podInventory["items"].each do |item| - # Extract needed fields using getPodInventoryRecords and create a hash mapping uid -> record + # Extract needed fields using getPodInventoryRecords and create a hash mapping uid -> record podInventoryRecords = getPodInventoryRecords(item, servRecords, batchTime) podInventoryRecords.each { |record| uid = record["PodUid"] @@ -618,7 +618,10 @@ def parse_and_emit_merge_updates(podInventoryRecords) podInventoryRecords.each do |uid, record| # $log.info("parse_and_emit_merge_updates:: inside each method. uid: #{uid}. record: #{record}") if !record.nil? - $log.info("parse_and_emit_merge_updates:: record is not null.") + $log.info("parse_and_emit_merge_updates:: emitTime: #{emitTime}. batchTime: #{batchTime}. record collection time: #{record["CollectionTime"]}") + # $log.info("parse_and_emit_merge_updates:: record is not null.") + record["CollectionTime"] = batchTime + $log.info("parse_and_emit_merge_updates:: record collection time: #{record["CollectionTime"]}") eventStream.add(emitTime, record) if record @inventoryToMdmConvertor.process_pod_inventory_record(record) end From c2be217873ebf8487a1f1507786d2a5392ccd321 Mon Sep 17 00:00:00 2001 From: Khushi Chaudhari Date: Thu, 29 Jul 2021 15:41:40 -0700 Subject: [PATCH 48/76] fixed logging -- checking if watch catches notifications --- .../dockerbuild/existingClusterParam.json | 2 +- source/plugins/ruby/in_kube_podinventory.rb | 49 +++++-------------- 2 files changed, 12 insertions(+), 39 deletions(-) diff --git a/kubernetes/linux/dockerbuild/existingClusterParam.json b/kubernetes/linux/dockerbuild/existingClusterParam.json index abf0e0a47..6fdcc601e 100644 --- a/kubernetes/linux/dockerbuild/existingClusterParam.json +++ b/kubernetes/linux/dockerbuild/existingClusterParam.json @@ -9,7 +9,7 @@ "value": "West US 2" }, "workspaceResourceId": { - "value": "/subscriptions/b4b0cbab-64dc-42bc-9d95-ea7c8d0cf188/resourcegroups/defaultresourcegroup-suk/providers/microsoft.operationalinsights/workspaces/defaultworkspace-b4b0cbab-64dc-42bc-9d95-ea7c8d0cf188-suk" + "value": "/subscriptions/b4b0cbab-64dc-42bc-9d95-ea7c8d0cf188/resourcegroups/defaultresourcegroup-scus/providers/microsoft.operationalinsights/workspaces/defaultworkspace-b4b0cbab-64dc-42bc-9d95-ea7c8d0cf188-scus" } } } \ No newline at end of file diff --git a/source/plugins/ruby/in_kube_podinventory.rb b/source/plugins/ruby/in_kube_podinventory.rb index 211162eb6..e64fd9f93 100644 --- a/source/plugins/ruby/in_kube_podinventory.rb +++ b/source/plugins/ruby/in_kube_podinventory.rb @@ -86,12 +86,6 @@ def start @useMmap = true end - #TODO: assumption made regarding mode of new file - might mess up regular file case - # File.new("testing-podinventory.json", "w") - # if @useMmap - # @mmap = Mmap.new("/var/opt/microsoft/docker-cimprov/log/testing-podinventory.json", "rw") - # end - $log.info("in_kube_podinventory::start: use mmap is: #{@useMmap}") # create kubernetes watch client @@ -107,10 +101,7 @@ def start @finished = false @condition = ConditionVariable.new @mutex = Mutex.new - - $log.info("in_kube:podinventory::start: create thread for watch") @watchthread = Thread.new(&method(:watch)) - $log.info("in_kube:podinventory::start: create thread for run_periodic") @thread = Thread.new(&method(:run_periodic)) @@podTelemetryTimeTracker = DateTime.now.to_time.to_i end @@ -131,40 +122,30 @@ def shutdown def write_to_file(podInventory) batchTime = Time.now.utc.iso8601 #TODO: check if you can pass @serviceRecords into getPodInventoryRecords rather than creating a local copy - servRecords= @serviceRecords + #TODO: if no, update code so we simply use instance variable everywhere and there is no need to pass serviceRecords + serviceRecords= @serviceRecords begin if !podInventory["items"].nil? && !podInventory["items"].empty? podInventory["items"].each do |item| # Extract needed fields using getPodInventoryRecords and create a hash mapping uid -> record - podInventoryRecords = getPodInventoryRecords(item, servRecords, batchTime) + podInventoryRecords = getPodInventoryRecords(item, serviceRecords, batchTime) podInventoryRecords.each { |record| uid = record["PodUid"] @podInventoryHash[uid] = record } end else - $log.info("write_to_file:: else case coming from merge. podInventory: #{podInventory}") @podInventoryHash = podInventory - $log.info("write_to_file:: else case: size: #{podInventory.size()}") end # Write to mmap or regular file based on value of @useMmap flag if @useMmap - $log.info("in_kube_podinventory::write_to_file : creating new mmap file.") + $log.info("in_kube_podinventory::write_to_file : writing to mmap file case") # this is to ensure that we clear file contents before writing to file, check if there is a better way to do this File.open("/var/opt/microsoft/docker-cimprov/log/testing-podinventory.json", "w") @mmap = Mmap.new("/var/opt/microsoft/docker-cimprov/log/testing-podinventory.json", "rw") - - $log.info("in_kube_podinventory::write_to_file : writing to mmap file case") - $log.info("write_to_file:: podInventoryHash: #{@podInventoryHash}") @mmap << JSON.pretty_generate(@podInventoryHash).to_s - - sanityCheck = "" - sanityCheck = sanityCheck.dup if sanityCheck.frozen? - sanityCheck << @mmap - - $log.info("write_to_file:: sanity check: #{sanityCheck}") else $log.info("in_kube_podinventory::write_to_file : writing to regular file case") File.open("/var/opt/microsoft/docker-cimprov/log/testing-podinventory.json", "w") { |file| @@ -172,8 +153,7 @@ def write_to_file(podInventory) } end - $log.info("in_kube_podinventory::write_to_file : successfully finished writing to file") - $log.info("in_kube_podinventory::write_to_file : size of written file = #{File.size("/var/opt/microsoft/docker-cimprov/log/testing-podinventory.json") / 1000000.0} MB") + $log.info("in_kube_podinventory::write_to_file : successfully finished writing to file. size of written file = #{File.size("/var/opt/microsoft/docker-cimprov/log/testing-podinventory.json") / 1000000.0} MB") rescue => exception $log.info("in_kube_podinventory::write_to_file : writing to file failed. exception: #{exception} backtrace: #{exception.backtrace}") end @@ -281,9 +261,10 @@ def watch $log.info("in_kube_podinventory::watch : calling enumerate to make API server call and populate file with initial pod inventory data.") enumerate + #TODO: Check if watch pods restarts after connection is broken loop do #TODO: check if collection_version is correct when continuation token is not null and collection_version changes - $log.info("in_kube_pod_inventory::watch : inside infinite loop for watch pods. collection version: #{@collection_version}") + $log.info("in_kube_podinventory::watch : inside infinite loop for watch pods. collection version: #{@collection_version}") begin @KubernetesWatchClient.watch_pods(resource_version: @collection_version, as: :parsed) do |notice| $log.info("in_kube_podinventory::watch : inside watch pods! current time: #{Time.now.utc.iso8601}.") @@ -419,7 +400,6 @@ def enumerate(podList = nil) end def parse_and_emit_records(podInventory, serviceRecords, continuationToken, batchTime = Time.utc.iso8601) - # $log.info("parse_and_emit_records:: podInventory: #{podInventory}") currentTime = Time.now emitTime = Fluent::Engine.now #batchTime = currentTime.utc.iso8601 @@ -597,8 +577,6 @@ def parse_and_emit_records(podInventory, serviceRecords, continuationToken, batc end def parse_and_emit_merge_updates(podInventoryRecords) - # $log.info("parse_and_emit_merge_updates:: podInventory: #{podInventoryRecords}") - currentTime = Time.now emitTime = Fluent::Engine.now batchTime = currentTime.utc.iso8601 @@ -613,15 +591,9 @@ def parse_and_emit_merge_updates(podInventoryRecords) begin #begin block start # Getting windows nodes from kubeapi winNodes = KubernetesApiClient.getWindowsNodesArray - - $log.info("parse_and_emit_merge_updates:: podInventoryRecords: #{podInventoryRecords}") podInventoryRecords.each do |uid, record| - # $log.info("parse_and_emit_merge_updates:: inside each method. uid: #{uid}. record: #{record}") if !record.nil? - $log.info("parse_and_emit_merge_updates:: emitTime: #{emitTime}. batchTime: #{batchTime}. record collection time: #{record["CollectionTime"]}") - # $log.info("parse_and_emit_merge_updates:: record is not null.") record["CollectionTime"] = batchTime - $log.info("parse_and_emit_merge_updates:: record collection time: #{record["CollectionTime"]}") eventStream.add(emitTime, record) if record @inventoryToMdmConvertor.process_pod_inventory_record(record) end @@ -663,9 +635,8 @@ def merge_updates end $log.info("in_kube_podinventory::merge_updates : file contents read") if !fileContents.empty? - $log.info("merge_updates:: file contents before parsing: #{fileContents}") @podInventoryHash = Yajl::Parser.parse(fileContents) - $log.info("in_kube_podinventory::merge_updates : parse successful. size of hash: #{@podInventoryHash.size()} podInventoryHash: #{@podInventoryHash}") + $log.info("in_kube_podinventory::merge_updates : parse successful. size of hash: #{@podInventoryHash.size()}") end rescue => error $log.info("in_kube_podinventory::merge_updates : something went wrong with reading file. #{error}: #{error.backtrace}") @@ -720,9 +691,11 @@ def merge_updates $log.info("in_kube_podinventory:: merge_updates : podInventoryHash not null and not empty, will write to file") # only write if there is a change write_to_file(@podInventoryHash) - $log.info("merge_updates:: number of items in podInventoryHash: #{@podInventoryHash.length}. podInventoryHash: #{@podInventoryHash}") + $log.info("merge_updates:: number of items in podInventoryHash: #{@podInventoryHash.length}") + # $log.info("merge_updates:: number of items in podInventoryHash: #{@podInventoryHash.length}. podInventoryHash: #{@podInventoryHash}") parse_and_emit_merge_updates(@podInventoryHash) @podInventoryHash.clear + $log.info("merge_updates:: number of items in podInventoryHash after clear: #{@podInventoryHash.length}") else $log.info("in_kube_podinventory:: merge_updates : podInventoryHash was either null or empty, so NOT writing to file - should never be in this case") end From 27332e6a2abd83f35f37599b16bc098135e2db20 Mon Sep 17 00:00:00 2001 From: Khushi Chaudhari Date: Sun, 1 Aug 2021 13:01:11 -0700 Subject: [PATCH 49/76] added logging to see why watch doesnt always emit, also removed instance var podInvHash made it local --- source/plugins/ruby/in_kube_podinventory.rb | 70 +++++++++++++-------- 1 file changed, 43 insertions(+), 27 deletions(-) diff --git a/source/plugins/ruby/in_kube_podinventory.rb b/source/plugins/ruby/in_kube_podinventory.rb index e64fd9f93..becc7e235 100644 --- a/source/plugins/ruby/in_kube_podinventory.rb +++ b/source/plugins/ruby/in_kube_podinventory.rb @@ -45,7 +45,7 @@ def initialize @podsAPIE2ELatencyMs = 0 @noticeHash = {} - @podInventoryHash = {} + # @podInventoryHash = {} @useMmap = false @kubeperfTag = "oneagent.containerInsights.LINUX_PERF_BLOB" @@ -124,6 +124,7 @@ def write_to_file(podInventory) #TODO: check if you can pass @serviceRecords into getPodInventoryRecords rather than creating a local copy #TODO: if no, update code so we simply use instance variable everywhere and there is no need to pass serviceRecords serviceRecords= @serviceRecords + podInventoryHash = {} begin if !podInventory["items"].nil? && !podInventory["items"].empty? @@ -132,24 +133,26 @@ def write_to_file(podInventory) podInventoryRecords = getPodInventoryRecords(item, serviceRecords, batchTime) podInventoryRecords.each { |record| uid = record["PodUid"] - @podInventoryHash[uid] = record + podInventoryHash[uid] = record } end else - @podInventoryHash = podInventory + podInventoryHash = podInventory end + $log.info("write_to_file:: podInventoryHash size before write: #{podInventoryHash.size()}") + # Write to mmap or regular file based on value of @useMmap flag if @useMmap $log.info("in_kube_podinventory::write_to_file : writing to mmap file case") # this is to ensure that we clear file contents before writing to file, check if there is a better way to do this File.open("/var/opt/microsoft/docker-cimprov/log/testing-podinventory.json", "w") @mmap = Mmap.new("/var/opt/microsoft/docker-cimprov/log/testing-podinventory.json", "rw") - @mmap << JSON.pretty_generate(@podInventoryHash).to_s + @mmap << JSON.pretty_generate(podInventoryHash).to_s else $log.info("in_kube_podinventory::write_to_file : writing to regular file case") File.open("/var/opt/microsoft/docker-cimprov/log/testing-podinventory.json", "w") { |file| - file.write(JSON.pretty_generate(@podInventoryHash)) + file.write(JSON.pretty_generate(podInventoryHash)) } end @@ -258,16 +261,18 @@ def getNoticeRecord(notice) end def watch - $log.info("in_kube_podinventory::watch : calling enumerate to make API server call and populate file with initial pod inventory data.") - enumerate + # enumerate #TODO: Check if watch pods restarts after connection is broken loop do + $log.info("in_kube_podinventory::watch - inside infinite loop for watch pods. calling enumerate.") + enumerate + #TODO: check if collection_version is correct when continuation token is not null and collection_version changes $log.info("in_kube_podinventory::watch : inside infinite loop for watch pods. collection version: #{@collection_version}") begin @KubernetesWatchClient.watch_pods(resource_version: @collection_version, as: :parsed) do |notice| - $log.info("in_kube_podinventory::watch : inside watch pods! current time: #{Time.now.utc.iso8601}.") + $log.info("in_kube_podinventory::watch : inside watch pods! collection version: #{@collection_version}.") if !notice.nil? && !notice.empty? $log.info("in_kube_podinventory::watch : received a notice that is not null and not empty. notice type: #{notice["type"]}") @@ -275,6 +280,8 @@ def watch # Construct record with necessary fields (same fields as getPodInventoryRecords) record = getNoticeRecord(notice) + $log.info("watch:: record constructed looks like: #{record}") + @mutex.synchronize { @noticeHash[item["metadata"]["uid"]] = record } @@ -349,6 +356,9 @@ def enumerate(podList = nil) $log.info("in_kube_podinventory::enumerate : continuation token is not null and not empty") podsAPIChunkStartTime = (Time.now.to_f * 1000).to_i continuationToken, podInventory = KubernetesApiClient.getResourcesAndContinuationToken("pods?limit=#{@PODS_CHUNK_SIZE}&continue=#{continuationToken}") + # need to add collection resource version here + @collection_version = podInventory["metadata"]["resourceVersion"] + $log.info("in_kube_podinventory::enumerate : continuation token was not null. received collection version: #{@collection_version}") podsAPIChunkEndTime = (Time.now.to_f * 1000).to_i @podsAPIE2ELatencyMs = @podsAPIE2ELatencyMs + (podsAPIChunkEndTime - podsAPIChunkStartTime) if (!podInventory.nil? && !podInventory.empty? && podInventory.key?("items") && !podInventory["items"].nil? && !podInventory["items"].empty?) @@ -624,6 +634,8 @@ def parse_and_emit_merge_updates(podInventoryRecords) end def merge_updates + podInventoryHash = {} + begin fileContents = "" # Read file @@ -635,8 +647,8 @@ def merge_updates end $log.info("in_kube_podinventory::merge_updates : file contents read") if !fileContents.empty? - @podInventoryHash = Yajl::Parser.parse(fileContents) - $log.info("in_kube_podinventory::merge_updates : parse successful. size of hash: #{@podInventoryHash.size()}") + podInventoryHash = Yajl::Parser.parse(fileContents) + $log.info("in_kube_podinventory::merge_updates : parse successful. size of hash: #{podInventoryHash.size()}") end rescue => error $log.info("in_kube_podinventory::merge_updates : something went wrong with reading file. #{error}: #{error.backtrace}") @@ -654,48 +666,52 @@ def merge_updates case record["NoticeType"] when "ADDED" - @podInventoryHash[uid] = record + podInventoryHash[uid] = record $log.info("in_kube_podinventory::merge_updates : added new record to podInventoryHash") when "MODIFIED" - if @podInventoryHash[uid].nil? + if podInventoryHash[uid].nil? $log.info("in_kube_podinventory::merge_updates : modify case where uid for add was overwritten to modify within same minute") - @podInventoryHash[uid] = record + podInventoryHash[uid] = record else - $log.info("in_kube_podinventory::merge_updates : modify case where it is only a modification. old status: #{@podInventoryHash[uid]["PodStatus"]}. new status: #{record["PodStatus"]}") - val = @podInventoryHash[uid] + $log.info("in_kube_podinventory::merge_updates : modify case where it is only a modification. old status: #{podInventoryHash[uid]["PodStatus"]}. new status: #{record["PodStatus"]}") + val = podInventoryHash[uid] val["PodStatus"] = record["PodStatus"] - @podInventoryHash[uid] = val + podInventoryHash[uid] = val end $log.info("in_kube_podinventory::merge_updates :: modified and changes reflected in podInventoryHash") when "DELETED" - @podInventoryHash.delete(uid) - $log.info("in_kube_podinventory::merge_updates :: deleted from podInventoryHash") + if podInventoryHash.key?(uid) + podInventoryHash.delete(uid) + $log.info("in_kube_podinventory::merge_updates :: deleted from podInventoryHash") + else + $log.info("merge_updates:: key did not exist in hash so unable to delete") + end else $log.info("in_kube_podinventory::merge_updates :: something went wrong and didn't enter any cases for switch, notice type was #{record["NoticeType"]}") end - # $log.info("in_kube_podinventory::merge_updates :: end of switch") end + $log.info("merge_updates:: uid list length: #{uidList.size()}") # remove all looked at uids from the noticeHash uidList.each do |uid| @noticeHash.delete(uid) end - # copy noticeHash to tempHash and use tempHash to loop through so we dont lock on it for a long time + # TODO: copy noticeHash to tempHash and use tempHash to loop through so we dont lock on it for a long time $log.info("in_kube_podinventory::merge_updates :: removed all visited uids from noticeHash") } #TODO: Look for a way to replace only necessary contents, rather than everything $log.info("in_kube_podinventory:: merge_updates : about to replace entire contents of testing-podinventory.json") - if (!@podInventoryHash.nil? && !@podInventoryHash.empty?) + if (!podInventoryHash.nil? && !podInventoryHash.empty?) $log.info("in_kube_podinventory:: merge_updates : podInventoryHash not null and not empty, will write to file") # only write if there is a change - write_to_file(@podInventoryHash) - $log.info("merge_updates:: number of items in podInventoryHash: #{@podInventoryHash.length}") - # $log.info("merge_updates:: number of items in podInventoryHash: #{@podInventoryHash.length}. podInventoryHash: #{@podInventoryHash}") - parse_and_emit_merge_updates(@podInventoryHash) - @podInventoryHash.clear - $log.info("merge_updates:: number of items in podInventoryHash after clear: #{@podInventoryHash.length}") + write_to_file(podInventoryHash) + $log.info("merge_updates:: number of items in podInventoryHash: #{podInventoryHash.length}") + # $log.info("merge_updates:: number of items in podInventoryHash: #{@podInventoryHash.length}. podInventoryHash: #{podInventoryHash}") + parse_and_emit_merge_updates(podInventoryHash) + podInventoryHash.clear + $log.info("merge_updates:: number of items in podInventoryHash after clear: #{podInventoryHash.length}") else $log.info("in_kube_podinventory:: merge_updates : podInventoryHash was either null or empty, so NOT writing to file - should never be in this case") end From 135cf21c27a97fddd811b11fb6cceb94b6bc9284 Mon Sep 17 00:00:00 2001 From: Khushi Chaudhari Date: Mon, 2 Aug 2021 21:59:33 -0700 Subject: [PATCH 50/76] added logging to check watch api loop --- source/plugins/ruby/in_kube_podinventory.rb | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/source/plugins/ruby/in_kube_podinventory.rb b/source/plugins/ruby/in_kube_podinventory.rb index becc7e235..31a502fc4 100644 --- a/source/plugins/ruby/in_kube_podinventory.rb +++ b/source/plugins/ruby/in_kube_podinventory.rb @@ -288,6 +288,7 @@ def watch $log.info("in_kube_podinventory::watch : number of items in noticeHash = #{@noticeHash.size}") end + $log.info("in_kube_podinventory::watch : sanity check at the end of watch pods, need to jump back to the top. collection version: #{@collection_version}") end rescue => exception $log.warn("in_kube_podinventory::watch : watch events session got broken and re-establishing the session. backtrace: #{exception.backtrace}") @@ -597,6 +598,7 @@ def parse_and_emit_merge_updates(podInventoryRecords) @@istestvar = ENV["ISTEST"] continuationToken = nil + emittedPodCount = 0 begin #begin block start # Getting windows nodes from kubeapi @@ -610,11 +612,12 @@ def parse_and_emit_merge_updates(podInventoryRecords) end if @PODS_EMIT_STREAM_BATCH_SIZE > 0 && eventStream.count >= @PODS_EMIT_STREAM_BATCH_SIZE - $log.info("in_kube_podinventory::parse_and_emit_merge_updates: number of pod inventory records emitted #{@PODS_EMIT_STREAM_BATCH_SIZE} @ #{Time.now.utc.iso8601}") + $log.info("in_kube_podinventory::parse_and_emit_merge_updates: number of pod inventory records emitted #{eventStream.count} @ #{Time.now.utc.iso8601}") if (!@@istestvar.nil? && !@@istestvar.empty? && @@istestvar.casecmp("true") == 0) $log.info("kubePodInventoryEmitStreamSuccess @ #{Time.now.utc.iso8601}") end router.emit_stream(@tag, eventStream) if eventStream + emittedPodCount += eventStream.count eventStream = Fluent::MultiEventStream.new end @@ -624,8 +627,11 @@ def parse_and_emit_merge_updates(podInventoryRecords) if (!@@istestvar.nil? && !@@istestvar.empty? && @@istestvar.casecmp("true") == 0) $log.info("kubePodInventoryEmitStreamSuccess @ #{Time.now.utc.iso8601}") end + emittedPodCount += eventStream.count eventStream = nil end + + $log.info("parse_and_emit_merge_updates:: emittedPodCount = #{emittedPodCount}") rescue => errorStr $log.warn "Failed in parse_and_emit_merge_updates pod inventory: #{errorStr}. backtrace: #{errorStr.backtrace}" $log.debug_backtrace(errorStr.backtrace) @@ -652,7 +658,7 @@ def merge_updates end rescue => error $log.info("in_kube_podinventory::merge_updates : something went wrong with reading file. #{error}: #{error.backtrace}") - end + end $log.info("in_kube_podinventory::merge_updates : before noticeHash loop, number of items in hash: #{@noticeHash.size()}") @@ -684,7 +690,7 @@ def merge_updates podInventoryHash.delete(uid) $log.info("in_kube_podinventory::merge_updates :: deleted from podInventoryHash") else - $log.info("merge_updates:: key did not exist in hash so unable to delete") + $log.info("merge_updates:: error: key did not exist in hash so unable to delete (probably add and delete in same min)") end else $log.info("in_kube_podinventory::merge_updates :: something went wrong and didn't enter any cases for switch, notice type was #{record["NoticeType"]}") @@ -698,13 +704,13 @@ def merge_updates end # TODO: copy noticeHash to tempHash and use tempHash to loop through so we dont lock on it for a long time - $log.info("in_kube_podinventory::merge_updates :: removed all visited uids from noticeHash") + $log.info("in_kube_podinventory::merge_updates :: removed all visited uids from noticeHash. noticeHash size: #{noticeHash.size()}. uidList size: #{uidList.size()}") } #TODO: Look for a way to replace only necessary contents, rather than everything $log.info("in_kube_podinventory:: merge_updates : about to replace entire contents of testing-podinventory.json") if (!podInventoryHash.nil? && !podInventoryHash.empty?) - $log.info("in_kube_podinventory:: merge_updates : podInventoryHash not null and not empty, will write to file") + $log.info("in_kube_podinventory:: merge_updates : podInventoryHash not null and not empty, will write to file. podInventoryHash size after hash loop: #{podInventoryHash.size()}") # only write if there is a change write_to_file(podInventoryHash) $log.info("merge_updates:: number of items in podInventoryHash: #{podInventoryHash.length}") From e88b39eb66bec13d6db1af047a075cb7c9a8c099 Mon Sep 17 00:00:00 2001 From: Khushi Chaudhari Date: Tue, 3 Aug 2021 10:04:06 -0700 Subject: [PATCH 51/76] add time logging to merge_updates --- source/plugins/ruby/in_kube_podinventory.rb | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/source/plugins/ruby/in_kube_podinventory.rb b/source/plugins/ruby/in_kube_podinventory.rb index 31a502fc4..7f9e2dd8f 100644 --- a/source/plugins/ruby/in_kube_podinventory.rb +++ b/source/plugins/ruby/in_kube_podinventory.rb @@ -640,6 +640,8 @@ def parse_and_emit_merge_updates(podInventoryRecords) end def merge_updates + startTime = Time.now.utc.iso8601 + $log.info("merge_updates:: Start time: #{startTime}") podInventoryHash = {} begin @@ -704,7 +706,7 @@ def merge_updates end # TODO: copy noticeHash to tempHash and use tempHash to loop through so we dont lock on it for a long time - $log.info("in_kube_podinventory::merge_updates :: removed all visited uids from noticeHash. noticeHash size: #{noticeHash.size()}. uidList size: #{uidList.size()}") + $log.info("in_kube_podinventory::merge_updates :: removed all visited uids from noticeHash. noticeHash size: #{@noticeHash.size()}. uidList size: #{uidList.size()}") } #TODO: Look for a way to replace only necessary contents, rather than everything @@ -723,6 +725,9 @@ def merge_updates end $log.info("in_kube_podinventory:: merge_updates : finished replacing contents of testing-podinventory.json") + endTime = Time.now.utc.iso8601 + $log.info("merge_updates:: End time: #{endTime}") + $log.info("merge_updates:: total time taken = #{endTime - startTime}") end def run_periodic From 64a42103a13fc3038098b79260478c7e54d112b6 Mon Sep 17 00:00:00 2001 From: Khushi Chaudhari Date: Tue, 3 Aug 2021 14:33:15 -0700 Subject: [PATCH 52/76] added append function for continuation token and only writes to file after merge if any changes were made --- source/plugins/ruby/in_kube_podinventory.rb | 61 +++++++++++++++++++-- 1 file changed, 55 insertions(+), 6 deletions(-) diff --git a/source/plugins/ruby/in_kube_podinventory.rb b/source/plugins/ruby/in_kube_podinventory.rb index 7f9e2dd8f..6fbfeccd1 100644 --- a/source/plugins/ruby/in_kube_podinventory.rb +++ b/source/plugins/ruby/in_kube_podinventory.rb @@ -119,6 +119,46 @@ def shutdown end end + def append_to_file(podInventory) + # only to be called from enumerate continuation token + batchTime = Time.now.utc.iso8601 + serviceRecords = @serviceRecords + podInventoryHash = {} + + begin + if !podInventory["items"].nil? && !podInventory["items"].empty? + podInventory["items"].each do |item| + # Extract needed fields using getPodInventoryRecords and create a hash mapping uid -> record + podInventoryRecords = getPodInventoryRecords(item, serviceRecords, batchTime) + podInventoryRecords.each { |record| + uid = record["PodUid"] + podInventoryHash[uid] = record + } + end + end + + $log.info("append_to_file:: podInventoryHash size before write: #{podInventoryHash.size()}") + + # Write to mmap or regular file based on value of @useMmap flag + if @useMmap + $log.info("in_kube_podinventory::append_to_file : writing to mmap file case") + # this is to ensure that we clear file contents before writing to file, check if there is a better way to do this + File.open("/var/opt/microsoft/docker-cimprov/log/testing-podinventory.json", "a") + @mmap = Mmap.new("/var/opt/microsoft/docker-cimprov/log/testing-podinventory.json", "rw") + @mmap << JSON.pretty_generate(podInventoryHash).to_s + else + $log.info("in_kube_podinventory::append_to_file : writing to regular file case") + File.open("/var/opt/microsoft/docker-cimprov/log/testing-podinventory.json", "a") { |file| + file.write(JSON.pretty_generate(podInventoryHash)) + } + end + + $log.info("in_kube_podinventory::append_to_file : successfully finished appending to file. size of written file = #{File.size("/var/opt/microsoft/docker-cimprov/log/testing-podinventory.json") / 1000000.0} MB") + rescue => exception + $log.info("in_kube_podinventory::append_to_file : appending to file failed. exception: #{exception} backtrace: #{exception.backtrace}") + end + end + def write_to_file(podInventory) batchTime = Time.now.utc.iso8601 #TODO: check if you can pass @serviceRecords into getPodInventoryRecords rather than creating a local copy @@ -283,7 +323,8 @@ def watch $log.info("watch:: record constructed looks like: #{record}") @mutex.synchronize { - @noticeHash[item["metadata"]["uid"]] = record + # could be an issue here + @noticeHash[item["metadata"]["uid"]] = record } $log.info("in_kube_podinventory::watch : number of items in noticeHash = #{@noticeHash.size}") @@ -364,7 +405,7 @@ def enumerate(podList = nil) @podsAPIE2ELatencyMs = @podsAPIE2ELatencyMs + (podsAPIChunkEndTime - podsAPIChunkStartTime) if (!podInventory.nil? && !podInventory.empty? && podInventory.key?("items") && !podInventory["items"].nil? && !podInventory["items"].empty?) $log.info("in_kube_podinventory::enumerate : number of pod items :#{podInventory["items"].length} from Kube API @ #{Time.now.utc.iso8601}") - write_to_file(podInventory) + append_to_file(podInventory) parse_and_emit_records(podInventory, @serviceRecords, continuationToken, batchTime) else $log.warn "in_kube_podinventory::enumerate:Received empty podInventory" @@ -640,7 +681,7 @@ def parse_and_emit_merge_updates(podInventoryRecords) end def merge_updates - startTime = Time.now.utc.iso8601 + startTime = Time.now $log.info("merge_updates:: Start time: #{startTime}") podInventoryHash = {} @@ -667,6 +708,9 @@ def merge_updates uidList = [] @mutex.synchronize { + + shouldUpdateFile = @noticeHash.size() == 0 ? false : true + @noticeHash.each do |uid, record| $log.info("in_kube_podinventory::merge_updates : looping through noticeHash, type of notice: #{record["NoticeType"]}") @@ -712,12 +756,17 @@ def merge_updates #TODO: Look for a way to replace only necessary contents, rather than everything $log.info("in_kube_podinventory:: merge_updates : about to replace entire contents of testing-podinventory.json") if (!podInventoryHash.nil? && !podInventoryHash.empty?) - $log.info("in_kube_podinventory:: merge_updates : podInventoryHash not null and not empty, will write to file. podInventoryHash size after hash loop: #{podInventoryHash.size()}") + $log.info("in_kube_podinventory:: merge_updates : podInventoryHash not null and not empty. podInventoryHash size after hash loop: #{podInventoryHash.size()}") # only write if there is a change - write_to_file(podInventoryHash) + if shouldUpdateFile + $log.info("in_kube_podinventory:: merge_updates : shouldUpdateFile evals to true, therefore writing to file.") + write_to_file(podInventoryHash) + end $log.info("merge_updates:: number of items in podInventoryHash: #{podInventoryHash.length}") # $log.info("merge_updates:: number of items in podInventoryHash: #{@podInventoryHash.length}. podInventoryHash: #{podInventoryHash}") parse_and_emit_merge_updates(podInventoryHash) + + #TODO: bottom two are not necessary - can remove later podInventoryHash.clear $log.info("merge_updates:: number of items in podInventoryHash after clear: #{podInventoryHash.length}") else @@ -725,7 +774,7 @@ def merge_updates end $log.info("in_kube_podinventory:: merge_updates : finished replacing contents of testing-podinventory.json") - endTime = Time.now.utc.iso8601 + endTime = Time.now $log.info("merge_updates:: End time: #{endTime}") $log.info("merge_updates:: total time taken = #{endTime - startTime}") end From 928181595665fe289216eca02a75ed84d5955e3d Mon Sep 17 00:00:00 2001 From: Khushi Chaudhari Date: Tue, 3 Aug 2021 15:44:42 -0700 Subject: [PATCH 53/76] changed sleep time, fixed shouldUpdateFile scope bug, collection version init in initialize --- source/plugins/ruby/in_kube_podinventory.rb | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/source/plugins/ruby/in_kube_podinventory.rb b/source/plugins/ruby/in_kube_podinventory.rb index 6fbfeccd1..a531945e3 100644 --- a/source/plugins/ruby/in_kube_podinventory.rb +++ b/source/plugins/ruby/in_kube_podinventory.rb @@ -47,6 +47,7 @@ def initialize @noticeHash = {} # @podInventoryHash = {} @useMmap = false + @collection_version = "" @kubeperfTag = "oneagent.containerInsights.LINUX_PERF_BLOB" @kubeservicesTag = "oneagent.containerInsights.KUBE_SERVICES_BLOB" @@ -335,8 +336,9 @@ def watch $log.warn("in_kube_podinventory::watch : watch events session got broken and re-establishing the session. backtrace: #{exception.backtrace}") # $log.debug_backtrace(exception.backtrace) end - #TODO: check if 300 is the correct number to use here - sleep 300 + #TODO: check if 30 is the correct number to use here + # currently sleeping for 30 seconds before restarting + sleep 30 end end @@ -684,6 +686,7 @@ def merge_updates startTime = Time.now $log.info("merge_updates:: Start time: #{startTime}") podInventoryHash = {} + shouldUpdateFile = false begin fileContents = "" @@ -710,6 +713,7 @@ def merge_updates @mutex.synchronize { shouldUpdateFile = @noticeHash.size() == 0 ? false : true + $log.info("merge_updates:: shouldUpdateFile value is #{shouldUpdateFile}") @noticeHash.each do |uid, record| $log.info("in_kube_podinventory::merge_updates : looping through noticeHash, type of notice: #{record["NoticeType"]}") @@ -743,7 +747,7 @@ def merge_updates end end - $log.info("merge_updates:: uid list length: #{uidList.size()}") + $log.info("merge_updates:: uid list length: #{uidList.size()}. noticeHash size before deletion: #{@noticeHash.size()}") # remove all looked at uids from the noticeHash uidList.each do |uid| @noticeHash.delete(uid) From ea2e3bb8e4757cc3add2f521bb8c25f9459fd891 Mon Sep 17 00:00:00 2001 From: Khushi Chaudhari Date: Tue, 3 Aug 2021 19:01:45 -0700 Subject: [PATCH 54/76] adding logging before and after sleep --- source/plugins/ruby/in_kube_podinventory.rb | 2 ++ 1 file changed, 2 insertions(+) diff --git a/source/plugins/ruby/in_kube_podinventory.rb b/source/plugins/ruby/in_kube_podinventory.rb index a531945e3..3d53e7684 100644 --- a/source/plugins/ruby/in_kube_podinventory.rb +++ b/source/plugins/ruby/in_kube_podinventory.rb @@ -338,7 +338,9 @@ def watch end #TODO: check if 30 is the correct number to use here # currently sleeping for 30 seconds before restarting + $log.info("in_kube_podinventory::watch : makes it to the sleep command. time: #{Time.now.utc.iso8601}") sleep 30 + $log.info("in_kube_podinventory::watch : after sleep command. time: #{Time.now.utc.iso8601}") end end From 5b3863ff5629e3df519bcb7483e09e22b61c80a4 Mon Sep 17 00:00:00 2001 From: Khushi Chaudhari Date: Tue, 3 Aug 2021 20:10:45 -0700 Subject: [PATCH 55/76] adding timeouts --- source/plugins/ruby/in_kube_podinventory.rb | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/source/plugins/ruby/in_kube_podinventory.rb b/source/plugins/ruby/in_kube_podinventory.rb index 3d53e7684..da2f48f3c 100644 --- a/source/plugins/ruby/in_kube_podinventory.rb +++ b/source/plugins/ruby/in_kube_podinventory.rb @@ -94,9 +94,13 @@ def start ca_file: "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt", verify_ssl: OpenSSL::SSL::VERIFY_PEER, } + timeouts = { + open: 60 # default setting (in seconds) + read: nil # read will never timeout + } getTokenStr = "Bearer " + KubernetesApiClient.getTokenStr auth_options = { bearer_token: KubernetesApiClient.getTokenStr } - @KubernetesWatchClient = Kubeclient::Client.new("https://#{ENV["KUBERNETES_SERVICE_HOST"]}:#{ENV["KUBERNETES_PORT_443_TCP_PORT"]}/api/", "v1", ssl_options: ssl_options, auth_options: auth_options, as: :parsed) + @KubernetesWatchClient = Kubeclient::Client.new("https://#{ENV["KUBERNETES_SERVICE_HOST"]}:#{ENV["KUBERNETES_PORT_443_TCP_PORT"]}/api/", "v1", ssl_options: ssl_options, auth_options: auth_options, as: :parsed, timeouts: timeouts) $log.info("in_kube:podinventory::start: successfully created kubernetes watch client") $log.info("in_kube_podinventory::start: PODS_EMIT_STREAM_BATCH_SIZE @ #{@PODS_EMIT_STREAM_BATCH_SIZE}") @finished = false From 092fc64d1bb8c868e69dcf829563b6f7865e9567 Mon Sep 17 00:00:00 2001 From: Khushi Chaudhari Date: Tue, 3 Aug 2021 23:58:24 -0700 Subject: [PATCH 56/76] appending function json fix --- source/plugins/ruby/in_kube_podinventory.rb | 30 ++++++++++++++++++--- 1 file changed, 26 insertions(+), 4 deletions(-) diff --git a/source/plugins/ruby/in_kube_podinventory.rb b/source/plugins/ruby/in_kube_podinventory.rb index da2f48f3c..71fd5380a 100644 --- a/source/plugins/ruby/in_kube_podinventory.rb +++ b/source/plugins/ruby/in_kube_podinventory.rb @@ -86,7 +86,6 @@ def start if (!ENV["USEMMAP"].nil? && !ENV["USEMMAP"].empty? && ENV["USEMMAP"].casecmp("true") == 0) @useMmap = true end - $log.info("in_kube_podinventory::start: use mmap is: #{@useMmap}") # create kubernetes watch client @@ -95,7 +94,7 @@ def start verify_ssl: OpenSSL::SSL::VERIFY_PEER, } timeouts = { - open: 60 # default setting (in seconds) + open: 60, # default setting (in seconds) read: nil # read will never timeout } getTokenStr = "Bearer " + KubernetesApiClient.getTokenStr @@ -130,6 +129,27 @@ def append_to_file(podInventory) serviceRecords = @serviceRecords podInventoryHash = {} + # have to read file first + # TODO: make podInventoryHash an instance variable so we don't have read everytime + begin + fileContents = "" + # Read file + if @useMmap + fileContents = fileContents.dup if fileContents.frozen? + fileContents << @mmap + else + # define path above instead of hardcoding here + fileContents = File.read("/var/opt/microsoft/docker-cimprov/log/testing-podinventory.json") + end + $log.info("in_kube_podinventory::append_to_file : file contents read") + if !fileContents.empty? + podInventoryHash = Yajl::Parser.parse(fileContents) + $log.info("in_kube_podinventory::append_to_file : parse successful. size of hash: #{podInventoryHash.size()}") + end + rescue => error + $log.info("in_kube_podinventory::append_to_file : something went wrong with reading file. #{error}: #{error.backtrace}") + end + begin if !podInventory["items"].nil? && !podInventory["items"].empty? podInventory["items"].each do |item| @@ -148,12 +168,12 @@ def append_to_file(podInventory) if @useMmap $log.info("in_kube_podinventory::append_to_file : writing to mmap file case") # this is to ensure that we clear file contents before writing to file, check if there is a better way to do this - File.open("/var/opt/microsoft/docker-cimprov/log/testing-podinventory.json", "a") + File.open("/var/opt/microsoft/docker-cimprov/log/testing-podinventory.json", "w") @mmap = Mmap.new("/var/opt/microsoft/docker-cimprov/log/testing-podinventory.json", "rw") @mmap << JSON.pretty_generate(podInventoryHash).to_s else $log.info("in_kube_podinventory::append_to_file : writing to regular file case") - File.open("/var/opt/microsoft/docker-cimprov/log/testing-podinventory.json", "a") { |file| + File.open("/var/opt/microsoft/docker-cimprov/log/testing-podinventory.json", "w") { |file| file.write(JSON.pretty_generate(podInventoryHash)) } end @@ -701,6 +721,7 @@ def merge_updates fileContents = fileContents.dup if fileContents.frozen? fileContents << @mmap else + # define path above instead of hardcoding here fileContents = File.read("/var/opt/microsoft/docker-cimprov/log/testing-podinventory.json") end $log.info("in_kube_podinventory::merge_updates : file contents read") @@ -735,6 +756,7 @@ def merge_updates $log.info("in_kube_podinventory::merge_updates : modify case where uid for add was overwritten to modify within same minute") podInventoryHash[uid] = record else + # TODO: will need to modify other fields later $log.info("in_kube_podinventory::merge_updates : modify case where it is only a modification. old status: #{podInventoryHash[uid]["PodStatus"]}. new status: #{record["PodStatus"]}") val = podInventoryHash[uid] val["PodStatus"] = record["PodStatus"] From ece8ac90cd1183a994751e708dd3ac1eaf42f745 Mon Sep 17 00:00:00 2001 From: Khushi Chaudhari Date: Mon, 9 Aug 2021 22:37:08 -0700 Subject: [PATCH 57/76] cleaned up code and changed read timeout to :open --- source/plugins/ruby/in_kube_podinventory.rb | 202 +++++++++++--------- 1 file changed, 113 insertions(+), 89 deletions(-) diff --git a/source/plugins/ruby/in_kube_podinventory.rb b/source/plugins/ruby/in_kube_podinventory.rb index 71fd5380a..1988bb975 100644 --- a/source/plugins/ruby/in_kube_podinventory.rb +++ b/source/plugins/ruby/in_kube_podinventory.rb @@ -95,12 +95,11 @@ def start } timeouts = { open: 60, # default setting (in seconds) - read: nil # read will never timeout + read: :open # read will never timeout } getTokenStr = "Bearer " + KubernetesApiClient.getTokenStr auth_options = { bearer_token: KubernetesApiClient.getTokenStr } @KubernetesWatchClient = Kubeclient::Client.new("https://#{ENV["KUBERNETES_SERVICE_HOST"]}:#{ENV["KUBERNETES_PORT_443_TCP_PORT"]}/api/", "v1", ssl_options: ssl_options, auth_options: auth_options, as: :parsed, timeouts: timeouts) - $log.info("in_kube:podinventory::start: successfully created kubernetes watch client") $log.info("in_kube_podinventory::start: PODS_EMIT_STREAM_BATCH_SIZE @ #{@PODS_EMIT_STREAM_BATCH_SIZE}") @finished = false @condition = ConditionVariable.new @@ -123,14 +122,68 @@ def shutdown end end - def append_to_file(podInventory) - # only to be called from enumerate continuation token - batchTime = Time.now.utc.iso8601 - serviceRecords = @serviceRecords - podInventoryHash = {} - - # have to read file first - # TODO: make podInventoryHash an instance variable so we don't have read everytime + # def append_to_file(podInventory) + # # only to be called from enumerate continuation token + # batchTime = Time.now.utc.iso8601 + # serviceRecords = @serviceRecords + # podInventoryHash = {} + + # # have to read file first + # # TODO: make podInventoryHash an instance variable so we don't have read everytime + # begin + # fileContents = "" + # # Read file + # if @useMmap + # fileContents = fileContents.dup if fileContents.frozen? + # fileContents << @mmap + # else + # # define path above instead of hardcoding here + # fileContents = File.read("/var/opt/microsoft/docker-cimprov/log/testing-podinventory.json") + # end + # $log.info("in_kube_podinventory::append_to_file : file contents read") + # if !fileContents.empty? + # podInventoryHash = Yajl::Parser.parse(fileContents) + # $log.info("in_kube_podinventory::append_to_file : parse successful. size of hash: #{podInventoryHash.size()}") + # end + # rescue => error + # $log.info("in_kube_podinventory::append_to_file : something went wrong with reading file. #{error}: #{error.backtrace}") + # end + + # begin + # if !podInventory["items"].nil? && !podInventory["items"].empty? + # podInventory["items"].each do |item| + # # Extract needed fields using getPodInventoryRecords and create a hash mapping uid -> record + # podInventoryRecords = getPodInventoryRecords(item, serviceRecords, batchTime) + # podInventoryRecords.each { |record| + # uid = record["PodUid"] + # podInventoryHash[uid] = record + # } + # end + # end + + # $log.info("append_to_file:: podInventoryHash size before write: #{podInventoryHash.size()}") + + # # Write to mmap or regular file based on value of @useMmap flag + # if @useMmap + # $log.info("in_kube_podinventory::append_to_file : writing to mmap file case") + # # this is to ensure that we clear file contents before writing to file, check if there is a better way to do this + # File.open("/var/opt/microsoft/docker-cimprov/log/testing-podinventory.json", "w") + # @mmap = Mmap.new("/var/opt/microsoft/docker-cimprov/log/testing-podinventory.json", "rw") + # @mmap << JSON.pretty_generate(podInventoryHash).to_s + # else + # $log.info("in_kube_podinventory::append_to_file : writing to regular file case") + # File.open("/var/opt/microsoft/docker-cimprov/log/testing-podinventory.json", "w") { |file| + # file.write(JSON.pretty_generate(podInventoryHash)) + # } + # end + + # $log.info("in_kube_podinventory::append_to_file : successfully finished appending to file. size of written file = #{File.size("/var/opt/microsoft/docker-cimprov/log/testing-podinventory.json") / 1000000.0} MB") + # rescue => exception + # $log.info("in_kube_podinventory::append_to_file : appending to file failed. exception: #{exception} backtrace: #{exception.backtrace}") + # end + # end + + def read_file begin fileContents = "" # Read file @@ -141,56 +194,31 @@ def append_to_file(podInventory) # define path above instead of hardcoding here fileContents = File.read("/var/opt/microsoft/docker-cimprov/log/testing-podinventory.json") end - $log.info("in_kube_podinventory::append_to_file : file contents read") + # $log.info("in_kube_podinventory::append_to_file : file contents read") if !fileContents.empty? podInventoryHash = Yajl::Parser.parse(fileContents) - $log.info("in_kube_podinventory::append_to_file : parse successful. size of hash: #{podInventoryHash.size()}") + # $log.info("in_kube_podinventory::append_to_file : parse successful. size of hash: #{podInventoryHash.size()}") end rescue => error $log.info("in_kube_podinventory::append_to_file : something went wrong with reading file. #{error}: #{error.backtrace}") end + return podInventoryHash + end - begin - if !podInventory["items"].nil? && !podInventory["items"].empty? - podInventory["items"].each do |item| - # Extract needed fields using getPodInventoryRecords and create a hash mapping uid -> record - podInventoryRecords = getPodInventoryRecords(item, serviceRecords, batchTime) - podInventoryRecords.each { |record| - uid = record["PodUid"] - podInventoryHash[uid] = record - } - end - end - - $log.info("append_to_file:: podInventoryHash size before write: #{podInventoryHash.size()}") - - # Write to mmap or regular file based on value of @useMmap flag - if @useMmap - $log.info("in_kube_podinventory::append_to_file : writing to mmap file case") - # this is to ensure that we clear file contents before writing to file, check if there is a better way to do this - File.open("/var/opt/microsoft/docker-cimprov/log/testing-podinventory.json", "w") - @mmap = Mmap.new("/var/opt/microsoft/docker-cimprov/log/testing-podinventory.json", "rw") - @mmap << JSON.pretty_generate(podInventoryHash).to_s - else - $log.info("in_kube_podinventory::append_to_file : writing to regular file case") - File.open("/var/opt/microsoft/docker-cimprov/log/testing-podinventory.json", "w") { |file| - file.write(JSON.pretty_generate(podInventoryHash)) - } - end - - $log.info("in_kube_podinventory::append_to_file : successfully finished appending to file. size of written file = #{File.size("/var/opt/microsoft/docker-cimprov/log/testing-podinventory.json") / 1000000.0} MB") - rescue => exception - $log.info("in_kube_podinventory::append_to_file : appending to file failed. exception: #{exception} backtrace: #{exception.backtrace}") - end - end - def write_to_file(podInventory) + def write_to_file(podInventory, append) batchTime = Time.now.utc.iso8601 #TODO: check if you can pass @serviceRecords into getPodInventoryRecords rather than creating a local copy #TODO: if no, update code so we simply use instance variable everywhere and there is no need to pass serviceRecords serviceRecords= @serviceRecords podInventoryHash = {} + if append + # have to read file first + # TODO: make podInventoryHash an instance variable so we don't have read everytime + read_file + end + begin if !podInventory["items"].nil? && !podInventory["items"].empty? podInventory["items"].each do |item| @@ -205,7 +233,7 @@ def write_to_file(podInventory) podInventoryHash = podInventory end - $log.info("write_to_file:: podInventoryHash size before write: #{podInventoryHash.size()}") + # $log.info("write_to_file:: podInventoryHash size before write: #{podInventoryHash.size()}") # Write to mmap or regular file based on value of @useMmap flag if @useMmap @@ -285,9 +313,7 @@ def getNoticeRecord(notice) end record["Computer"] = nodeName - #TODO: replace w KubernetesApiClient.getClusterId in agent code record["ClusterId"] = KubernetesApiClient.getClusterId - #TODO: replace w KubernetesApiClient.getClusterName in agent code record["ClusterName"] = KubernetesApiClient.getClusterName #TODO: make a call to getServiceNameFromLabels -- need to pass in serviceRecords for this record["ServiceName"] = "" @@ -326,14 +352,9 @@ def getNoticeRecord(notice) end def watch - # enumerate - - #TODO: Check if watch pods restarts after connection is broken loop do - $log.info("in_kube_podinventory::watch - inside infinite loop for watch pods. calling enumerate.") + # $log.info("in_kube_podinventory::watch - inside infinite loop for watch pods. calling enumerate.") enumerate - - #TODO: check if collection_version is correct when continuation token is not null and collection_version changes $log.info("in_kube_podinventory::watch : inside infinite loop for watch pods. collection version: #{@collection_version}") begin @KubernetesWatchClient.watch_pods(resource_version: @collection_version, as: :parsed) do |notice| @@ -344,7 +365,6 @@ def watch item = notice["object"] # Construct record with necessary fields (same fields as getPodInventoryRecords) record = getNoticeRecord(notice) - $log.info("watch:: record constructed looks like: #{record}") @mutex.synchronize { @@ -364,7 +384,7 @@ def watch # currently sleeping for 30 seconds before restarting $log.info("in_kube_podinventory::watch : makes it to the sleep command. time: #{Time.now.utc.iso8601}") sleep 30 - $log.info("in_kube_podinventory::watch : after sleep command. time: #{Time.now.utc.iso8601}") + # $log.info("in_kube_podinventory::watch : after sleep command. time: #{Time.now.utc.iso8601}") end end @@ -415,7 +435,7 @@ def enumerate(podList = nil) @podsAPIE2ELatencyMs = (podsAPIChunkEndTime - podsAPIChunkStartTime) if (!podInventory.nil? && !podInventory.empty? && podInventory.key?("items") && !podInventory["items"].nil? && !podInventory["items"].empty?) $log.info("in_kube_podinventory::enumerate : number of pod items :#{podInventory["items"].length} from Kube API @ #{Time.now.utc.iso8601}") - write_to_file(podInventory) + write_to_file(podInventory, false) parse_and_emit_records(podInventory, @serviceRecords, continuationToken, batchTime) else $log.warn "in_kube_podinventory::enumerate:Received empty podInventory" @@ -433,7 +453,7 @@ def enumerate(podList = nil) @podsAPIE2ELatencyMs = @podsAPIE2ELatencyMs + (podsAPIChunkEndTime - podsAPIChunkStartTime) if (!podInventory.nil? && !podInventory.empty? && podInventory.key?("items") && !podInventory["items"].nil? && !podInventory["items"].empty?) $log.info("in_kube_podinventory::enumerate : number of pod items :#{podInventory["items"].length} from Kube API @ #{Time.now.utc.iso8601}") - append_to_file(podInventory) + write_to_file(podInventory, true) parse_and_emit_records(podInventory, @serviceRecords, continuationToken, batchTime) else $log.warn "in_kube_podinventory::enumerate:Received empty podInventory" @@ -709,29 +729,31 @@ def parse_and_emit_merge_updates(podInventoryRecords) end def merge_updates - startTime = Time.now - $log.info("merge_updates:: Start time: #{startTime}") + # startTime = Time.now + # $log.info("merge_updates:: Start time: #{startTime}") podInventoryHash = {} shouldUpdateFile = false - begin - fileContents = "" - # Read file - if @useMmap - fileContents = fileContents.dup if fileContents.frozen? - fileContents << @mmap - else - # define path above instead of hardcoding here - fileContents = File.read("/var/opt/microsoft/docker-cimprov/log/testing-podinventory.json") - end - $log.info("in_kube_podinventory::merge_updates : file contents read") - if !fileContents.empty? - podInventoryHash = Yajl::Parser.parse(fileContents) - $log.info("in_kube_podinventory::merge_updates : parse successful. size of hash: #{podInventoryHash.size()}") - end - rescue => error - $log.info("in_kube_podinventory::merge_updates : something went wrong with reading file. #{error}: #{error.backtrace}") - end + # read file + read_file + # begin + # fileContents = "" + # # Read file + # if @useMmap + # fileContents = fileContents.dup if fileContents.frozen? + # fileContents << @mmap + # else + # # define path above instead of hardcoding here + # fileContents = File.read("/var/opt/microsoft/docker-cimprov/log/testing-podinventory.json") + # end + # $log.info("in_kube_podinventory::merge_updates : file contents read") + # if !fileContents.empty? + # podInventoryHash = Yajl::Parser.parse(fileContents) + # $log.info("in_kube_podinventory::merge_updates : parse successful. size of hash: #{podInventoryHash.size()}") + # end + # rescue => error + # $log.info("in_kube_podinventory::merge_updates : something went wrong with reading file. #{error}: #{error.backtrace}") + # end $log.info("in_kube_podinventory::merge_updates : before noticeHash loop, number of items in hash: #{@noticeHash.size()}") @@ -758,11 +780,13 @@ def merge_updates else # TODO: will need to modify other fields later $log.info("in_kube_podinventory::merge_updates : modify case where it is only a modification. old status: #{podInventoryHash[uid]["PodStatus"]}. new status: #{record["PodStatus"]}") - val = podInventoryHash[uid] - val["PodStatus"] = record["PodStatus"] - podInventoryHash[uid] = val + # val = podInventoryHash[uid] + # val["PodStatus"] = record["PodStatus"] + # podInventoryHash[uid] = val + + # TODO: only update modified fields or update everything? + podInventoryHash[uid] = record end - $log.info("in_kube_podinventory::merge_updates :: modified and changes reflected in podInventoryHash") when "DELETED" if podInventoryHash.key?(uid) podInventoryHash.delete(uid) @@ -792,23 +816,23 @@ def merge_updates # only write if there is a change if shouldUpdateFile $log.info("in_kube_podinventory:: merge_updates : shouldUpdateFile evals to true, therefore writing to file.") - write_to_file(podInventoryHash) + write_to_file(podInventoryHash, false) end $log.info("merge_updates:: number of items in podInventoryHash: #{podInventoryHash.length}") # $log.info("merge_updates:: number of items in podInventoryHash: #{@podInventoryHash.length}. podInventoryHash: #{podInventoryHash}") parse_and_emit_merge_updates(podInventoryHash) #TODO: bottom two are not necessary - can remove later - podInventoryHash.clear - $log.info("merge_updates:: number of items in podInventoryHash after clear: #{podInventoryHash.length}") + # podInventoryHash.clear + # $log.info("merge_updates:: number of items in podInventoryHash after clear: #{podInventoryHash.length}") else $log.info("in_kube_podinventory:: merge_updates : podInventoryHash was either null or empty, so NOT writing to file - should never be in this case") end $log.info("in_kube_podinventory:: merge_updates : finished replacing contents of testing-podinventory.json") - endTime = Time.now - $log.info("merge_updates:: End time: #{endTime}") - $log.info("merge_updates:: total time taken = #{endTime - startTime}") + # endTime = Time.now + # $log.info("merge_updates:: End time: #{endTime}") + # $log.info("merge_updates:: total time taken = #{endTime - startTime}") end def run_periodic From f6f201344a21ebc5e0f9079d2b38337fdf9702a9 Mon Sep 17 00:00:00 2001 From: Khushi Chaudhari Date: Tue, 10 Aug 2021 09:40:00 -0700 Subject: [PATCH 58/76] changing read timeout back to nil --- source/plugins/ruby/in_kube_podinventory.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/plugins/ruby/in_kube_podinventory.rb b/source/plugins/ruby/in_kube_podinventory.rb index 1988bb975..74ddc6871 100644 --- a/source/plugins/ruby/in_kube_podinventory.rb +++ b/source/plugins/ruby/in_kube_podinventory.rb @@ -95,7 +95,7 @@ def start } timeouts = { open: 60, # default setting (in seconds) - read: :open # read will never timeout + read: nil # read will never timeout } getTokenStr = "Bearer " + KubernetesApiClient.getTokenStr auth_options = { bearer_token: KubernetesApiClient.getTokenStr } From 6d5c091377086c78db91dcfce8b23c6bc034ac3d Mon Sep 17 00:00:00 2001 From: Khushi Chaudhari Date: Tue, 10 Aug 2021 11:30:53 -0700 Subject: [PATCH 59/76] parse and emit merge error fix potentially --- source/plugins/ruby/in_kube_podinventory.rb | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/source/plugins/ruby/in_kube_podinventory.rb b/source/plugins/ruby/in_kube_podinventory.rb index 74ddc6871..e253014d8 100644 --- a/source/plugins/ruby/in_kube_podinventory.rb +++ b/source/plugins/ruby/in_kube_podinventory.rb @@ -186,6 +186,7 @@ def shutdown def read_file begin fileContents = "" + podInventoryHash = {} # Read file if @useMmap fileContents = fileContents.dup if fileContents.frozen? @@ -197,11 +198,12 @@ def read_file # $log.info("in_kube_podinventory::append_to_file : file contents read") if !fileContents.empty? podInventoryHash = Yajl::Parser.parse(fileContents) - # $log.info("in_kube_podinventory::append_to_file : parse successful. size of hash: #{podInventoryHash.size()}") end rescue => error - $log.info("in_kube_podinventory::append_to_file : something went wrong with reading file. #{error}: #{error.backtrace}") + $log.info("in_kube_podinventory::read_file : something went wrong with reading file. #{error}: #{error.backtrace}") end + + $log.info("in_kube_podinventory::read_file : read successful. size of hash: #{podInventoryHash.size()}") return podInventoryHash end @@ -216,7 +218,8 @@ def write_to_file(podInventory, append) if append # have to read file first # TODO: make podInventoryHash an instance variable so we don't have read everytime - read_file + $log.info("write_to_file:: append is true, so will read file first") + podInventoryHash = read_file end begin @@ -233,7 +236,7 @@ def write_to_file(podInventory, append) podInventoryHash = podInventory end - # $log.info("write_to_file:: podInventoryHash size before write: #{podInventoryHash.size()}") + $log.info("write_to_file:: podInventoryHash size before write: #{podInventoryHash.size()}") # Write to mmap or regular file based on value of @useMmap flag if @useMmap @@ -735,7 +738,8 @@ def merge_updates shouldUpdateFile = false # read file - read_file + podInventoryHash = read_file + $log.info("merge_updates:: podInventoryHash size : #{podInventoryHash.size()}") # begin # fileContents = "" # # Read file From fe7a1be62827a67defc9715ba8e2dcde2b9c5dca Mon Sep 17 00:00:00 2001 From: Khushi Chaudhari Date: Tue, 10 Aug 2021 12:56:31 -0700 Subject: [PATCH 60/76] clean up and change open timeoiut to nil --- source/plugins/ruby/in_kube_podinventory.rb | 80 +++------------------ 1 file changed, 10 insertions(+), 70 deletions(-) diff --git a/source/plugins/ruby/in_kube_podinventory.rb b/source/plugins/ruby/in_kube_podinventory.rb index e253014d8..a4f8790f3 100644 --- a/source/plugins/ruby/in_kube_podinventory.rb +++ b/source/plugins/ruby/in_kube_podinventory.rb @@ -45,7 +45,6 @@ def initialize @podsAPIE2ELatencyMs = 0 @noticeHash = {} - # @podInventoryHash = {} @useMmap = false @collection_version = "" @@ -53,6 +52,7 @@ def initialize @kubeservicesTag = "oneagent.containerInsights.KUBE_SERVICES_BLOB" @containerInventoryTag = "oneagent.containerInsights.CONTAINER_INVENTORY_BLOB" @insightsMetricsTag = "oneagent.containerInsights.INSIGHTS_METRICS_BLOB" + @podInventoryFile = "/var/opt/microsoft/docker-cimprov/log/testing-podinventory.json" end config_param :run_interval, :time, :default => 60 @@ -86,7 +86,7 @@ def start if (!ENV["USEMMAP"].nil? && !ENV["USEMMAP"].empty? && ENV["USEMMAP"].casecmp("true") == 0) @useMmap = true end - $log.info("in_kube_podinventory::start: use mmap is: #{@useMmap}") + $log.info("in_kube_podinventory::start: use mmap state is: #{@useMmap}") # create kubernetes watch client ssl_options = { @@ -94,7 +94,7 @@ def start verify_ssl: OpenSSL::SSL::VERIFY_PEER, } timeouts = { - open: 60, # default setting (in seconds) + open: nil, # default setting (in seconds) read: nil # read will never timeout } getTokenStr = "Bearer " + KubernetesApiClient.getTokenStr @@ -122,67 +122,6 @@ def shutdown end end - # def append_to_file(podInventory) - # # only to be called from enumerate continuation token - # batchTime = Time.now.utc.iso8601 - # serviceRecords = @serviceRecords - # podInventoryHash = {} - - # # have to read file first - # # TODO: make podInventoryHash an instance variable so we don't have read everytime - # begin - # fileContents = "" - # # Read file - # if @useMmap - # fileContents = fileContents.dup if fileContents.frozen? - # fileContents << @mmap - # else - # # define path above instead of hardcoding here - # fileContents = File.read("/var/opt/microsoft/docker-cimprov/log/testing-podinventory.json") - # end - # $log.info("in_kube_podinventory::append_to_file : file contents read") - # if !fileContents.empty? - # podInventoryHash = Yajl::Parser.parse(fileContents) - # $log.info("in_kube_podinventory::append_to_file : parse successful. size of hash: #{podInventoryHash.size()}") - # end - # rescue => error - # $log.info("in_kube_podinventory::append_to_file : something went wrong with reading file. #{error}: #{error.backtrace}") - # end - - # begin - # if !podInventory["items"].nil? && !podInventory["items"].empty? - # podInventory["items"].each do |item| - # # Extract needed fields using getPodInventoryRecords and create a hash mapping uid -> record - # podInventoryRecords = getPodInventoryRecords(item, serviceRecords, batchTime) - # podInventoryRecords.each { |record| - # uid = record["PodUid"] - # podInventoryHash[uid] = record - # } - # end - # end - - # $log.info("append_to_file:: podInventoryHash size before write: #{podInventoryHash.size()}") - - # # Write to mmap or regular file based on value of @useMmap flag - # if @useMmap - # $log.info("in_kube_podinventory::append_to_file : writing to mmap file case") - # # this is to ensure that we clear file contents before writing to file, check if there is a better way to do this - # File.open("/var/opt/microsoft/docker-cimprov/log/testing-podinventory.json", "w") - # @mmap = Mmap.new("/var/opt/microsoft/docker-cimprov/log/testing-podinventory.json", "rw") - # @mmap << JSON.pretty_generate(podInventoryHash).to_s - # else - # $log.info("in_kube_podinventory::append_to_file : writing to regular file case") - # File.open("/var/opt/microsoft/docker-cimprov/log/testing-podinventory.json", "w") { |file| - # file.write(JSON.pretty_generate(podInventoryHash)) - # } - # end - - # $log.info("in_kube_podinventory::append_to_file : successfully finished appending to file. size of written file = #{File.size("/var/opt/microsoft/docker-cimprov/log/testing-podinventory.json") / 1000000.0} MB") - # rescue => exception - # $log.info("in_kube_podinventory::append_to_file : appending to file failed. exception: #{exception} backtrace: #{exception.backtrace}") - # end - # end - def read_file begin fileContents = "" @@ -193,7 +132,7 @@ def read_file fileContents << @mmap else # define path above instead of hardcoding here - fileContents = File.read("/var/opt/microsoft/docker-cimprov/log/testing-podinventory.json") + fileContents = File.read(@podInventoryFile) end # $log.info("in_kube_podinventory::append_to_file : file contents read") if !fileContents.empty? @@ -242,17 +181,17 @@ def write_to_file(podInventory, append) if @useMmap $log.info("in_kube_podinventory::write_to_file : writing to mmap file case") # this is to ensure that we clear file contents before writing to file, check if there is a better way to do this - File.open("/var/opt/microsoft/docker-cimprov/log/testing-podinventory.json", "w") - @mmap = Mmap.new("/var/opt/microsoft/docker-cimprov/log/testing-podinventory.json", "rw") + File.open(@podInventoryFile, "w") + @mmap = Mmap.new(@podInventoryFile, "rw") @mmap << JSON.pretty_generate(podInventoryHash).to_s else $log.info("in_kube_podinventory::write_to_file : writing to regular file case") - File.open("/var/opt/microsoft/docker-cimprov/log/testing-podinventory.json", "w") { |file| + File.open(@podInventoryFile, "w") { |file| file.write(JSON.pretty_generate(podInventoryHash)) } end - $log.info("in_kube_podinventory::write_to_file : successfully finished writing to file. size of written file = #{File.size("/var/opt/microsoft/docker-cimprov/log/testing-podinventory.json") / 1000000.0} MB") + $log.info("in_kube_podinventory::write_to_file : successfully finished writing to file. size of written file = #{File.size(@podInventoryFile) / 1000000.0} MB") rescue => exception $log.info("in_kube_podinventory::write_to_file : writing to file failed. exception: #{exception} backtrace: #{exception.backtrace}") end @@ -264,6 +203,7 @@ def getNoticeRecord(notice) item = notice["object"] #TODO: check assumption that batch time can be current time (CollectionTime) batchTime = Time.now.utc.iso8601 + serviceRecords = @serviceRecords begin record["CollectionTime"] = batchTime @@ -319,7 +259,7 @@ def getNoticeRecord(notice) record["ClusterId"] = KubernetesApiClient.getClusterId record["ClusterName"] = KubernetesApiClient.getClusterName #TODO: make a call to getServiceNameFromLabels -- need to pass in serviceRecords for this - record["ServiceName"] = "" + record["ServiceName"] = getServiceNameFromLabels(item["metadata"]["namespace"], item["metadata"]["label"], serviceRecords) if !item["metadata"]["ownerReferences"].nil? record["ControllerKind"] = item["metadata"]["ownerReferences"][0]["kind"] From 340a6ab3d10bb342e7fb51f2f882b974f946d352 Mon Sep 17 00:00:00 2001 From: Khushi Chaudhari Date: Tue, 10 Aug 2021 13:59:15 -0700 Subject: [PATCH 61/76] reverting to an old commit from last week to see if timeouts are still an issue --- source/plugins/ruby/in_kube_podinventory.rb | 164 ++++++++++++-------- 1 file changed, 98 insertions(+), 66 deletions(-) diff --git a/source/plugins/ruby/in_kube_podinventory.rb b/source/plugins/ruby/in_kube_podinventory.rb index a4f8790f3..ad048a9a1 100644 --- a/source/plugins/ruby/in_kube_podinventory.rb +++ b/source/plugins/ruby/in_kube_podinventory.rb @@ -45,6 +45,7 @@ def initialize @podsAPIE2ELatencyMs = 0 @noticeHash = {} + # @podInventoryHash = {} @useMmap = false @collection_version = "" @@ -52,7 +53,6 @@ def initialize @kubeservicesTag = "oneagent.containerInsights.KUBE_SERVICES_BLOB" @containerInventoryTag = "oneagent.containerInsights.CONTAINER_INVENTORY_BLOB" @insightsMetricsTag = "oneagent.containerInsights.INSIGHTS_METRICS_BLOB" - @podInventoryFile = "/var/opt/microsoft/docker-cimprov/log/testing-podinventory.json" end config_param :run_interval, :time, :default => 60 @@ -86,7 +86,7 @@ def start if (!ENV["USEMMAP"].nil? && !ENV["USEMMAP"].empty? && ENV["USEMMAP"].casecmp("true") == 0) @useMmap = true end - $log.info("in_kube_podinventory::start: use mmap state is: #{@useMmap}") + $log.info("in_kube_podinventory::start: use mmap is: #{@useMmap}") # create kubernetes watch client ssl_options = { @@ -94,12 +94,13 @@ def start verify_ssl: OpenSSL::SSL::VERIFY_PEER, } timeouts = { - open: nil, # default setting (in seconds) - read: nil # read will never timeout + open: 60, # default setting (in seconds) + read: nil # read will never timeout } getTokenStr = "Bearer " + KubernetesApiClient.getTokenStr auth_options = { bearer_token: KubernetesApiClient.getTokenStr } @KubernetesWatchClient = Kubeclient::Client.new("https://#{ENV["KUBERNETES_SERVICE_HOST"]}:#{ENV["KUBERNETES_PORT_443_TCP_PORT"]}/api/", "v1", ssl_options: ssl_options, auth_options: auth_options, as: :parsed, timeouts: timeouts) + $log.info("in_kube:podinventory::start: successfully created kubernetes watch client") $log.info("in_kube_podinventory::start: PODS_EMIT_STREAM_BATCH_SIZE @ #{@PODS_EMIT_STREAM_BATCH_SIZE}") @finished = false @condition = ConditionVariable.new @@ -122,45 +123,74 @@ def shutdown end end - def read_file + def append_to_file(podInventory) + # only to be called from enumerate continuation token + batchTime = Time.now.utc.iso8601 + serviceRecords = @serviceRecords + podInventoryHash = {} + + # have to read file first + # TODO: make podInventoryHash an instance variable so we don't have read everytime begin fileContents = "" - podInventoryHash = {} # Read file if @useMmap fileContents = fileContents.dup if fileContents.frozen? fileContents << @mmap else # define path above instead of hardcoding here - fileContents = File.read(@podInventoryFile) + fileContents = File.read("/var/opt/microsoft/docker-cimprov/log/testing-podinventory.json") end - # $log.info("in_kube_podinventory::append_to_file : file contents read") + $log.info("in_kube_podinventory::append_to_file : file contents read") if !fileContents.empty? podInventoryHash = Yajl::Parser.parse(fileContents) + $log.info("in_kube_podinventory::append_to_file : parse successful. size of hash: #{podInventoryHash.size()}") end rescue => error - $log.info("in_kube_podinventory::read_file : something went wrong with reading file. #{error}: #{error.backtrace}") + $log.info("in_kube_podinventory::append_to_file : something went wrong with reading file. #{error}: #{error.backtrace}") end - $log.info("in_kube_podinventory::read_file : read successful. size of hash: #{podInventoryHash.size()}") - return podInventoryHash - end + begin + if !podInventory["items"].nil? && !podInventory["items"].empty? + podInventory["items"].each do |item| + # Extract needed fields using getPodInventoryRecords and create a hash mapping uid -> record + podInventoryRecords = getPodInventoryRecords(item, serviceRecords, batchTime) + podInventoryRecords.each { |record| + uid = record["PodUid"] + podInventoryHash[uid] = record + } + end + end + + $log.info("append_to_file:: podInventoryHash size before write: #{podInventoryHash.size()}") + + # Write to mmap or regular file based on value of @useMmap flag + if @useMmap + $log.info("in_kube_podinventory::append_to_file : writing to mmap file case") + # this is to ensure that we clear file contents before writing to file, check if there is a better way to do this + File.open("/var/opt/microsoft/docker-cimprov/log/testing-podinventory.json", "w") + @mmap = Mmap.new("/var/opt/microsoft/docker-cimprov/log/testing-podinventory.json", "rw") + @mmap << JSON.pretty_generate(podInventoryHash).to_s + else + $log.info("in_kube_podinventory::append_to_file : writing to regular file case") + File.open("/var/opt/microsoft/docker-cimprov/log/testing-podinventory.json", "w") { |file| + file.write(JSON.pretty_generate(podInventoryHash)) + } + end + $log.info("in_kube_podinventory::append_to_file : successfully finished appending to file. size of written file = #{File.size("/var/opt/microsoft/docker-cimprov/log/testing-podinventory.json") / 1000000.0} MB") + rescue => exception + $log.info("in_kube_podinventory::append_to_file : appending to file failed. exception: #{exception} backtrace: #{exception.backtrace}") + end + end - def write_to_file(podInventory, append) + def write_to_file(podInventory) batchTime = Time.now.utc.iso8601 #TODO: check if you can pass @serviceRecords into getPodInventoryRecords rather than creating a local copy #TODO: if no, update code so we simply use instance variable everywhere and there is no need to pass serviceRecords serviceRecords= @serviceRecords podInventoryHash = {} - if append - # have to read file first - # TODO: make podInventoryHash an instance variable so we don't have read everytime - $log.info("write_to_file:: append is true, so will read file first") - podInventoryHash = read_file - end - begin if !podInventory["items"].nil? && !podInventory["items"].empty? podInventory["items"].each do |item| @@ -181,17 +211,17 @@ def write_to_file(podInventory, append) if @useMmap $log.info("in_kube_podinventory::write_to_file : writing to mmap file case") # this is to ensure that we clear file contents before writing to file, check if there is a better way to do this - File.open(@podInventoryFile, "w") - @mmap = Mmap.new(@podInventoryFile, "rw") + File.open("/var/opt/microsoft/docker-cimprov/log/testing-podinventory.json", "w") + @mmap = Mmap.new("/var/opt/microsoft/docker-cimprov/log/testing-podinventory.json", "rw") @mmap << JSON.pretty_generate(podInventoryHash).to_s else $log.info("in_kube_podinventory::write_to_file : writing to regular file case") - File.open(@podInventoryFile, "w") { |file| + File.open("/var/opt/microsoft/docker-cimprov/log/testing-podinventory.json", "w") { |file| file.write(JSON.pretty_generate(podInventoryHash)) } end - $log.info("in_kube_podinventory::write_to_file : successfully finished writing to file. size of written file = #{File.size(@podInventoryFile) / 1000000.0} MB") + $log.info("in_kube_podinventory::write_to_file : successfully finished writing to file. size of written file = #{File.size("/var/opt/microsoft/docker-cimprov/log/testing-podinventory.json") / 1000000.0} MB") rescue => exception $log.info("in_kube_podinventory::write_to_file : writing to file failed. exception: #{exception} backtrace: #{exception.backtrace}") end @@ -203,7 +233,6 @@ def getNoticeRecord(notice) item = notice["object"] #TODO: check assumption that batch time can be current time (CollectionTime) batchTime = Time.now.utc.iso8601 - serviceRecords = @serviceRecords begin record["CollectionTime"] = batchTime @@ -256,10 +285,12 @@ def getNoticeRecord(notice) end record["Computer"] = nodeName + #TODO: replace w KubernetesApiClient.getClusterId in agent code record["ClusterId"] = KubernetesApiClient.getClusterId + #TODO: replace w KubernetesApiClient.getClusterName in agent code record["ClusterName"] = KubernetesApiClient.getClusterName #TODO: make a call to getServiceNameFromLabels -- need to pass in serviceRecords for this - record["ServiceName"] = getServiceNameFromLabels(item["metadata"]["namespace"], item["metadata"]["label"], serviceRecords) + record["ServiceName"] = "" if !item["metadata"]["ownerReferences"].nil? record["ControllerKind"] = item["metadata"]["ownerReferences"][0]["kind"] @@ -295,9 +326,14 @@ def getNoticeRecord(notice) end def watch + # enumerate + + #TODO: Check if watch pods restarts after connection is broken loop do - # $log.info("in_kube_podinventory::watch - inside infinite loop for watch pods. calling enumerate.") + $log.info("in_kube_podinventory::watch - inside infinite loop for watch pods. calling enumerate.") enumerate + + #TODO: check if collection_version is correct when continuation token is not null and collection_version changes $log.info("in_kube_podinventory::watch : inside infinite loop for watch pods. collection version: #{@collection_version}") begin @KubernetesWatchClient.watch_pods(resource_version: @collection_version, as: :parsed) do |notice| @@ -308,6 +344,7 @@ def watch item = notice["object"] # Construct record with necessary fields (same fields as getPodInventoryRecords) record = getNoticeRecord(notice) + $log.info("watch:: record constructed looks like: #{record}") @mutex.synchronize { @@ -327,7 +364,7 @@ def watch # currently sleeping for 30 seconds before restarting $log.info("in_kube_podinventory::watch : makes it to the sleep command. time: #{Time.now.utc.iso8601}") sleep 30 - # $log.info("in_kube_podinventory::watch : after sleep command. time: #{Time.now.utc.iso8601}") + $log.info("in_kube_podinventory::watch : after sleep command. time: #{Time.now.utc.iso8601}") end end @@ -378,7 +415,7 @@ def enumerate(podList = nil) @podsAPIE2ELatencyMs = (podsAPIChunkEndTime - podsAPIChunkStartTime) if (!podInventory.nil? && !podInventory.empty? && podInventory.key?("items") && !podInventory["items"].nil? && !podInventory["items"].empty?) $log.info("in_kube_podinventory::enumerate : number of pod items :#{podInventory["items"].length} from Kube API @ #{Time.now.utc.iso8601}") - write_to_file(podInventory, false) + write_to_file(podInventory) parse_and_emit_records(podInventory, @serviceRecords, continuationToken, batchTime) else $log.warn "in_kube_podinventory::enumerate:Received empty podInventory" @@ -396,7 +433,7 @@ def enumerate(podList = nil) @podsAPIE2ELatencyMs = @podsAPIE2ELatencyMs + (podsAPIChunkEndTime - podsAPIChunkStartTime) if (!podInventory.nil? && !podInventory.empty? && podInventory.key?("items") && !podInventory["items"].nil? && !podInventory["items"].empty?) $log.info("in_kube_podinventory::enumerate : number of pod items :#{podInventory["items"].length} from Kube API @ #{Time.now.utc.iso8601}") - write_to_file(podInventory, true) + append_to_file(podInventory) parse_and_emit_records(podInventory, @serviceRecords, continuationToken, batchTime) else $log.warn "in_kube_podinventory::enumerate:Received empty podInventory" @@ -672,32 +709,29 @@ def parse_and_emit_merge_updates(podInventoryRecords) end def merge_updates - # startTime = Time.now - # $log.info("merge_updates:: Start time: #{startTime}") + startTime = Time.now + $log.info("merge_updates:: Start time: #{startTime}") podInventoryHash = {} shouldUpdateFile = false - # read file - podInventoryHash = read_file - $log.info("merge_updates:: podInventoryHash size : #{podInventoryHash.size()}") - # begin - # fileContents = "" - # # Read file - # if @useMmap - # fileContents = fileContents.dup if fileContents.frozen? - # fileContents << @mmap - # else - # # define path above instead of hardcoding here - # fileContents = File.read("/var/opt/microsoft/docker-cimprov/log/testing-podinventory.json") - # end - # $log.info("in_kube_podinventory::merge_updates : file contents read") - # if !fileContents.empty? - # podInventoryHash = Yajl::Parser.parse(fileContents) - # $log.info("in_kube_podinventory::merge_updates : parse successful. size of hash: #{podInventoryHash.size()}") - # end - # rescue => error - # $log.info("in_kube_podinventory::merge_updates : something went wrong with reading file. #{error}: #{error.backtrace}") - # end + begin + fileContents = "" + # Read file + if @useMmap + fileContents = fileContents.dup if fileContents.frozen? + fileContents << @mmap + else + # define path above instead of hardcoding here + fileContents = File.read("/var/opt/microsoft/docker-cimprov/log/testing-podinventory.json") + end + $log.info("in_kube_podinventory::merge_updates : file contents read") + if !fileContents.empty? + podInventoryHash = Yajl::Parser.parse(fileContents) + $log.info("in_kube_podinventory::merge_updates : parse successful. size of hash: #{podInventoryHash.size()}") + end + rescue => error + $log.info("in_kube_podinventory::merge_updates : something went wrong with reading file. #{error}: #{error.backtrace}") + end $log.info("in_kube_podinventory::merge_updates : before noticeHash loop, number of items in hash: #{@noticeHash.size()}") @@ -724,13 +758,11 @@ def merge_updates else # TODO: will need to modify other fields later $log.info("in_kube_podinventory::merge_updates : modify case where it is only a modification. old status: #{podInventoryHash[uid]["PodStatus"]}. new status: #{record["PodStatus"]}") - # val = podInventoryHash[uid] - # val["PodStatus"] = record["PodStatus"] - # podInventoryHash[uid] = val - - # TODO: only update modified fields or update everything? - podInventoryHash[uid] = record + val = podInventoryHash[uid] + val["PodStatus"] = record["PodStatus"] + podInventoryHash[uid] = val end + $log.info("in_kube_podinventory::merge_updates :: modified and changes reflected in podInventoryHash") when "DELETED" if podInventoryHash.key?(uid) podInventoryHash.delete(uid) @@ -760,23 +792,23 @@ def merge_updates # only write if there is a change if shouldUpdateFile $log.info("in_kube_podinventory:: merge_updates : shouldUpdateFile evals to true, therefore writing to file.") - write_to_file(podInventoryHash, false) + write_to_file(podInventoryHash) end $log.info("merge_updates:: number of items in podInventoryHash: #{podInventoryHash.length}") # $log.info("merge_updates:: number of items in podInventoryHash: #{@podInventoryHash.length}. podInventoryHash: #{podInventoryHash}") parse_and_emit_merge_updates(podInventoryHash) #TODO: bottom two are not necessary - can remove later - # podInventoryHash.clear - # $log.info("merge_updates:: number of items in podInventoryHash after clear: #{podInventoryHash.length}") + podInventoryHash.clear + $log.info("merge_updates:: number of items in podInventoryHash after clear: #{podInventoryHash.length}") else $log.info("in_kube_podinventory:: merge_updates : podInventoryHash was either null or empty, so NOT writing to file - should never be in this case") end $log.info("in_kube_podinventory:: merge_updates : finished replacing contents of testing-podinventory.json") - # endTime = Time.now - # $log.info("merge_updates:: End time: #{endTime}") - # $log.info("merge_updates:: total time taken = #{endTime - startTime}") + endTime = Time.now + $log.info("merge_updates:: End time: #{endTime}") + $log.info("merge_updates:: total time taken = #{endTime - startTime}") end def run_periodic @@ -1057,4 +1089,4 @@ def getServiceNameFromLabels(namespace, labels, serviceRecords) return serviceName end end # Kube_Pod_Input -end # module +end # module \ No newline at end of file From bb52ba403ced2ea3bff5dc6fb18c731b6abcb4be Mon Sep 17 00:00:00 2001 From: Khushi Chaudhari Date: Fri, 13 Aug 2021 11:23:56 -0700 Subject: [PATCH 62/76] testing w local kubeclient gem for tcp timeout --- kubernetes/linux/setup.sh | 3 ++- source/plugins/ruby/in_kube_podinventory.rb | 5 ++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/kubernetes/linux/setup.sh b/kubernetes/linux/setup.sh index 22cf50280..c79328584 100644 --- a/kubernetes/linux/setup.sh +++ b/kubernetes/linux/setup.sh @@ -54,7 +54,8 @@ fluentd --setup ./fluent gem install gyoku iso8601 --no-doc # kubeclient gem sudo apt-get install libmagickwand-dev -y -sudo gem install kubeclient --no-document +# sudo gem install kubeclient --no-document +sudo gem install /home/khushic/kubeclient/kubeclient-4.9.2.gem # mmap2 gem sudo gem install mmap2 diff --git a/source/plugins/ruby/in_kube_podinventory.rb b/source/plugins/ruby/in_kube_podinventory.rb index ad048a9a1..efff8f5f3 100644 --- a/source/plugins/ruby/in_kube_podinventory.rb +++ b/source/plugins/ruby/in_kube_podinventory.rb @@ -97,6 +97,9 @@ def start open: 60, # default setting (in seconds) read: nil # read will never timeout } + # socket_options = { + + # } getTokenStr = "Bearer " + KubernetesApiClient.getTokenStr auth_options = { bearer_token: KubernetesApiClient.getTokenStr } @KubernetesWatchClient = Kubeclient::Client.new("https://#{ENV["KUBERNETES_SERVICE_HOST"]}:#{ENV["KUBERNETES_PORT_443_TCP_PORT"]}/api/", "v1", ssl_options: ssl_options, auth_options: auth_options, as: :parsed, timeouts: timeouts) @@ -260,7 +263,7 @@ def getNoticeRecord(notice) #podStatus # NodeLost scenario -- pod(s) in the lost node is still being reported as running - podReadyCondition = true + podReadyCondition = true if !item["status"]["reason"].nil? && item["status"]["reason"] == "NodeLost" && !item["status"]["conditions"].nil? item["status"]["conditions"].each do |condition| if condition["type"] == "Ready" && condition["status"] == "False" From fc326f24b0ea874a5f5e03c3720a3de02e0dcc5e Mon Sep 17 00:00:00 2001 From: Khushi Chaudhari Date: Fri, 13 Aug 2021 13:33:47 -0700 Subject: [PATCH 63/76] updated dockerfile to support setup.sh changes for local kubeclient gem --- kubernetes/linux/Dockerfile | 2 +- kubernetes/linux/setup.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/kubernetes/linux/Dockerfile b/kubernetes/linux/Dockerfile index 1ae7bef61..b0ceda4ae 100644 --- a/kubernetes/linux/Dockerfile +++ b/kubernetes/linux/Dockerfile @@ -17,7 +17,7 @@ ENV KUBE_CLIENT_BACKOFF_BASE 1 ENV KUBE_CLIENT_BACKOFF_DURATION 0 ENV RUBY_GC_HEAP_OLDOBJECT_LIMIT_FACTOR 0.9 RUN /usr/bin/apt-get update && /usr/bin/apt-get install -y libc-bin wget openssl curl sudo python-ctypes init-system-helpers net-tools rsyslog cron vim dmidecode apt-transport-https gnupg && rm -rf /var/lib/apt/lists/* -COPY setup.sh main.sh defaultpromenvvariables defaultpromenvvariables-rs defaultpromenvvariables-sidecar mdsd.xml envmdsd $tmpdir/ +COPY kubeclient-4.9.2.gem setup.sh main.sh defaultpromenvvariables defaultpromenvvariables-rs defaultpromenvvariables-sidecar mdsd.xml envmdsd $tmpdir/ WORKDIR ${tmpdir} # copy docker provider shell bundle to use the agent image diff --git a/kubernetes/linux/setup.sh b/kubernetes/linux/setup.sh index c79328584..f125b81ef 100644 --- a/kubernetes/linux/setup.sh +++ b/kubernetes/linux/setup.sh @@ -55,7 +55,7 @@ gem install gyoku iso8601 --no-doc # kubeclient gem sudo apt-get install libmagickwand-dev -y # sudo gem install kubeclient --no-document -sudo gem install /home/khushic/kubeclient/kubeclient-4.9.2.gem +sudo gem install $TMPDIR/kubeclient-4.9.2.gem -y # mmap2 gem sudo gem install mmap2 From b5ebecb8b2e475634cb85a3e7958727028cbbdd5 Mon Sep 17 00:00:00 2001 From: Khushi Chaudhari Date: Tue, 17 Aug 2021 12:08:53 -0700 Subject: [PATCH 64/76] read timeout is 4 mins --- build/linux/installer/datafiles/base_container.data | 2 ++ kubernetes/linux/setup.sh | 4 ++-- source/plugins/ruby/in_kube_podinventory.rb | 5 +---- 3 files changed, 5 insertions(+), 6 deletions(-) diff --git a/build/linux/installer/datafiles/base_container.data b/build/linux/installer/datafiles/base_container.data index bdacf647d..fc50075ea 100644 --- a/build/linux/installer/datafiles/base_container.data +++ b/build/linux/installer/datafiles/base_container.data @@ -287,6 +287,8 @@ chmod 666 /var/opt/microsoft/docker-cimprov/log/arc_k8s_cluster_identity.log touch /var/opt/microsoft/docker-cimprov/log/fluentd.log chmod 666 /var/opt/microsoft/docker-cimprov/log/fluentd.log +touch /var/opt/microsoft/docker-cimprov/log/testing-podinventory.json +chmod 666 /var/opt/microsoft/docker-cimprov/log/testing-podinventory.json %Postuninstall_10 # If we're an upgrade, skip all of this cleanup diff --git a/kubernetes/linux/setup.sh b/kubernetes/linux/setup.sh index f125b81ef..5d1c2bc3f 100644 --- a/kubernetes/linux/setup.sh +++ b/kubernetes/linux/setup.sh @@ -54,8 +54,8 @@ fluentd --setup ./fluent gem install gyoku iso8601 --no-doc # kubeclient gem sudo apt-get install libmagickwand-dev -y -# sudo gem install kubeclient --no-document -sudo gem install $TMPDIR/kubeclient-4.9.2.gem -y +sudo gem install kubeclient --no-document +# sudo gem install kubeclient-4.9.2.gem # mmap2 gem sudo gem install mmap2 diff --git a/source/plugins/ruby/in_kube_podinventory.rb b/source/plugins/ruby/in_kube_podinventory.rb index efff8f5f3..3880f8ea7 100644 --- a/source/plugins/ruby/in_kube_podinventory.rb +++ b/source/plugins/ruby/in_kube_podinventory.rb @@ -95,11 +95,8 @@ def start } timeouts = { open: 60, # default setting (in seconds) - read: nil # read will never timeout + read: 240 # read timeout = 4 min } - # socket_options = { - - # } getTokenStr = "Bearer " + KubernetesApiClient.getTokenStr auth_options = { bearer_token: KubernetesApiClient.getTokenStr } @KubernetesWatchClient = Kubeclient::Client.new("https://#{ENV["KUBERNETES_SERVICE_HOST"]}:#{ENV["KUBERNETES_PORT_443_TCP_PORT"]}/api/", "v1", ssl_options: ssl_options, auth_options: auth_options, as: :parsed, timeouts: timeouts) From 430d859ed8a26103c87ef30ee0502d75bb038218 Mon Sep 17 00:00:00 2001 From: Khushi Chaudhari Date: Wed, 18 Aug 2021 09:29:46 -0700 Subject: [PATCH 65/76] watch restarts every 25 minutes --- source/plugins/ruby/in_kube_podinventory.rb | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/source/plugins/ruby/in_kube_podinventory.rb b/source/plugins/ruby/in_kube_podinventory.rb index 3880f8ea7..ca41846ff 100644 --- a/source/plugins/ruby/in_kube_podinventory.rb +++ b/source/plugins/ruby/in_kube_podinventory.rb @@ -45,7 +45,6 @@ def initialize @podsAPIE2ELatencyMs = 0 @noticeHash = {} - # @podInventoryHash = {} @useMmap = false @collection_version = "" @@ -95,11 +94,12 @@ def start } timeouts = { open: 60, # default setting (in seconds) - read: 240 # read timeout = 4 min + read: nil # read never times out } getTokenStr = "Bearer " + KubernetesApiClient.getTokenStr auth_options = { bearer_token: KubernetesApiClient.getTokenStr } @KubernetesWatchClient = Kubeclient::Client.new("https://#{ENV["KUBERNETES_SERVICE_HOST"]}:#{ENV["KUBERNETES_PORT_443_TCP_PORT"]}/api/", "v1", ssl_options: ssl_options, auth_options: auth_options, as: :parsed, timeouts: timeouts) + @Watcher = nil $log.info("in_kube:podinventory::start: successfully created kubernetes watch client") $log.info("in_kube_podinventory::start: PODS_EMIT_STREAM_BATCH_SIZE @ #{@PODS_EMIT_STREAM_BATCH_SIZE}") @finished = false @@ -108,6 +108,7 @@ def start @watchthread = Thread.new(&method(:watch)) @thread = Thread.new(&method(:run_periodic)) @@podTelemetryTimeTracker = DateTime.now.to_time.to_i + @@WatcherTimeTracker = DateTime.now.to_time.to_i end end @@ -336,7 +337,8 @@ def watch #TODO: check if collection_version is correct when continuation token is not null and collection_version changes $log.info("in_kube_podinventory::watch : inside infinite loop for watch pods. collection version: #{@collection_version}") begin - @KubernetesWatchClient.watch_pods(resource_version: @collection_version, as: :parsed) do |notice| + @Watcher = @KubernetesWatchClient.watch_pods(resource_version: @collection_version, as: :parsed) + @Watcher.each do |notice| $log.info("in_kube_podinventory::watch : inside watch pods! collection version: #{@collection_version}.") if !notice.nil? && !notice.empty? $log.info("in_kube_podinventory::watch : received a notice that is not null and not empty. notice type: #{notice["type"]}") @@ -670,6 +672,16 @@ def parse_and_emit_merge_updates(podInventoryRecords) emittedPodCount = 0 begin #begin block start + timeDifference = (DateTime.now.to_time.to_i - @@WatcherTimeTracker).abs + timeDifferenceInMinutes = timeDifference / 60 + if (timeDifferenceInMinutes >= 25) + $log.info("parse_and_emit_merge_updates::resetting watcher to handle api server timeout :#{Time.now.utc.iso8601}") + @@WatcherTimeTracker = DateTime.now.to_time.to_i + if !@Watcher.nil? + @Watcher.finish + end + end + # Getting windows nodes from kubeapi winNodes = KubernetesApiClient.getWindowsNodesArray podInventoryRecords.each do |uid, record| From ae90e21da371aabcbed35a27546644e7270e77ff Mon Sep 17 00:00:00 2001 From: Khushi Chaudhari Date: Wed, 18 Aug 2021 11:01:56 -0700 Subject: [PATCH 66/76] changed sleep for 30 to 1 s --- source/plugins/ruby/in_kube_podinventory.rb | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/source/plugins/ruby/in_kube_podinventory.rb b/source/plugins/ruby/in_kube_podinventory.rb index ca41846ff..e3e5af662 100644 --- a/source/plugins/ruby/in_kube_podinventory.rb +++ b/source/plugins/ruby/in_kube_podinventory.rb @@ -362,10 +362,8 @@ def watch $log.warn("in_kube_podinventory::watch : watch events session got broken and re-establishing the session. backtrace: #{exception.backtrace}") # $log.debug_backtrace(exception.backtrace) end - #TODO: check if 30 is the correct number to use here - # currently sleeping for 30 seconds before restarting $log.info("in_kube_podinventory::watch : makes it to the sleep command. time: #{Time.now.utc.iso8601}") - sleep 30 + sleep 1 $log.info("in_kube_podinventory::watch : after sleep command. time: #{Time.now.utc.iso8601}") end end From 08ee87aa3b28eaabeffac3551c41cb92ab964c02 Mon Sep 17 00:00:00 2001 From: Khushi Chaudhari Date: Wed, 18 Aug 2021 11:09:08 -0700 Subject: [PATCH 67/76] eval in-mem approach --- source/plugins/ruby/in_kube_podinventory.rb | 175 ++++++++++---------- 1 file changed, 88 insertions(+), 87 deletions(-) diff --git a/source/plugins/ruby/in_kube_podinventory.rb b/source/plugins/ruby/in_kube_podinventory.rb index e3e5af662..154171f2c 100644 --- a/source/plugins/ruby/in_kube_podinventory.rb +++ b/source/plugins/ruby/in_kube_podinventory.rb @@ -47,6 +47,7 @@ def initialize @noticeHash = {} @useMmap = false @collection_version = "" + @podInventoryHash = {} @kubeperfTag = "oneagent.containerInsights.LINUX_PERF_BLOB" @kubeservicesTag = "oneagent.containerInsights.KUBE_SERVICES_BLOB" @@ -128,28 +129,28 @@ def append_to_file(podInventory) # only to be called from enumerate continuation token batchTime = Time.now.utc.iso8601 serviceRecords = @serviceRecords - podInventoryHash = {} + # podInventoryHash = {} # have to read file first # TODO: make podInventoryHash an instance variable so we don't have read everytime - begin - fileContents = "" - # Read file - if @useMmap - fileContents = fileContents.dup if fileContents.frozen? - fileContents << @mmap - else - # define path above instead of hardcoding here - fileContents = File.read("/var/opt/microsoft/docker-cimprov/log/testing-podinventory.json") - end - $log.info("in_kube_podinventory::append_to_file : file contents read") - if !fileContents.empty? - podInventoryHash = Yajl::Parser.parse(fileContents) - $log.info("in_kube_podinventory::append_to_file : parse successful. size of hash: #{podInventoryHash.size()}") - end - rescue => error - $log.info("in_kube_podinventory::append_to_file : something went wrong with reading file. #{error}: #{error.backtrace}") - end + # begin + # fileContents = "" + # # Read file + # if @useMmap + # fileContents = fileContents.dup if fileContents.frozen? + # fileContents << @mmap + # else + # # define path above instead of hardcoding here + # fileContents = File.read("/var/opt/microsoft/docker-cimprov/log/testing-podinventory.json") + # end + # $log.info("in_kube_podinventory::append_to_file : file contents read") + # if !fileContents.empty? + # podInventoryHash = Yajl::Parser.parse(fileContents) + # $log.info("in_kube_podinventory::append_to_file : parse successful. size of hash: #{podInventoryHash.size()}") + # end + # rescue => error + # $log.info("in_kube_podinventory::append_to_file : something went wrong with reading file. #{error}: #{error.backtrace}") + # end begin if !podInventory["items"].nil? && !podInventory["items"].empty? @@ -158,28 +159,28 @@ def append_to_file(podInventory) podInventoryRecords = getPodInventoryRecords(item, serviceRecords, batchTime) podInventoryRecords.each { |record| uid = record["PodUid"] - podInventoryHash[uid] = record + @podInventoryHash[uid] = record } end end - $log.info("append_to_file:: podInventoryHash size before write: #{podInventoryHash.size()}") + $log.info("append_to_file:: podInventoryHash size before write: #{@podInventoryHash.size()}") # Write to mmap or regular file based on value of @useMmap flag - if @useMmap - $log.info("in_kube_podinventory::append_to_file : writing to mmap file case") - # this is to ensure that we clear file contents before writing to file, check if there is a better way to do this - File.open("/var/opt/microsoft/docker-cimprov/log/testing-podinventory.json", "w") - @mmap = Mmap.new("/var/opt/microsoft/docker-cimprov/log/testing-podinventory.json", "rw") - @mmap << JSON.pretty_generate(podInventoryHash).to_s - else - $log.info("in_kube_podinventory::append_to_file : writing to regular file case") - File.open("/var/opt/microsoft/docker-cimprov/log/testing-podinventory.json", "w") { |file| - file.write(JSON.pretty_generate(podInventoryHash)) - } - end - - $log.info("in_kube_podinventory::append_to_file : successfully finished appending to file. size of written file = #{File.size("/var/opt/microsoft/docker-cimprov/log/testing-podinventory.json") / 1000000.0} MB") + # if @useMmap + # $log.info("in_kube_podinventory::append_to_file : writing to mmap file case") + # # this is to ensure that we clear file contents before writing to file, check if there is a better way to do this + # File.open("/var/opt/microsoft/docker-cimprov/log/testing-podinventory.json", "w") + # @mmap = Mmap.new("/var/opt/microsoft/docker-cimprov/log/testing-podinventory.json", "rw") + # @mmap << JSON.pretty_generate(podInventoryHash).to_s + # else + # $log.info("in_kube_podinventory::append_to_file : writing to regular file case") + # File.open("/var/opt/microsoft/docker-cimprov/log/testing-podinventory.json", "w") { |file| + # file.write(JSON.pretty_generate(podInventoryHash)) + # } + # end + + # $log.info("in_kube_podinventory::append_to_file : successfully finished appending to file. size of written file = #{File.size("/var/opt/microsoft/docker-cimprov/log/testing-podinventory.json") / 1000000.0} MB") rescue => exception $log.info("in_kube_podinventory::append_to_file : appending to file failed. exception: #{exception} backtrace: #{exception.backtrace}") end @@ -190,7 +191,7 @@ def write_to_file(podInventory) #TODO: check if you can pass @serviceRecords into getPodInventoryRecords rather than creating a local copy #TODO: if no, update code so we simply use instance variable everywhere and there is no need to pass serviceRecords serviceRecords= @serviceRecords - podInventoryHash = {} + # podInventoryHash = {} begin if !podInventory["items"].nil? && !podInventory["items"].empty? @@ -199,30 +200,30 @@ def write_to_file(podInventory) podInventoryRecords = getPodInventoryRecords(item, serviceRecords, batchTime) podInventoryRecords.each { |record| uid = record["PodUid"] - podInventoryHash[uid] = record + @podInventoryHash[uid] = record } end else - podInventoryHash = podInventory + @podInventoryHash = podInventory end - $log.info("write_to_file:: podInventoryHash size before write: #{podInventoryHash.size()}") + $log.info("write_to_file:: podInventoryHash size before write: #{@podInventoryHash.size()}") # Write to mmap or regular file based on value of @useMmap flag - if @useMmap - $log.info("in_kube_podinventory::write_to_file : writing to mmap file case") - # this is to ensure that we clear file contents before writing to file, check if there is a better way to do this - File.open("/var/opt/microsoft/docker-cimprov/log/testing-podinventory.json", "w") - @mmap = Mmap.new("/var/opt/microsoft/docker-cimprov/log/testing-podinventory.json", "rw") - @mmap << JSON.pretty_generate(podInventoryHash).to_s - else - $log.info("in_kube_podinventory::write_to_file : writing to regular file case") - File.open("/var/opt/microsoft/docker-cimprov/log/testing-podinventory.json", "w") { |file| - file.write(JSON.pretty_generate(podInventoryHash)) - } - end - - $log.info("in_kube_podinventory::write_to_file : successfully finished writing to file. size of written file = #{File.size("/var/opt/microsoft/docker-cimprov/log/testing-podinventory.json") / 1000000.0} MB") + # if @useMmap + # $log.info("in_kube_podinventory::write_to_file : writing to mmap file case") + # # this is to ensure that we clear file contents before writing to file, check if there is a better way to do this + # File.open("/var/opt/microsoft/docker-cimprov/log/testing-podinventory.json", "w") + # @mmap = Mmap.new("/var/opt/microsoft/docker-cimprov/log/testing-podinventory.json", "rw") + # @mmap << JSON.pretty_generate(podInventoryHash).to_s + # else + # $log.info("in_kube_podinventory::write_to_file : writing to regular file case") + # File.open("/var/opt/microsoft/docker-cimprov/log/testing-podinventory.json", "w") { |file| + # file.write(JSON.pretty_generate(podInventoryHash)) + # } + # end + + # $log.info("in_kube_podinventory::write_to_file : successfully finished writing to file. size of written file = #{File.size("/var/opt/microsoft/docker-cimprov/log/testing-podinventory.json") / 1000000.0} MB") rescue => exception $log.info("in_kube_podinventory::write_to_file : writing to file failed. exception: #{exception} backtrace: #{exception.backtrace}") end @@ -721,27 +722,27 @@ def parse_and_emit_merge_updates(podInventoryRecords) def merge_updates startTime = Time.now $log.info("merge_updates:: Start time: #{startTime}") - podInventoryHash = {} + # podInventoryHash = {} shouldUpdateFile = false - begin - fileContents = "" - # Read file - if @useMmap - fileContents = fileContents.dup if fileContents.frozen? - fileContents << @mmap - else - # define path above instead of hardcoding here - fileContents = File.read("/var/opt/microsoft/docker-cimprov/log/testing-podinventory.json") - end - $log.info("in_kube_podinventory::merge_updates : file contents read") - if !fileContents.empty? - podInventoryHash = Yajl::Parser.parse(fileContents) - $log.info("in_kube_podinventory::merge_updates : parse successful. size of hash: #{podInventoryHash.size()}") - end - rescue => error - $log.info("in_kube_podinventory::merge_updates : something went wrong with reading file. #{error}: #{error.backtrace}") - end + # begin + # fileContents = "" + # # Read file + # if @useMmap + # fileContents = fileContents.dup if fileContents.frozen? + # fileContents << @mmap + # else + # # define path above instead of hardcoding here + # fileContents = File.read("/var/opt/microsoft/docker-cimprov/log/testing-podinventory.json") + # end + # $log.info("in_kube_podinventory::merge_updates : file contents read") + # if !fileContents.empty? + # podInventoryHash = Yajl::Parser.parse(fileContents) + # $log.info("in_kube_podinventory::merge_updates : parse successful. size of hash: #{podInventoryHash.size()}") + # end + # rescue => error + # $log.info("in_kube_podinventory::merge_updates : something went wrong with reading file. #{error}: #{error.backtrace}") + # end $log.info("in_kube_podinventory::merge_updates : before noticeHash loop, number of items in hash: #{@noticeHash.size()}") @@ -759,23 +760,23 @@ def merge_updates case record["NoticeType"] when "ADDED" - podInventoryHash[uid] = record + @podInventoryHash[uid] = record $log.info("in_kube_podinventory::merge_updates : added new record to podInventoryHash") when "MODIFIED" - if podInventoryHash[uid].nil? + if @podInventoryHash[uid].nil? $log.info("in_kube_podinventory::merge_updates : modify case where uid for add was overwritten to modify within same minute") - podInventoryHash[uid] = record + @podInventoryHash[uid] = record else # TODO: will need to modify other fields later - $log.info("in_kube_podinventory::merge_updates : modify case where it is only a modification. old status: #{podInventoryHash[uid]["PodStatus"]}. new status: #{record["PodStatus"]}") - val = podInventoryHash[uid] + $log.info("in_kube_podinventory::merge_updates : modify case where it is only a modification. old status: #{@podInventoryHash[uid]["PodStatus"]}. new status: #{record["PodStatus"]}") + val = @podInventoryHash[uid] val["PodStatus"] = record["PodStatus"] - podInventoryHash[uid] = val + @podInventoryHash[uid] = val end $log.info("in_kube_podinventory::merge_updates :: modified and changes reflected in podInventoryHash") when "DELETED" - if podInventoryHash.key?(uid) - podInventoryHash.delete(uid) + if @podInventoryHash.key?(uid) + @podInventoryHash.delete(uid) $log.info("in_kube_podinventory::merge_updates :: deleted from podInventoryHash") else $log.info("merge_updates:: error: key did not exist in hash so unable to delete (probably add and delete in same min)") @@ -797,20 +798,20 @@ def merge_updates #TODO: Look for a way to replace only necessary contents, rather than everything $log.info("in_kube_podinventory:: merge_updates : about to replace entire contents of testing-podinventory.json") - if (!podInventoryHash.nil? && !podInventoryHash.empty?) - $log.info("in_kube_podinventory:: merge_updates : podInventoryHash not null and not empty. podInventoryHash size after hash loop: #{podInventoryHash.size()}") + if (!@podInventoryHash.nil? && !@podInventoryHash.empty?) + $log.info("in_kube_podinventory:: merge_updates : podInventoryHash not null and not empty. podInventoryHash size after hash loop: #{@podInventoryHash.size()}") # only write if there is a change if shouldUpdateFile $log.info("in_kube_podinventory:: merge_updates : shouldUpdateFile evals to true, therefore writing to file.") - write_to_file(podInventoryHash) + write_to_file(@podInventoryHash) end - $log.info("merge_updates:: number of items in podInventoryHash: #{podInventoryHash.length}") + $log.info("merge_updates:: number of items in podInventoryHash: #{@podInventoryHash.length}") # $log.info("merge_updates:: number of items in podInventoryHash: #{@podInventoryHash.length}. podInventoryHash: #{podInventoryHash}") - parse_and_emit_merge_updates(podInventoryHash) + parse_and_emit_merge_updates(@podInventoryHash) #TODO: bottom two are not necessary - can remove later - podInventoryHash.clear - $log.info("merge_updates:: number of items in podInventoryHash after clear: #{podInventoryHash.length}") + @podInventoryHash.clear + $log.info("merge_updates:: number of items in podInventoryHash after clear: #{@podInventoryHash.length}") else $log.info("in_kube_podinventory:: merge_updates : podInventoryHash was either null or empty, so NOT writing to file - should never be in this case") end From 7ed7cd41b093c74509b62b66b65ed184a6fd0b34 Mon Sep 17 00:00:00 2001 From: Khushi Chaudhari Date: Fri, 20 Aug 2021 14:13:23 -0700 Subject: [PATCH 68/76] changing setup.sh to use local kubeclient gem and remove podInvHash clear --- kubernetes/linux/setup.sh | 4 ++-- source/plugins/ruby/in_kube_podinventory.rb | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/kubernetes/linux/setup.sh b/kubernetes/linux/setup.sh index 5d1c2bc3f..1d3b68521 100644 --- a/kubernetes/linux/setup.sh +++ b/kubernetes/linux/setup.sh @@ -54,8 +54,8 @@ fluentd --setup ./fluent gem install gyoku iso8601 --no-doc # kubeclient gem sudo apt-get install libmagickwand-dev -y -sudo gem install kubeclient --no-document -# sudo gem install kubeclient-4.9.2.gem +# sudo gem install kubeclient --no-document +sudo gem install kubeclient-4.9.2.gem # mmap2 gem sudo gem install mmap2 diff --git a/source/plugins/ruby/in_kube_podinventory.rb b/source/plugins/ruby/in_kube_podinventory.rb index 154171f2c..3faf752b6 100644 --- a/source/plugins/ruby/in_kube_podinventory.rb +++ b/source/plugins/ruby/in_kube_podinventory.rb @@ -810,8 +810,8 @@ def merge_updates parse_and_emit_merge_updates(@podInventoryHash) #TODO: bottom two are not necessary - can remove later - @podInventoryHash.clear - $log.info("merge_updates:: number of items in podInventoryHash after clear: #{@podInventoryHash.length}") + # @podInventoryHash.clear + # $log.info("merge_updates:: number of items in podInventoryHash after clear: #{@podInventoryHash.length}") else $log.info("in_kube_podinventory:: merge_updates : podInventoryHash was either null or empty, so NOT writing to file - should never be in this case") end From 2c962f9880ca05c3e1ab64c4ae8dcd52d118f5c9 Mon Sep 17 00:00:00 2001 From: Khushi Chaudhari Date: Sun, 22 Aug 2021 17:06:01 -0700 Subject: [PATCH 69/76] adding timeout seconds to watch call --- source/plugins/ruby/in_kube_podinventory.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/plugins/ruby/in_kube_podinventory.rb b/source/plugins/ruby/in_kube_podinventory.rb index 3faf752b6..7ceff9c6c 100644 --- a/source/plugins/ruby/in_kube_podinventory.rb +++ b/source/plugins/ruby/in_kube_podinventory.rb @@ -338,7 +338,7 @@ def watch #TODO: check if collection_version is correct when continuation token is not null and collection_version changes $log.info("in_kube_podinventory::watch : inside infinite loop for watch pods. collection version: #{@collection_version}") begin - @Watcher = @KubernetesWatchClient.watch_pods(resource_version: @collection_version, as: :parsed) + @Watcher = @KubernetesWatchClient.watch_pods(resource_version: @collection_version, timeoutSeconds: 380, as: :parsed) @Watcher.each do |notice| $log.info("in_kube_podinventory::watch : inside watch pods! collection version: #{@collection_version}.") if !notice.nil? && !notice.empty? From e6ef54afb000c44a142d3de8038551262d5e269a Mon Sep 17 00:00:00 2001 From: Khushi Chaudhari Date: Mon, 23 Aug 2021 17:00:00 -0700 Subject: [PATCH 70/76] cleaned up changes --- kubernetes/linux/kubeclient-4.9.2.gem | Bin 0 -> 35840 bytes source/plugins/ruby/in_kube_podinventory.rb | 223 ++------------------ 2 files changed, 20 insertions(+), 203 deletions(-) create mode 100644 kubernetes/linux/kubeclient-4.9.2.gem diff --git a/kubernetes/linux/kubeclient-4.9.2.gem b/kubernetes/linux/kubeclient-4.9.2.gem new file mode 100644 index 0000000000000000000000000000000000000000..84f2aaba171904dd94cb52b8654ee08d8abab947 GIT binary patch literal 35840 zcmeFXQ*bU!@IIIm+jx^VwrwXTwr$(CZQCcdZQFKoVxHLh?e4{H{ciT^U%MCEH$7cF zGu1uQ^;B2aV{7VSXkzGMNN?^1^1n(L{}WbLR*?T)|0n-bGjXu9fH1K#F)^~SGqbV% zw~vX1m5mXEknw+2K>zo9U7cMFo&F2S-NMw==Kt>aAM5`o|NoiW|CQYTIR5`tlOP%z z#O5!tAt>0j=9b+NJF@SZdb#HY#YFlQX3aeP>JHmt!3Memrmuw}GCEh1=0j1E3U2ZG z*E1s7MkJ8J@mXm^G#7+@0(&=5H~6S zkkvio!2GTb9Z686XT@TVzUMPC;4^flc9`7ri|2f@V8V|RpSa65@xet3*L=uFT&sIs zi}($Z6>J8YrP21QPgA=Z*|}@6dsQcB#4TLppvaxh&5bW+RQm)#_elEc{`l{sxD+Mp zg=w{yy_*mCM_*vbuCEeVW%K~Gq@vRx6#@>me5f!;3TiDTnGV1=u$ycpjdi!-@qPD? z`QM|czAmm!JMu96*3aepK9p%~1X)r#26;(z>cg`8u~Cld|t&t)uh-PS2_hl>}z^i4u-t!1k^y zj4DFrxVtJZuE>A=7@b^F{97h%uIY}CqksFO@?}cZ{o>Z_?dFKwb3woKxz7$UPE+x% z)Ec#IWTc&u{kvjzz^dQdg*RKTJKBgdqS;bQik2e=-N<&yZ}2=$X@3wHO_3~|A}%XW_1QfOC2foq$|}uKKwIE)E;SJw zO4-sd&))U1;Pv?wpZt(~zA;BpakwtREX?`o;`Z+4<;NuZl{&U+!|nOtwz)Z7YQ8iv za_)J`Tnu;SQwvvT2{WnMU#}eH<6zu>()JGNat=+4YdDVH3Sn2A>vlcpjtIfgN37Jo zLj-z{G^^hj6>LMobmJjfIN6X=w*;?oGQL#L{Bwa7$RY2>O=>RQ;89n~!GQ!{SKN z&?>~M%`rWJXDn^GiEDsCy?`1p9UBytmiyUh!)Vb&YO-H2GXcCo;m4jHU?17d>y1YD z;3fCW8u9Omr#|9E+4Q6)Q{2~Kv0pAJw*)VgU6kvDX6Cau6OOr*lMC8S@b-&d#~k#Z z+)sn%uhz@vJVGC_28u{b`4dQ#r`Y;qfsfh^2=&OtlZ0EXT>k@~oB4k*FI( zZm!@v;acTz96#FYsW<(S!1*6Q^nYOge+c;h9{zK%aj>xdPyA=$;Q0UX|9^bd{}S{6 z!vCutZ*8D>D+90prSfvW45_EfUo&lQT&Uoy`rtq?&P?1eIyNORJ_)N_JDBSAt5ctd z7qs5|Z+wEno{%#CiRd|s-5`DnM)A5Oe@@Z5<%{-u7rno1u82M~Sj8IZxfK9Qxx?NYi`0=};W zVfcknFD;B}FE5{LltXir3ls< zDPe4`$BIjJ|BGUmL&>eBB0I8l&lyARmp^8Asp3gm;ka&Zmc=W1S`#DoGIVRo&_r|I zD=6)O;mOhafp1F|eG`A(D{H*@`;Ve<%#r_eCZK2#~#wV75Zic zbFmXW+e-+}%f*zlj5gjYN07kHks5MD%hytlBX ze@9+VqP}Ct{Oa)EIIo!aL=`cEI;(M_O>MQ{b=jI_3fRSdo!-Xgzn~hEu;!?k#oNn7uF3sAr`_{g*%ee&2zypbTU4{GN_t1#3q-CoxB-Nx63#I(IR>nodR z+nDc<+z0<@W$>aP4L?a)e=!@*SbmYbk$WJ6(-iwPEnL%_|1%`;Y2Jgyo;v#*%449leCb}Gm^9i&+eN097AbczZ_Bjkv zB-B1_ULy6DQKE?R^b<9j$!r;7 zqm7kMfD&(B2c|EDUdjomUE7!5tBz}_qnjum8Vj*M8KELrnFx4c(PH`6 z_UtFdcod7{){PIzX((DBN2~?RrF{=&?=$2_;hOv!Xq}G#zmo6cZijt@Nb9P;Qaz-8 zSOM0HD`?Y&_%E8DFAo7i%6rIYq8dJ2c{7lmH#^q5ql{Mm zcKA;V$E`i@`={rC^+qB(F*zna#SJsKCAZo0L3d*a6yJPktZ>5q9K4W| zMkRxIAJ80rmkC0W0-#8!1|ymsysl}u=S9dddb;k6sD*HcHjsJ+)btitJYg1@BNsRx z2QD+$XyrbNy_#0b*)INS9fXIh5NaJOdxyaVdiY@@7Br}syWB*YCW;Z|jD06Wtqh*Q z#-gjKuXH8W?v+unoca}3c1mK^ti~ziC;G3c7T?Kv2J_fs0-XRf_9fkp24M<;C!kAY z3)^5d_H4R3*#aOc#|psR3-mR zF$YXBb^~a#Q4DqfDGE=&R+S{wqhoDt!29MQAdRXQu`IYZ z2sX2YO@q{k)FM-5j{s>flL3dwCFnBjE{V|M4CPEm=A6zk5lc1e$ThV7dB=6W3-WHh z0rx4gTq=63S}w-QfE8Q95_AXuo&l{-xoV};ijp?IlLSZHB!m)T=56rOm^LfdbOq6| z9nELcf;ELel!!x9YiHd@OL6TW15h|%AU6M**Xr%s^A@w}Rs0rH#a)>nNKr}f)&wh< zEO4n>463DVZBw0oP0X&cLT_ly*Rn#Pr3fi;u^1*oxBBFQF@vN|7s#P-H1J3bFzjP| zUHU@%6CcX)XJz2kR)4O%&SWUBs4aA7f?m})Y1FcC6bxj2bd&{5%PiR)rZn>Ys(4nF zGxL?irn(-51^3Q>_#nbvA=U@SS^vao1?V2gV^{Ge@!YTleDtY`7x2FwMcn$RgMm0q3sg$V#fIhtAGavECP(h zj9J5;PFW$#zIYfwog&}1xLf_du+!IpM%2q|xg12uY@GAijMT`ZS%I^C&vsy0KOXzB zVk&uFSnyCL9WcwIw)UB-?7#-b)MA;r(KSMX7*A#yen;R(nR)zC}3lW);qH3)3a@7>JZ-`-pH5Ed|S353O)0w8IT zuxJq?{Npk@$}ws(Y&bn)!_Bz^8LeGmM0K_X&a2=z$*S9svI=}jnF?)F*()9JL+b-5 zF~!Re;EAw?Qu{a1mry>l5<%^r@#8?*(2!n@Q8^Q)k}s3cHh6>NPPtiScVzhBi7ITO z&DtX0fma@0-)-7Ugbh${=Rf(EQ7vcR8 z-x71}A{a;)A0{DGabq!mF|K%OE@_pt12&JKF;a-$IowzJ1jBTUt>x-5Yd@65#@e;r z2UNS|U1y+dbUM3|@^;RUw7pwgN(Rvxo)NSFShWfQDwn_z-dKm%y)Xt$7kbxV5{PaR z2v0ntswKD`I3*rX0$_;(5c1B!Boo)(!U4|aasrvlgMC-^kWqn1+B$IkvqBQ6J_T~ZD6yozPL_o2!aGt z{o=L~mt&i0R()?4kIX3>q;|gw1ouVqNaqepk&g*`!@b+og^k#G{q!!yMoBvGof*tP zAc_F(f#7=DiwFB04UGJh=H6UuRA2JAKrq>spLx@i&&_xB_QTbCR-7k_M!0d~MhV5t zN5a`azv505YT}|sM*k`g28??yK)lAte!e8*yt2u9{lUu#f1n2msY4r}1y`Idn*`H< zFfR0!At2P05sz`^GYeqwffOeX1g90m3A@Q4(vhQ326$XI06go&y!~1TCh#r{HC)i~NYrQPQ3kQ*4O1 zlZCP5a3-PTowvIR_9JIuXnxjnJ3m^f4TfOCvAZD-(lve#Jcz5!a1rewC_^mwr6Q-1YAs=L#_PsHc7tAQz!-}= zhjCuu)kF1U^RxcRM=nG}gvMrY>H4?cOxGsn=^FpaPZ?LF8v!+nV`PK216X#$#IlO! z+6JEL`&*4z7YCY&MIfCfoC%68n)^&7`R!$UkU~HNGm8-XVAd=WJ(3LSSCc9RGj?aj z8ACRgS(~B?#BXhN*>BaVg`yDD5K2B=wvb;_rt0odMtAexJ@y8kKU-k(Geo)z@qp4p zAMZ6z@6$&Z(nCwnT%*py%&Z^c=1I0w_)Ac{b*%$X zQp4=CEHXn=K<=X~YwlZ;hrjtxFK9TMrSreelx1!lwDAxMaTX^GAD@$j34Ra+5{@Wq zS!kVRR{H%n#bPe!clB~YGHmL6jvF!)OV zE+jSOqX26tgW(g{Nr5eMZsZk;)|tAZl+m^4}~l-%vW*dOHu(tRV#c;Wt5&7s-I z5mll=@Dx{?^NK($R9$rj7OE;fLRSH8X)RgPKB}u)LYN!mn9;3@Ujw4GQnk6S-N0x>&2ZEXm7WnYt!Df?xCft zJ$HeoA88jVhwW$@^N#w!W}f4ft*QHY4YxunhtWlZntaEiYb3Qh$Gv0`5L(hxFob{p zcbN7&%E}tBzv<li&oLew!5Wc+AF^f*&+NO_CnTCv+1-lmkz@6>s z!+{j$*!w{?qIo+fp8(g);ER2{VCe@>O>n57<6qvK!=HY(RkT{o?dTIztbRd%^`_0`aO8Th zcSAeko)>Q;GY46Zvx-;(FT1c=pji#19s+*hxUUh}Q6lv9K7Vq(y7nS2RZ0bD-S7#n zi9V~@N=7^}5cazBGdXK)xJEkz5!dTffk5o{_UJCQaz~BnW{u*M&6MH{sH6DWp_eS&+YI#cDEocPMJ!!^%XNKozgl951ai1am5Kh>=YBCrGd z&^$|q3Uh%hhDw}YFWcObjm)@~$I;{yr*jD(oWazRih_U01T&`#dwP$|GFjqshE{J8 zmke3VWiZp_GnNHEWDlX6#txm&;&y7yR5n}d{j1borcW`Du!CqSO~u0&tx0!?NOvnC zbMC%b7s0oPWgt`jhP6P5qu;WQ;`QXqiF+T9D6bbPQb+0@Bp=q!9^!r{M)htHNtC5a%!=im*I6Ac=*MOll|)2vpr824nk4V$_t7xrolOp5u>ikw;>oY zrw_y>t_TGpZlCvUt~4YTwPvy6`W36b&0XIwfS=WM=5)vPh_F4KD(fKFB(Le0Q^Zns?yWl*11LmpfE`XWBw6J+@wSaJLbC;Jz<75;@rMVYhx z@puhEB5uhX`ZF}g1a%vl6^cQEH`**zb!6Q)A8nXYE149Gm$%W3l6@2;_PhArAM$1kN%t7Gz&?Osd|1k6AI@ zc_20!t!TAt90tBQM4MiSBej#Pj^fcdM9sYGpLwMDM~JT3VULL_6{gOv^;79nrurmy zHL|6vO~YyTDCbe^FelG6c2$8h1>9ElkFpcVWU6&YDhNlHK{&_6u)l3 zgWBDNP_gGk-MrA{EXtX|*>C0CRQ-dh5bRoEpOX+PDb2ZQ3#?uL3JPf$@Tr+)wg3C* zt&v02&;M@r-B$CaxHx;g8|&G5uGQM$FFZ|+dbhlN{1|dCi;}{%C&>A(M%1sm4$fae z@`mb475=anpLeJlGH!W!w(zg((N^?^;`z9BdqLbSS64ye2>I{^FI_%6FBA^%tmt81 z@z(X&;txw7I)AR}`RZB9r@46AJ^!`oBO7a1ReXK4)RXJ}u{CyawsXI5Aer5{M~91x ze}&i0U#hohzXessc9GhtbWLzBlJGU+gH3m@bKz}81Kiw>;3Rm@&4`L-zLkT z-vamHy;b+TwTK~WW$y2NYb$zf*SF)hbfOi$JlP=3#MhbMYyUuVVW0iI#TDw7ziM)N zxBr-dPs2KOq^$i^)|2ceYD;jNYapaYK%n%XAy?@*9;?t3+}a%UN38;{kl4e|eNk?- zacg(CtW{27gHGo|itp$2D}w%=!{ysE;+q3_c^Mna_1$-$`ty_f`Xd5Ohd%oaEg<>F zHX1EF3$D)p?^*=xjFpf^JpBb?G~S+GYK_2LQjlr$H*BH)GK}K3We)2t_!6O(s7<)? z%rMDhyg~_&%;??viCz@SM08;|5CeY;(eAHX1QFp4i6t#SZ{^Us^>QS|!F4T}*|ohs zpzMAa+pw3#Tt<%Xh%2aG`au;o)K+o&oCeW?*H5B0AOes#SSIRI!)&*HA)7r8mdN<| z5%;}ed)p?z)xHLP4{3SQ6-DUXcD~ItKtr(%3zv}@A;hc0zEMN5VsEr#EuO%lQl-^l zt8|UL`)eU0qcVaE?Qo#|DTAl675rSNVGyEwZeyki5K{=oE6H?))Vor#)LCBhOqv7B z;?dmbJ9F>g(I`nNT;lvAI#`0Ki8q2+4IVXth#|_;G_##-KmErgR9(+UcfOFJecjN6 zq>mGGX845r@6H{gq<|7ZueHL~p`Sp;8SpxC2h=>nOJ1)Wg5N^aR|Lv%4yY4;U-DFa zUme&&ZMDaGj-sZp6_x9<^33^4%s1>JN6~ZXBECh^V6Lx^d;kfS+y{nB;BXS;kD zV~GR7O8CBPb>P)8+VDqeyF)9m6y0BVPbP<)u=x*T>AzrSN#5sa{_4u)WZ%uO$VPS- zEgEwfbO13~0P!i5{JTb5Pj(uMr)IbR)bEzY0E(M3t@S9Y6JCf||0IY;$6BDTeSm9&v~) zb^ciV-Yv9ct8IvFZRJ4UV_M(d@g!;OChwk#Q91TH8ElT))vnr?WL=^a&-v%E;Kx#b z(uFt#H0usVSx2z-`b}x$Cvs;ES7QzDb9MeVWAlU(_SaLq_IPdeYvo@!?(|01KlbZz zE!!)0&J%=Un<+{SGxcOfUXr>jowX-XF>D2T7G7Z)iFQ7QAJRW{@lVaaE7#hh|n!5Gwpx$?0f=VK~D zHiHZC7EE$))01NGF^ZH+;5G{$AK32;YfRMXuI#_TIY%OD$Gc3)m^H_Tt1eEmb%QMl zIdQQo79AoI9~JVOMozY`39Se79U7=DtsQ!AG7qGlH$xQR-m;7-rOXm;EgMcz2yeB; zKf443{u#s()(|c=?=lp}th(r(rg&(lo&)bzy$H1eS6$~)>ABzxIjvonZKTjyAA?>0 zc&T0LYGtY(Y3r%oK$(PPbgerXGFfHQ z{F5Up)iHmOoV(m){JgAL{ZYzA!oNX+xhTw^R$MvLYPpg0(m7LUmkg4S=*X70@Q#=a z1Q-a@7UD~rQ95ZSADTADk_S4r(lP?CsNP{Zk3VqF9gA*beUSLu%!Zwcf5#fa-|1HG zc`5Q`#a%S)6+hsZuL{0cnvIZLtY*+Pbt0}SJ)x3sL+XCz9QeK`h*(`r{*6b>GUDZ{ z&^B}or4ti1FgLHxl9V1u|BaxXKg;Cv?iEx#?r`!FIvx5oc}hD8JAIUsHm9Yf72NP~ z+7co0oIm`1Yz$>)n+a327rjv+<@HIIN<&7(%b$b3-XX2crJd~5AMgu`osEs)z5D!N z9cz_RZh%IJHCx|@o0GfsiVrGlK!ukNgI0O-efJjjMoUgeubs2zrS6- z1`oI(Lw>1Q-_{Qr)?!T++BIKx?UV0kWCaHbl;emsMB!JpE(p}wFZp|i{{;=`yd8QY zo*CyUm;OkvGMZDVgEDL&B$?f3{YsMkr*$sbtX0M%PVu>eJN@U6yxjAjN37+HlCbSb zUSLeWOkWvZ=rCV_oB6cn6Mq8|>G31=r|#2N==!d^wGo_VVIL}ur3VnY09W+jqd%VD z7vUp9vXafGEdT|Kko4EgD_AkawH=83Ux`6gjV|eNt+>3_G;#&AgF04zTPTs_%SFPmQl{u7oZ!bk}= z$8ra(fcFqkOGp81B-kmK(Lrm*l)%@R_0^Q$mLS%PdfNvG23+7{ztK=$q;ymmQ;J{! z@iAv?2}=h@fhk|d-6Ch^u?6t#f#_d@IUcY7nmLh!v{aWA`$?(8x>N~>B7`Kh?s~?| zKZBZn=VY#?^BkBt(0!z9dZ6tNAnFIPyC8i=f^@khaFe@K;2CR$Qp+Q;Y+9{Ls(LD& zr?XXAne;%JiwG*=lbmLPm|y_erK{dM=^p1-+5-_TjSu3*H>rrO288j95U(NcfrKSS z8n&OXCnc$q!VLH{_(FCVFa7}5=zS@R@$pWF$=cD;`79`-?puf@39w4iMXWUhv1QC_ z0qu4MfMAE}_&fQrw+8e}5#Ouk#&4sBcs%EvY7aqj$|w(OrcU6D3u7E-hD}PJH$Zx$|AP$Ru*`5hh?H-aaeW~65cdaw_(aNd1OK@4S~CS}xyib+wd`L||j?%<>Fo3~6puI@hU|0oF9fb*e-QB2q|JUh0?E-1ppEU-p_d<&g zMaEXK7d4^xNoLW7=6VWv{0cES{nIz01oAMFpQBGNj>N@RuIbem{+0utBnnbp9#dZh zQedoVk4iLX$++|V{_QM)_m6K!=6G4A#k7fxO+1AQe<)wCneb};#-a%3I%;>Q>27B| zMJ4w3>}F-(VqyE3%D0Vl}>CzguvZmc0{bAQr&D#L=K%FWF4u5 ze@hcf5Mne6&J1sm!RXqkNrkI_Xb6!sb_f29D)Z2fN)}>V2Gr_W6WxpZ%3gGC?w4Eb zxO+;}1W|nn%a@nxGlHy-=S?zWzdJ)HA6HF>(xpUsi9DS=v{*S`YCgR^ST!vmyD6}S z&|$8^{KCLwNg)RS6KR3aqbsz3T*OQ3ADr|)OthyvV$dM`P*WBV2;t~=zCs8W!t_HM zJl;FCl6`_+d#>BSSp))qF-LVqKgCpgF*l+aRiLFgK*=q&-R@cZU|5-yy}J?5)5mu5p&a5$%G+W?v6Aq zyM_0W2G-!fe!aAYQ47gW1sRbg-koHgD5GDH9=K~P^z9!{!iCOl?)(?RZib!^qBts< zufUWuj%OkF&R7k@_O8O(b9}J{$?+LAL~fG3nNe>5<1r`IIiTVr9>;p z7$TS~9+=Q9?=~kXS#WWPJ)|(uza`{Kg zNF`PAaXN@FbQdmf0OJH2hxgB>VRB;J-WT&kGfLVv0Op)#$sEtY1Ove`#ZX=ye@1qb z){mIWTV$E0(5|nHZMzJa5wz@VFU<$<=K7|djP@oAAovCzaosCIBMlq@t?y6@3r}qh z^o8SDCFh}-cM*<*73#CHbN{1WUH(hzRH0CtFCW5!tSnz=nHkpp^1RWMVg~YXfMu?)^heY(ydsnu~g1JmIcX)`c8;zKA?qfPAVIedbSE4C1C4@o0?~Vd3AYi zqkadM2$*QbsmxM!J=^KKYUA>tNbjna4r+QO7~$hw?bI@hEauWMK7FGyo4}8iJj*eX z&CuT51(Hb|!d-7^$W=5=*o>Mhj-MMSjF`#5RLE%zLL)pp$#Xbw+ljaNWFd?W^Df48 zYt4*9xuwZiZFL@Jyfu_0f3qo4lm}p}MeMrhS0r_64Oajz2XS9Xlhs?rXVWHTwmzH^sGtrkEleis(j|^Dr1o^k3K0-DH5E zHtdb+4ddqg+9>kijH88ILXdLC`I2(})1IRd+F8`DX1a$>W^fSlB*?#4h9EN>wOahR zldmEc8Woi0XDoj zEr5AGN%)=Nl8*2ZKOO!Jo`a;}48=h>n+YHx)NAlByZ$^LMR(I=Wyc}?O)j}Agv1o9 zisv6(^x+Ga=8eX7tbmWpk0TnCk!`qZ&P_}!YCs=OJK8y!rPajc7K#u<7QJd#YB+nb zXmbNk1g<+~gDe^1`iZtVB4}noES6Kzs&sHeOH2l(ztG!36%i@Gb@wB*sp0GHo z*M{@-+i8Fd6BA7VDpzq4MPZb2PohqSPIGja%|+ATQTTvD-=une)NX7XDuSmC5kd{H zF^Iv0$#kGMC-dm4oxIdK!|Zo}MA>zj!3J+;MkUT96T!h4V`Q|GPy8Xp5gIFLdnq_|q2Ao^id>p!*WeqjXsnQ`Euiw0VrA6fDlNz(FHFo$A)Hd~!&lX5pSgSQ-7k6^>NaDOq0&6;mi%>HDC z4TEH_;a(u>rKn3v&tG2dPq!TeKwlj)FycG~t<6x9H@4?>*%mEy45EiG?C%j#c`!In$7Al->3r9J1JCX{%6 zmXrw*3SM81!Z1gz#l=gkpZ?1|UW0H{P)pr$D}QC36*qJev0>WrH1CwUSaVtY0*bls zsLI)OH5>!sU=&7yM#a=s4OSmfm=H2Ms||q&M~cxm44?^tJr*_UJe6+$-HOHeK*~wd z&XW6u>(((4KvtFM1$upqW}DKyoEaVmZNqTLeZ@xP;YdsMdQo2%rIxnb@uw2o3BFI_ z2~A!f$R8*!D_m@CYoF9Y$5anqH^=YS+u{#qfz4t~cgCGoH_-_*>avQUtv+-v&<{Gf zs0!Rbcs6p>kRRlc-5xS#apVJU$8QSp0=(xN*Zfjk&i?y%~X9Q#s_&9GSxSmrrm?E)Rd6V5`POw>HZ6LZ%bx`ci~;)czfR~C=@Wq-{RTKnJchA))(R|$@B0}q}GzOwers@ z^|kc>_NA!~oQ>F5AMJiu{_4HP(bD37<6g)uM?YYlTyyW3P@U!jG>PGhKLdX6(Tx_` z1ZK^klqlbyBXd90GM*4qB_)xkvi+7*Ur|+WmSpFXl!f|r0AQNNE!u=&G$Yg@35R_l zmXoP4wiWqK{mJWteo7}4jw=>cxmm@(+M|E@e~ zoE3LeXw1)1Wz(!+b^49i^Ql6lTXu;0!qy;+O;kI;V7zaiE{ME+GpyD4mT7sX;D$9OM*p`ms>UWE1#Q>DP*hjGmT2K8pVv zIa;1Ft{x4Wb?M9r^L|v=Uj%xlzo?{|FZe{u@07S${vp}r%skA~Lhsw4o7qut3`wQ7 zO&Cmu2ENfU{E;YWa#V;vfL{3I7(f5x;pn2@*r8$QiZ8m!-ptqv>J@pl#N z(m04b!*hOlS7*-nZ7hLzm6B?Xk_9-hqeL!jKw z2D&KiHmE(*dZ(z&h!gK7_gYDjKE+Z~odXg;V_ksuEGk%*EiaezIe? z48xyz5;fw!;G|^B_U`t!rlJLFjJdg}%-ZO2z6d%a*FMH4&7IQDSk+Q3+q{Yi#mHHt ziciVvIeZ5t5Ui<4MAr)oj=6Qc; zsA|!;(ZB9#3uMrstHO@dRjC8%Lf|l)P|m(gBYP?gd9mfxbm5EJIi^1urlyxQg4v!C zCFvRn;(nCU)Fhb% zNfBDTaBedwxS-!qZ7?T>S`~EH+tT2MDeH2SEWBBaOR2q)?^bIW3Ezg`XQ1OhBbiRI z%2H9kaJG&wd4k`(LHXS0Q7~1$m~_x$b3k%;U{l;^v@FZ$ZrO6B7x9O-FReuWVHOM- zc%gWhSh{$y_l6g&9Q%xIrWDn{Aye}X9UGK(N~EFH9m>D;RXEKXq(Y`pn{P& ztMS@0NuP6~6SdcnO)!(HY3z?4foFp_f|1Umc^O~E(v!@Rc0Prr1nf9jMEy1_b+Y@w z50v?IDM-?tNXsd6k%72w%ye^Zj((pqMvv(%w}5-yyDHOpB5Bl%DMLHWtE7&xmjsPV z)%Dw5iBe0!wi4w!Ql5P!wQSoV!4?xBqIc8Twz3pe(-Ek(_!|j~fT^~j#%fvD$rc3Q z`v^cDz<23nx;smr_PZ$=l#N>87$88#won#J@uMsM!mR0YAlmT}Q%zyMHXzaG1`2F` z@G8u7w*2%&m0y#%fpR7WsL-t>$d9wg|AlBp{6nJ)2P-rv?BO~L8KV-nKP`TfuGm+; zPr%pv4E;oEugUNK_sQ=5Ni^39XRyWHL+(GmnfJS6Z+n%U`*$7#XPra-TA6^3WYsoScfLJg zmCPJ(HmoOkJ~`kdz4JItStcN0NXCxGwNTcXNGHH z)&Y=g-}c#7mS7YEi>mGztBC@Q9y6pRsujSf+3DN+p6-2wbpfxJAj*re5^cY-`o46E zD_{}?9G*z@)R&n@M+qDJ^6X^$t?I02i;(%2BFu;{N zyVa50e3hNk`1ULN=&{_n1S#G*=igjn9Lalm6X}p+xw)5*nRnLgH!h5M)=)qRht(Bj z!((uhtg*OsChy#Bpq;5*!zm`M-l)+!ycNrk=^d$5eF3F)w&?RF`ft~o$J0U;o3nYf z6SetgQfuF4-_D0GdzZ4;@}6U^4sT2Mg62i>`uTDxbes-*a~h1lT0veUha7yeW&#c_ zk;Hoe=(o4GG*+U{Bw?P5*^Tc)X^*|xXI#z4!O*X0j5q3}ITEB?wGR>NGYc-qkao3j zf=(u8T*#`|s7ik^FRPHhj;uHjL1tUms9&f}C)rm1LK`|CnT@cePrsQR>XKu>ytYE; z5I~6lFN5dP#m%!YdYOX`_c1$}$(|d`Z2YQsDt+6B46zWZN659vwDqvZ61P+TqeT~x zwM7>CJ0d7{CQV~j&>TShyY)RmoR_K8!=SmU<>r-;RSjRDJIxx)2-v$~^_)1|6Jc;@ zwAr-7A_6l*^sUzujZ2hS3u`|dRF&1JQs8h?ow&zje>q;gk0>#yX3NfyC}5Q=3k-u! z@_YJ|$AYSNqN(P9#$rb}t3zsT%js0pZ3qjM53G(A^Kn2v6eLEYz!M7ERchU>bSY|} zP^rDrw^Ra3C_&JhYw0Z;jSqQapn~nkPOgrmpx{kuw>QR+N=9zaOTM5w(c-@2I=CSJ zp0%C}HSo*$38p)m$t;ou%Eg<1Dsv`#XIXCcu}B}$8) zKE=byvetqU&>((nusXPndsCzAezN;qKY5UJv5+%e(BE<>qg|*#z0?IoL&RXRjr*?x z%ros1Z9?cGVwNRKZE&JtZlvP$m`-@mb=m(n_TDK-lP+k~ZQGu<-Cx_B zwr$(CZJTe~w(aR@+qP}IXa1{mVqfiu6Z>xG&5DYstEz}vSu68-GQ4YI3kiSo&oA3j z>iO1hts1#05GDmjp#olSV@p=?xRKwk5Vv12_hGGlsp#2bDZfYwONvwuCcWYNgjb}aF~8d_A>M-IbH0*hIsbCPQvky5jqFzx3Ec?VLye$#cAVRPeC2Xcs-3gIO0b1udLvIHxXT|bC@R<5X__Pz@S-N zFPyBM1rMQe*#ALdFKMTk`C->QlS?i|bMQgU@r&*Z`MmnrwIvx2ZTO~MNpNrm?XHhB zB1($0IV!`Jy5t+bB@E2Ixo%`2+|FKS5-SkSF#1Vm*1dV&(*Fjq*etf|w(ST%{WrU& z#6z!+y1oUH2~*+P@1r`3S*ny)3k#4jjFT7cXTq*6>4jqRD4Po*&B@s>@PSk*!v6@T zhDg4q=;Lh_UkWL_6F|i9;WKItX?H?8Lij7XmfF@<+&|fLRH-?Kj>}}=@2;Rp17Gf$NC^FQMnvvG`&^y6H_I#5<735_6?Be%-zf4gOfe*E_ zINA#d2z8`h8$4Tiv~}#+vsH0%7aNO*Ww)_6m_WGQqo9uyJ zLfguu3(!d44EIFf37zx)aO`a08xJpuTY|xDO?|y!z78=ua7Q08u51`S-{H}ey#|#- ziL!Zj?Q>R}1TB_li@u4Muft1b%$+%>!=zYXR8~#+!jm6|{ObU3mtdSK3|wF@FV2Ig zg$VLL-_wsf7jsHifJB;LD}Hq;H$4rgB2}{a=xaEBGa>!C@no06Fp95VXw>0DBpKlz ziJ#F3N7m}#aYn%g{~@cBsA95%C%uAPjm&gAw@wxlMF^!cRVESdj#?+ z*QQz47fU~C_0Y%imq0z8fEV9tz__+!gHHDg7C!ybRikBmf#YV=2&F(o8wHEENOHd+ zW*vDc{I?Hy8WugIc`+g78HIVct)8Jnm~|L|i|>KWN^1x9E}G1!Cx3LKtnMdT6x9XC zqiDftrq!sd9*f`OB;0Pn7#(IGF3~f}+Uik{(zM*Os2w{k*C+kW^`k?bzjK$*+f57L zNzKG~D@Y9&Yqjr$JskyZqakg5s3een652;c)|onOwG8G8k&I)o)JDKEX_f%CpQci) z=B?6Q_6_G=7Ca9El;7Urw$BcmqmTn{=$vA?pRnDx5)F>g{&m!IcB(;A zWVfoam`18|Y+*I`5r=4qxf8E@q@_}}+9{zDymiYe~4=!_TrUltJ zv!};SRyIy3(`@QEIop2WP|oBsmYl?LMV<2Kk{<7`X`W+xAsvE6DgKroW>G3 zZc^~}6eawWrBtcW*=2Hb--d~{ZD$(=Me{b8x`Z$OUPtXANm&wR4y5zQxG~P{E3gt& zytZmxW0S07k3e8S7;olkjBi5o?)Eou&V|#mTUS9)c>RK{p(5nXMwV-+iWa}s&Jy-> z)FmO%Ly!(pUqC0FhI+JH%Zof9ARtRO6y_ok^cO)JchseB`2<}!ZdiW)AwkmAaDw1s zlGA6arh%m6oWqADDjFWc%|?9}A##q3AE#Yi|IM? z8fO%Q-uSry$4IkjT*q$SeBr2ht-?+tPxl=iUf&h6H$*Y(&1*d`wmLRq3QmwhQO4db zBG{Io@PdS_@8}UGobFtIhN=Nft!yazN!>9`&i02}l#djp%FL8chO8!!3SX->wB!%I zUTH|PisQucDXAI@S2G(QC@vN2BfB{p52Q{(EDZEsC_>VJnSdd`(d_nb{2flRla;n2n1(0;ecpeI~S>BGI7nV2qy8W{X?rJ zv?M|Ouxfsdjr{Ubqwtfn%@b`ay#^J7jl>Q`C%09q)z?>Frv)+>#yb0q1y#gnvjF|h z1dNeMz|49kFl3Wm2SZO#omiLgMHu3=M7sJKYUvAnkiz$Hb_m@J2*T7bXf(0fwM&ti zzb{;Be&!EPBre;Bv%WzFX;*?rz6`^GKwkrw;4m z;9mwNe$`3)z?D3Ezhj_3)z2*pT)@2qLF4a>#4X4?Nw(!Ma={S5M}_>}j?rVh@dKh} z0TO*6@{P|RtefwxX&>k{Hauhx`%D$X^XcA{80D#J)w_FAM_D$$di<6v?7!_D+-b#p z7?eQuqG&ld)km74>H373y$L;DhrzempW(5R7(EP%A|> zEm*nc&T9Yln%G6;Ia|ZX9QCa%j=T#TR=*V8eUqdnr;c@W0j>D1u(KWUUjOM)94%=r zqA~IGQW;a*#QibAPJ^ItWbjtsB7`9CtJE;Wi9RjE-ztNaZ?_OLqUj@>dD_ScoREYY z=V@plIBZk;TUyLhefTm#qv^=|&`0!IBm_ths6Ani+QHA6czVbA@6QS$1V2#{o$YnI zPq~=rWYGqJl*pHGW1cgL#&PF|OgmNrY@Re-K)|jhO-|X7O_Oq4*aoIlbMEFselapo z4U8Z$ZdrS#L$|-(PIAs&Dk_>IIlL3qKKKZ~3RLa&SR%-foEXwG^}orB9k#G|9(UOm zC-;o0V|nP<{6lH&uNTND)tpu+#y}yd{#_?P2M@>OJrglMA#l!Ay z)d_Bww`gvm`o6#NzP0Of#e>JsDAU#U8FQ_mvt)L1PS%oQLIqDxvsf+3;OwDd4qmJY zK_opPgbDeamnf0^1@(EE(x&gop0rr@Y^y~?n-;xbS-a~6kxq9OWO&I~DYh4RMz>f- z#uY+{K+qI{RA8I3iBahv(dkNKOEuWrQ&;mnd#&*S$Sz{{$e)jDV~}yoI}OV7fIEfk zot2`PUA>=8?SQV%^nD-)2WP(jG4LntNHApNWT)c~?K=#a|7ntEGS6YAFw*2{gU^oM z*8W#xlt6t)Pxwr3b5CMsw7VnskD`<@y+Gk>^-jmIm%XfU3G`nRL>}L`AEDna20$U^ z5!`{g!M!YSd}Y+Av@apCBi_%FBs{WX39Q4GZ2YosrN!3e9#DEVbAtFw6v3)GOw{!B zh54E_|L6iooR@O&-QN3PxcU8jk0~A43v;40Ya+l+?$4Ijroi`)<7?>Kpx|!q?_x#{ zj00U;X9cUcUakW2I+4Flm}n7+j~ix3pZ$&?+NJKB_3gXY`R?-aFT4>F<>c{zGqy}X zLDcxd`TKmyXcXzftyW{M_bqe4<^l98Kf4ee8Yx=CXC4+?2f*zhvmr3kFZU|tEG$5h zYs2T;<+Ng3;AH5vb9l$!}-NYI=8onumY$ zH779r%LHM}bzxsFUx$gjTN+z)XYzb!^JTwVrTts&dRWZ(a*zbp62DbUs62UOJ_UyW zZG{FPZ{5Z6){Pac)T+IkpGO!Kl#JM`Nx!N%fP=4|ZJ5IYm?$`rsUAA_%dF^lSqS^2&Sp-`)C33I7dx6394f5oiiwvCO zTVEQS{oB53oV{CH8k8+#&m@|6e*Y-xfdsl>y))k|yj`k`_`r-xkwp85fKyd@-wM+W zKOjpK?PUgnbSxP3Wmom&FXgMPzhymVi&J{OC>}i-V~cf$d(sEZb`lzZjKF3t4X!%r zKVEG|t#LOYIeSiq_3iI{818X&UHp=P4;n0A@rl{V*9%$Dpy5Z-ZL?WfW~b)n5%JFH zw2K*w=xj;w>Wpc{OcB}H&gZdVUQgA@ln9S;Wz%_=O4p((da&Oy=X_Z%nhoNk`pVb{V3j?(edgDRHATuXToa-6_#d{xjn`rU+f>-pT=Qb_2a85$5oF#GfjrUe7{e$F9B9QcoL#EnMCv#{41 zq~ewn96#ZL4+Q6j@%~f3Ke`{84ssm5S@dMtJX8EO;HVsy5kFxtxA4lAN`Z;Ugd@J7 zC4z;j@>tW+t>@%U0lmqG09!*czk5LUJTzFZG5F_K9s2ri81l$+2Eyhji@#Rifbcae zX_@!@mN^dvdN?OPzG%YnJr+9%F3#x>7Ln=p8Zo)i2G6j~)KFzIuQq?>q4Z~#CxI#;E7)a zCWbP10zH;CoecU;7cu@+uuO)|;6+1{+IwH4K|K$T8%+E7NY0zuzcp#qvd7STcrozy zqM1pJfdW%S;gXLq--+~K<&~>RxKS>l(Q{7}ZPECm+lUQ>8nLa7QG`j_46LQ`N2dB( zZ?KE4M9f{rHSQJPgGe>-g)hq31I+pmJ2+Cfd#)FB9Vm@KoH4feo2y7ACk;|Qs37bTl}_J9g^ox&71Y5`HZ>J^^x~I zNVod59j=h*|2!bPaODN1#%&h%DsS4^X{{4%vagi*7FCtO4C1^`q4mlvY} zE{)EEazEZ+tC8VWD2p^&rc65p2&&^SFkik1y#FMQ(E3a>KfU}IcK|`Lcruk7;uFqa%dw(lSCRGU z>BL8X|4$JMyt)*-U^@qVj#T~Ng|RJ_;At2|tdu!!;16h2xZ|A=4#_>#RY$2bSg-J@ zGV%3|s_^)BKfw#-1WW2jZyKp=JnBMOsg&{PG1(I&wh9%Y)PL$Wr9{}(U73*qNm^7s zOPnxKzkSfsqSbKRT%*e>YJn&?T@|z)fJ;(0+Xq}Z_apjS7qhxiNXvZDc2GcH5CJ`< z=wCSW|KP;E8-fmNXl|0Lxm!3nVd)j2;Z^DL8B?Yo+;n7G>PFj1GllK6*fw;E6IVVM zhQa3_iq&eD{5*OzDDR=Iv(k&8r>_@W)~y%SDR@Ar;1_@Ki1*q+P|gjy1}b;Yjgitz z@{bVF#=gnvIyhxk+jC-f{sh=%Ld8`@1CUV3BD^#D-h|_g+Dy1pz_1pn8s=uzmI#10;cjzFq zQ{|ev0zIWVuuUQk>A4C?Om-L*@pE=vMM*Pb<5W*Lu&l-x>*Q|_*ks*_&{CGu}zwdd#*cxop7l?AdJO9d+O$Pa8p*MFP z(ET-)sFWl&sGdm|-z11L>lr6#lIs)fO+A3_qRTt#A zTw98wtQo`v(S_l%DY&_+4BR%19~~xBn54Z zANiqIog$1~SI#fDT&(@RwL3_IYm@mmPdTRq^4NksX*5jX9v6#xP6yiepW+q=?BkZfk(tc z>VCSGDg5}t1@#2y3d~dw%9~@gpgJSD&V0Gk@iCRWi1ypiR9MvL;8z z2qr|8B$b9JX#W`&dbBmpwtAF^@oEd6&mJ?0zj06UTu<;orm9MyB8)c3Y`g<`w zJ@$AyJ>8Y)2qU&knA)TErwV^;3oh@07DtU6pd^jS#28`|yBGa2L&tl$p~lxsy)08h z@F}C^pYOG7Ff-}4vE9$f8(=^uT-fq$U5-q#{Zc(6AJE6Mnz*qZjoqtP=Q)3QMTuEE z`znA7N`|9ddk}1pD3pQ*a&!3+c4B;^p$U$PsEI&3)yN+vYKGaD-`ezoREHGt zT63(5diZxlc8qy?AmTAvWr00LHf`gE!4P?u(3*7BWP zx0elVVNIU@P}OkkXpi`jTt!Sdv_2HI-_Q&P`KSXdyp2ogITD95tVuN0{64duWevyR z@W;Q=VBL)+XsmXT)eb{8Bu>9A6^m2H;>e0H3Z$+>wF)H@e&MdGQa`e3Xm`iYbIGrB zSdy8T`7}FMqk?TpFX097iSEFv?t}&!(dHYm z+E!LwGZ-Q68QDRlSMFaBy$vhG9Xr_XT2NV4zs98GW~Dl!g9z}jCq$1O)G5F%VdiTW zpZ;j2#?GndR5mZ(l{Tad6cpv`m$Z5)LP&5B|3}rsXEcuPS`hZX9* zN;H_CZ*6b8?Rly#Sgk7Q{FoNgvbK<8y3*RxpGw)c0uU7p=u0zE#-_#+v8=z3Qec+} zS|itK8#z$?uO7NgoLaY`6=aM09%1e;7hYfbCakwX6LOg@nEnyIfIKlNpuEV&ZA>VT zsz8~t^H18)uU{o*MLmO#Jg#7=(C7g;G7_;>!AX0l&DByqW9;Po>adIEs58 z9{dE`j+EG|OJ}=K=-}KZXUVPN7O;8rj3c;Or-id3cv#=ork_!oTz2O$Ip?ui6i)pU z?+Q*5FsugBAKx8Q#NA-tJKQEdgwIl!1W&U34A1%h}vfZkt91%O>%) zo26INic@hgRG*OklCN?9Wmvq3du7}!!WZd_yVQP!qjJW0Lhvgo{1yby>A{Kj!+l#X z5i9d?XXD2KyY!3I>xrj()44mz{=u0Nd#!Z6`ZK&nt(ob+x8uBbn*Ic(dzADI-2xa@ zk+H;oT)m|uQhUcWwjdg+iKcM3fRJe#OT85ZLM$A97U;y^1}C zWS#MTN7c1mP+CAwYAK)j4i_rU*tFAF7VElkTuj?muzmyCfmoEbODrhuMr{U7x6NE> zTzG-z5go)@H+lBxfX6k$FTPL4!&*Pj z|5-L%UUd%4$aI@@UNrNyv#DFvWQ(q3z%;Jse`qTd=aK_-HBtUODda8i@*{y1x@i;! z-lLI$Qp0)~*$p-<{^Zz@EA@-6pB&qO2a>d#gzB#Mn>R69Ie=5Ue$B!h4}1>yy{=DW zW0!LOBt~nogceopq1i)XWs+=YjzThpa_$k0N>q9DQAkDcEF~AVDmBIBK=kqv^Cm09 ztblq1Z=201!DhKEf&QR|rjV-h3VJJGe2DDn%%$bUCG^opKC0_uP-w|7cfgiHLT^~o z`d4OI%E$5VGkn{!TLksxwk;Vj)|$w%6UaR?u_z?KxM=X8A+a>UxDLvDVk@dKNu$bB z?xe_sCTQVr3n~nKb`Gr?%ehu=$rW$1!p4gq*1+e*7pDmZu>c&4$UjyvKYMcI9UG{QsfWU3R-R|E3oQ4UK(-N zV+}ACVY!AM_k2s0n$c$o3jE$;QuD`)M_+J_o@`m?lIy=i;s_`g9g8sK8fX17S|>hp zv_+Qq6yHJ=PK*s%qFa^4rKF5^x#*UGBTldm3W(>SGAs89Q7mOU)79wPT>Anl( zg8Ev2Q962K;>L~B%}&Z5G-;#WRi=xhdH0-fgNT&XZU;z=Ik%HFqw{En^O=4Hssud} z-WlottMFAPYG*$_X*K^l)5z7*cs=m#fk3l>;nt%1?{eP$x-^=qBe-C3%6RS*_B$Ad zep>2UGb1n_LhMc_Lkne?9wQVY`UlgPWo;#fW$nQ0Mg4$X zMZ`>V_6xAOl1?*R4tL)&&zcm2b|hi!wA!OABlAP`MV2ZFQ0Cd+4tU1i6hB<*3Tlfh1= zGWWIDU5F#o#ll-Lr%vIQc*KbhpEta!*JFI7q*VVxuV8o zS9b6wnq=1Uqshts<$aks#OqvJECX9<3!H{kW;fYw?_e2>ZY8dH9oE;S@|Ls3gSB^) z|IiTQ;QZ*1b{*@$sH?Zmf$+sA#?)6k!O<|u|DPUh4yDM&lPs2O>;!FOJV_NsQR}yG zmgZc@`6(kuw`nINiKv@B6{+7Yz4yL!?D{^#U!=WBV`INojmclb!xZ3 zZsh3b%%&scn`ASAYw<1XVGjN=r}{&2l68Ne`v$&w^e;an1TnY<1A?dRI{9IvbHmG} zHHpwz956TNJ2}!o*bAH~xf^1(b_|MM=+q1^C-cC?iGA0&WMkPBD$jysr2*z^T+EK?_sYDnhep^cVO~_*}J@QKbZN`E3SfDs38uz zF927Z*st3+q<4U$<650Bk1u$O4dqD677jXHl9G0>UQUUvsas<=F7h^#M%C5Sd6_rZ^+!^-n zq@(ZE$VeO#bcWjPNsAfNX)|sv+~WWB1zUtpNsqfnQ)Gr;pW$t)IjuW+GU-!}yQ|QZ zCp%$nLYn;nFJ!qzzl1~LZfv5zf%2Z~qKqGl!qOKY#kU~gA0luF{n=YJc{mhsZ^>ul za|Ll|kw9doLOLm@d5ry-wI+U6RPLh+<0DbfFkHxr5|ehd!hLt42+BV!WyU*E&62_I zMmx<#3T$efW2{KlpqOB4_|re$ID^Hv-h{(y(s3-z?z;`BRmsW!xvZR@xj4q7kYnO710!B-oPlm}!P^e_^DZ zUhS7nF}kdVl)yg3;(duA%zstW8;={_xj#j(LtLXF-l5RBi$!(7Rqw2gXH%}sN2kLe z-{|5TvgluQB7Ql!NMyM#i7v4>ggyMHqP+0-Q^!5~yQ5Y;Ebmw@Xm?*907c{@psvpc zTME!B?nfJ#4Lmm>2CDey0XXqfL?TltW0`~`(f)q^?S63d#sp0z4Xj$1<=GUu&2U`m zi}C;ft`9=9KYDBy7o`IC`>XttKaOtS=XgIy>{njJMV*7Ccrm-y@|diE^#Nwgq_6|D zhm71H21>qQP!PX}bIF&TVA2lQ>OS=ODme(|9d?9-EqaQ)G!FMJ+>0|`F|(-K9|WK4 zf{oQhvZuCSo39+P;a6%{%K842rd#;cQn*GqV|8QOs8p)sFC)|Y%LmBDSiLEs!{BCa z5umS9UdTLa2veCIs-k%uogJnkC6=_h6@AVaCvxcsOTWD>3Oo`Q(Qc+8@IK0@+Dc~`T3uHt2~p#j~L^k zN4w&}BZWCUD9eHIM!58VtiQ%Wl01TC+ZMD0+n=moEpF`D3ezTMdo;$@8Ajxb-FguR zb@C>4kBOJwBg7%qCc%8(BMc6hr)wdV;gJ*ZIs6&-pVX!I&?T-`V^)KDAVz^5B-){z z*NaP}iu>xxS?TGL351UW5c>Abq{onU!$^G0#l3xzr@NNNXECkf&02Q0YB@QT;UtnO zlcVD`)jDGV2{0&5R0X-~)*S})5pN`rZKM5a0iIYIR%+Egew3?7EKdL2o3QQ>H=L^I z#Gacqwf>%?Tt(tCXg(QC-(l>7E8;W`>(9&h(2kz>M+$C8W|`F@lMmp(KI-FK`)zUm z96$AU-nXnz9R+Mzv-K|k11i4~Y_qg7$qZj0ugo=PUm~}>v!_aiNMXF>AFl7)hji~A zzN&@>HYow(cRNR|TG^hor#>IqepcpH5zZ=ReL5o1!NU<;>xEnMYN@R0(K!oc$~(^I;LB$rd5 z>_raxNtl=!@T!>&KvpYCoj9lsc!H0!eO&cjoN0o*+@@Alz1h36!rhABu;8I%AdLoF zTa1oemU-6wyalU-n|a(~*1%6`sfWi6j6y#h*w#$G;AvVASnk#oZ{up)t%QtC{I}9n zn9}PHYqs?!>Ffb$BjNS`*B3F9aDAB4I}HBJfaoRX+?bP}-@bdZnz?e|KOuE!f>&R` z4E9L2A7~mlqW1X97IXw@&v##K(_}+`F7a509V4(f^;}KVM-rY)>sn6bcuQ4SD6A14 z+CuEH&C}kVem6k%_S$Ujd&pgSY-}Vn%Uu>Mb_>ExcY|2*M#?!Su)dOTkuAF5W-dSG z)E>s5MXUSXf>B1y^ui5-S$?aUEDCk;L`pyxAQ~4cUExEyJVPfr{>LDfG8v4Z6J*pc z(|YkSz8WzrUb?OMp=V)yzR(TFFUza={fHg8oG+tOt*YcQbG4F|#+FWBUp{SVI#xAj zHkJyzbmHJ5_wtGF=Eh8_-;oqb%Z_NdHOmOtw(cqafTR!3h5k-V4h#1RBGrn7_Ma;Y zK^pxZa>z;hs~X+W9NyrqtrnLw0zHbU=xY`G>@tO`=e@fEeq?y<#ym(^PJb=;*mx*+ z|2Pj6wVPy2`dFwY#0(qq4~)| z;{2i2BON5`_Po0~6H8}_#pH~~mfHlW_zetqa2_zMV`Ykr^b0fHoe+{jx^o*x!|N5k zYD3SEX!)Z-`y(*|C@QM&lcbXdGhOy!iGjpas%13mJLH3F=?UF!;%65ZcASTNX1zVd zhTk&gJykDBaZj{tw!mUU4^6k`ci+SJ6I$8na^`^=batr)-GG;`ck|H~;t89&vfsY7 zE2AxJ6L%_d9j#~*Idvu#&r+KG2*XV(R|*oF<+~ar7|(7}q$c3)p_6l#b+r37uDRKp z^dv`Nyz)!Queb!1kd9hM4KB2p|80xeBM(j}vqtbe=(+2b5J&d@F=9$ws^ByV$$5<` zorWw-#cRM%-!eJ4)$_q)Fm1eu(wj7;Tp3HC)-;13<`c|d9Ox&vU4CdhY0zY^-?gw-l}@3M zQF_dAc8+c;{X?}?gIU4^=yp&1%(V_9p(8ROfsh*XtF+ux;+ z686${SIOO~Pj{6*LA1!Xp!~iI8=DUXj?=Kcg8sE~`{Q*^@eQ07!Y=vUPrlFR2(*HB zEDyr>mZfj+*qUhyS6t1q$gyN@lW&>wrVOn~m?%A!HK1hMdtP;EkNLL`2oE)J%uD@5 zr_FAFeSU+Tsc`o$PmjDEnjTulxbh=wZ5I#<2WMl9()qEB%h`RE3Spf|`~M8(8dG+Mjt(AFeXAJA3X*5uETq8sN`tmRCkKjkM*q4 zTD@tpo4mkboR#`mHxsKOv%{(RbJ{!0rFfi)OyDV!?fcEJNkN=(6&%CuO`?`y9O8W1T+$HlV!7i6sA(@(N|cu)inP{P4`>~JuT+PmpSBR z@9Uz*={~ze1uh6w>4vMNkUsfwsHCB}fm+Opr2h^D{qT1B;oo$?CUl%0%uK~I($KiY zj!Q`0JOAgt{~XKstM5u#cA@WzxjF0U*!NznGlR}^GiN|SBOJiJ+OQT#)EblEEOElq z90QJ0bqOg|(my*cNQNne)@*o-?79h-^0u|kSF^@i`aQfi=*f8 zik(oq7Fr3*@bjrx$la%>5FJOW$k4v3b~vu$g5uqBGbHH|zw-!Whqj^|$>I!Av9OEZg}U-c*jn+YZz^(IKW}Ntj<3jDQxnBPGNm(@ zWK#|MYG7Vf!Q9(9E6F_=#jJn|w{PcR3Z~d->Xa7*>*Z}H`=cE1(WrM(!%02x0ML@U zguhTolbUUrF)k<-4U}F&^0b-950N} zLwIUDb`%A?ufAJI^FxPSn~8GpB;6cfg=n&mH9*|nT3Hu|(Zw&dV(>fbViIf`Ri$ym70 zk55-0H!qMM@Y!F+VNeMmi4I8VG+?Kwh1bwcOEk49sPC?=peUa*K;aOni7NNh#23S! zptxva-Tg+PuZ}Vx77h-QWJbUl7dd6$=MV@&f^63OMcJo z{x+K@DI4aW>j_F(BpAf2T9FfOaOQ{&f!;T4f+CO50q5@UgvaE}Ro;SXlrZf;1dW8^ zJeSCiULVLktnE$~e)^A;pU{bm{Taj6$q99G1Azq$Y2Y;%%T_86;V6A%2rQEpzD-jL z$d<5}N8NsI+SWjYudI+7$I~3I=o5H{y-=lACB*=cjNRf=8|IW;UkfFx%DVz^wgKHSFtl0tGQPSV(yWWg{}Wsh8{!yO?} z>@N5%*U^#iLJX^^Np9ta0zeBCmiFf8PurEdBR=deNg*-IEnX*IMs_tXbnNSR{powphcBH{ewFIlHQt5vlpc1B|VJGu)!RnfHA zG^NR&Ye;?HyPpV@w}hf)W~vSP;1mGo`XEa0Fk&Rwp#z)3`0j6O?7O%CF0cOpC%?j| zl<2J77KDv+u(|Yn4pcbKNdFUKzA1gGQT>NEK-!ZAlEAGOV}N4VU5~_y&@1v!yd1&6 zgK8?yVkKtoU?juiG1*49Yuf)JxO$mB|G(8c+Xc=XSb>XA+EDF-|A0(!ThB(6z%nT15tY4d>xjrv(yVB(iT)7$STvxR&&4!zh-bqiqY#%5 z-FYS=Kh;M%>VBW?E;g>LXi*!x%2Gl^gI$BS=i~C{-D{)bfo&mJ!_|~l2+V8*h4vKk z$&Pj+a)?9iZllDbjdfQaqDh6H=47lE|8*`sp~ap$sEGe3F*?2~M(M*~9-E&@s^maZ z45Whdy%Wb!(ay|T#fgXXsg3Po;>$$@=31A=gdU!0Y*P;HJ%}rh>P45N3xKq4|93He zA((FCjZQlzkOT!PLhvw0l4LZBqb!84$789Y37L3J-t-)xxi~NSgpCGeWsEI6ULzYZ zoA(&h(U=n^Fu@f9+2u5!AmK&~&y0+D#N682+}y3A6-~=%aC^nSi=O`*{`vUL*hG{< z`0xTJBIYqC4(Jdd@mr?z_DuVhDjFrXJ9Z;2RP*ZiIo!vK*vgxK3aQ!VqOKGYpT+|HRgfdVEX8;r6tATz&npPp~Tg7B4cdaK#+|6zDLaou~17=Z9E+6aOubk zy3zZ^J|hv8!b%~u-!H<2O}~)NH6L0H6AX=WBT39q5^8S6eU*~3MpLVb(vEA1F@7Ui zs>b$jA~j+ju&lMPh3Y1D(7X4PKg?=)eH!f{Wyo25%e!@qxr%NgC>XO z1^^b7C;VXg>Ct;!F;b{-lw~pg6IZ>cWT)Ik*-t4AL7Xh<)YL&Z{X)Y}u23=L8@|V0 z0BIy3kk4I6+)r6>q-xE9%HU5q5ievb`wJFz6Z-p}HPj_YUOzVaUP_Q{QfLnw)jx1w zm)54*L`Jij7T_#XmH{Y5k8XqB?*nlT_G!`%w1PG%r?%$LeS|s$4y(%Mf$1aN6EN|XYLNR*;81KYt$lSi z$X-PvuY!=K)|vG)!KdOr$*pemEZrg3%(8aSrnLXbH~QkNJW#O(xmNbgBQT_{Rn`%j z!v@-z2ptI#g5k+O(K~zJNnTAsD^QQ2qlWytI@|_&@${sMzjd8oJ`-T7)>ng8CB~Uh za2T+^Z{+=vM=fbi%%*{e<5UaufkL{`NpPV{dOT!Ay!ZENd;jt!wDHq&z~n~#!@SF-S^x=Y+ddZaM_!5O zZPwr-yyYOW7GCN<>te_&o3(aG&JON8jWk)KIu);>(8YfP!ZTsXK5C zxHsLJlY@0 zu``fCbAXj2Cj=q^T1ttu6^OkSocz8Uy4hahw=P=ueDJ6XRyGYxbISm}o|nPoO#>sm zewwK+un<%A@To2m%NQyxDu@iuzpyv+PH1ckZwB!!FTMSEZ114bs|Najre}m8b|6#{JuZZ9Eo)|x2qR2Zq;Ogr0Hy`x0n*@5e=R5RzIjx1NTlH;yXf!u=H?OX3 z3-D3P5b$MTl&}syEd^|DZ#&rcu6>%cdv&}zzdq-Pq;%Pn5%(+Q%hUl<)FWVv z1i`vO0hHUMxpX!PEu%KtKORaWUj<%Inj0pnmlo%iu)H=C(n9PICJLmPWNC)6gbTZ3 z)svHiOH>YIy_}=tNHQB-A>*bYvpvJZvousD1EuJ0Ub>Z@YOx9rFQrbimTqI;$N-=O zT)?Yd=F@mP*-A1lfso9!+adAc# zrrLK)!i2jV^{5xcv4%3m$npOq>opse0Itb43&yWyjZ{iAOj{T)(P}{p3LAEXprYSO z98mC@PaTh>!JvWBkon&@(cp5U!c_`_1nb#64DWgpJy`&~E2MM$KfZ+i3r zf-iqHkzbalL+=rv_I3f z*1x!WN>Q>Zo~`d*IlORw`fvpRK|w%(KyVNdsH_T*tA7{&8)0H;W@6*wX6N#smyw2{r|q@|DFGCmc(6$A%r0M z$UUlSn7HcxX<>Q<7`P6LgpeCKxAKRHGMlJ+p! zkG~&}^jX|xyXJ8+PwS|Z+f&hP*At%2WqSW5d23DXq+9KCQf6QH`uj`O@%8@u_bzAH zs5;Z!XS!86<5SVsnTr=@Uo~9OI{TsmN6PQJcJ6!r7inFppJS%I#Kfl9SBEpYA&E;n zW0h1{(Fp^M9r>X`7iC%}nOU-*{JP+HF4Kp!)MEt=!VB%Zf3%%lZNBcE6t7>*W zjcotTmv2rx_AQoW%he_2`lqf|>`^J&cK4!Hv-l~i1D5JffG3C=%%2r`N>?i={d8Os zlc&f%(X4as2_Ig)IP)RXY)^^hwsU{aRsCVyy_37)3URrK;^-JPU^E0qLtr!n=oSJ1 DHrpOf literal 0 HcmV?d00001 diff --git a/source/plugins/ruby/in_kube_podinventory.rb b/source/plugins/ruby/in_kube_podinventory.rb index 7ceff9c6c..44036a1d1 100644 --- a/source/plugins/ruby/in_kube_podinventory.rb +++ b/source/plugins/ruby/in_kube_podinventory.rb @@ -36,6 +36,8 @@ def initialize @PODS_CHUNK_SIZE = 0 @PODS_EMIT_STREAM_BATCH_SIZE = 0 + @watchRestartCount = 0 + @podCount = 0 @serviceCount = 0 @controllerSet = Set.new [] @@ -45,10 +47,9 @@ def initialize @podsAPIE2ELatencyMs = 0 @noticeHash = {} - @useMmap = false @collection_version = "" @podInventoryHash = {} - + @kubeperfTag = "oneagent.containerInsights.LINUX_PERF_BLOB" @kubeservicesTag = "oneagent.containerInsights.KUBE_SERVICES_BLOB" @containerInventoryTag = "oneagent.containerInsights.CONTAINER_INVENTORY_BLOB" @@ -82,11 +83,6 @@ def start $log.warn("in_kube_podinventory::start: setting to default value since got PODS_EMIT_STREAM_BATCH_SIZE nil or empty") @PODS_EMIT_STREAM_BATCH_SIZE = 200 end - - if (!ENV["USEMMAP"].nil? && !ENV["USEMMAP"].empty? && ENV["USEMMAP"].casecmp("true") == 0) - @useMmap = true - end - $log.info("in_kube_podinventory::start: use mmap is: #{@useMmap}") # create kubernetes watch client ssl_options = { @@ -101,15 +97,13 @@ def start auth_options = { bearer_token: KubernetesApiClient.getTokenStr } @KubernetesWatchClient = Kubeclient::Client.new("https://#{ENV["KUBERNETES_SERVICE_HOST"]}:#{ENV["KUBERNETES_PORT_443_TCP_PORT"]}/api/", "v1", ssl_options: ssl_options, auth_options: auth_options, as: :parsed, timeouts: timeouts) @Watcher = nil - $log.info("in_kube:podinventory::start: successfully created kubernetes watch client") $log.info("in_kube_podinventory::start: PODS_EMIT_STREAM_BATCH_SIZE @ #{@PODS_EMIT_STREAM_BATCH_SIZE}") @finished = false @condition = ConditionVariable.new @mutex = Mutex.new @watchthread = Thread.new(&method(:watch)) - @thread = Thread.new(&method(:run_periodic)) + @runthread = Thread.new(&method(:run_periodic)) @@podTelemetryTimeTracker = DateTime.now.to_time.to_i - @@WatcherTimeTracker = DateTime.now.to_time.to_i end end @@ -120,38 +114,14 @@ def shutdown @condition.signal } @watchthread.join - @thread.join + @runthread.join super # This super must be at the end of shutdown method end end - def append_to_file(podInventory) - # only to be called from enumerate continuation token + def populate_hash(podInventory) batchTime = Time.now.utc.iso8601 serviceRecords = @serviceRecords - # podInventoryHash = {} - - # have to read file first - # TODO: make podInventoryHash an instance variable so we don't have read everytime - # begin - # fileContents = "" - # # Read file - # if @useMmap - # fileContents = fileContents.dup if fileContents.frozen? - # fileContents << @mmap - # else - # # define path above instead of hardcoding here - # fileContents = File.read("/var/opt/microsoft/docker-cimprov/log/testing-podinventory.json") - # end - # $log.info("in_kube_podinventory::append_to_file : file contents read") - # if !fileContents.empty? - # podInventoryHash = Yajl::Parser.parse(fileContents) - # $log.info("in_kube_podinventory::append_to_file : parse successful. size of hash: #{podInventoryHash.size()}") - # end - # rescue => error - # $log.info("in_kube_podinventory::append_to_file : something went wrong with reading file. #{error}: #{error.backtrace}") - # end - begin if !podInventory["items"].nil? && !podInventory["items"].empty? podInventory["items"].each do |item| @@ -163,71 +133,11 @@ def append_to_file(podInventory) } end end - - $log.info("append_to_file:: podInventoryHash size before write: #{@podInventoryHash.size()}") - - # Write to mmap or regular file based on value of @useMmap flag - # if @useMmap - # $log.info("in_kube_podinventory::append_to_file : writing to mmap file case") - # # this is to ensure that we clear file contents before writing to file, check if there is a better way to do this - # File.open("/var/opt/microsoft/docker-cimprov/log/testing-podinventory.json", "w") - # @mmap = Mmap.new("/var/opt/microsoft/docker-cimprov/log/testing-podinventory.json", "rw") - # @mmap << JSON.pretty_generate(podInventoryHash).to_s - # else - # $log.info("in_kube_podinventory::append_to_file : writing to regular file case") - # File.open("/var/opt/microsoft/docker-cimprov/log/testing-podinventory.json", "w") { |file| - # file.write(JSON.pretty_generate(podInventoryHash)) - # } - # end - - # $log.info("in_kube_podinventory::append_to_file : successfully finished appending to file. size of written file = #{File.size("/var/opt/microsoft/docker-cimprov/log/testing-podinventory.json") / 1000000.0} MB") rescue => exception - $log.info("in_kube_podinventory::append_to_file : appending to file failed. exception: #{exception} backtrace: #{exception.backtrace}") + $log.info("in_kube_podinventory::populate_hash : populating podInventoryHash failed.") + $log.debug_backtrace(exception.backtrace) end - end - - def write_to_file(podInventory) - batchTime = Time.now.utc.iso8601 - #TODO: check if you can pass @serviceRecords into getPodInventoryRecords rather than creating a local copy - #TODO: if no, update code so we simply use instance variable everywhere and there is no need to pass serviceRecords - serviceRecords= @serviceRecords - # podInventoryHash = {} - - begin - if !podInventory["items"].nil? && !podInventory["items"].empty? - podInventory["items"].each do |item| - # Extract needed fields using getPodInventoryRecords and create a hash mapping uid -> record - podInventoryRecords = getPodInventoryRecords(item, serviceRecords, batchTime) - podInventoryRecords.each { |record| - uid = record["PodUid"] - @podInventoryHash[uid] = record - } - end - else - @podInventoryHash = podInventory - end - - $log.info("write_to_file:: podInventoryHash size before write: #{@podInventoryHash.size()}") - - # Write to mmap or regular file based on value of @useMmap flag - # if @useMmap - # $log.info("in_kube_podinventory::write_to_file : writing to mmap file case") - # # this is to ensure that we clear file contents before writing to file, check if there is a better way to do this - # File.open("/var/opt/microsoft/docker-cimprov/log/testing-podinventory.json", "w") - # @mmap = Mmap.new("/var/opt/microsoft/docker-cimprov/log/testing-podinventory.json", "rw") - # @mmap << JSON.pretty_generate(podInventoryHash).to_s - # else - # $log.info("in_kube_podinventory::write_to_file : writing to regular file case") - # File.open("/var/opt/microsoft/docker-cimprov/log/testing-podinventory.json", "w") { |file| - # file.write(JSON.pretty_generate(podInventoryHash)) - # } - # end - - # $log.info("in_kube_podinventory::write_to_file : successfully finished writing to file. size of written file = #{File.size("/var/opt/microsoft/docker-cimprov/log/testing-podinventory.json") / 1000000.0} MB") - rescue => exception - $log.info("in_kube_podinventory::write_to_file : writing to file failed. exception: #{exception} backtrace: #{exception.backtrace}") - end - end + end def getNoticeRecord(notice) # Helper function that extracts necessary fields from notice JSON @@ -287,9 +197,7 @@ def getNoticeRecord(notice) end record["Computer"] = nodeName - #TODO: replace w KubernetesApiClient.getClusterId in agent code record["ClusterId"] = KubernetesApiClient.getClusterId - #TODO: replace w KubernetesApiClient.getClusterName in agent code record["ClusterName"] = KubernetesApiClient.getClusterName #TODO: make a call to getServiceNameFromLabels -- need to pass in serviceRecords for this record["ServiceName"] = "" @@ -328,44 +236,27 @@ def getNoticeRecord(notice) end def watch - # enumerate - - #TODO: Check if watch pods restarts after connection is broken loop do - $log.info("in_kube_podinventory::watch - inside infinite loop for watch pods. calling enumerate.") enumerate - - #TODO: check if collection_version is correct when continuation token is not null and collection_version changes - $log.info("in_kube_podinventory::watch : inside infinite loop for watch pods. collection version: #{@collection_version}") begin @Watcher = @KubernetesWatchClient.watch_pods(resource_version: @collection_version, timeoutSeconds: 380, as: :parsed) @Watcher.each do |notice| - $log.info("in_kube_podinventory::watch : inside watch pods! collection version: #{@collection_version}.") if !notice.nil? && !notice.empty? - $log.info("in_kube_podinventory::watch : received a notice that is not null and not empty. notice type: #{notice["type"]}") - item = notice["object"] # Construct record with necessary fields (same fields as getPodInventoryRecords) record = getNoticeRecord(notice) - - $log.info("watch:: record constructed looks like: #{record}") - @mutex.synchronize { - # could be an issue here @noticeHash[item["metadata"]["uid"]] = record } - $log.info("in_kube_podinventory::watch : number of items in noticeHash = #{@noticeHash.size}") end - $log.info("in_kube_podinventory::watch : sanity check at the end of watch pods, need to jump back to the top. collection version: #{@collection_version}") end rescue => exception - $log.warn("in_kube_podinventory::watch : watch events session got broken and re-establishing the session. backtrace: #{exception.backtrace}") - # $log.debug_backtrace(exception.backtrace) + $log.warn("in_kube_podinventory::watch : watch events session got broken and re-establishing the session.") + $log.debug_backtrace(exception.backtrace) end - $log.info("in_kube_podinventory::watch : makes it to the sleep command. time: #{Time.now.utc.iso8601}") sleep 1 - $log.info("in_kube_podinventory::watch : after sleep command. time: #{Time.now.utc.iso8601}") + @watchRestartCount += 1 end end @@ -410,13 +301,13 @@ def enumerate(podList = nil) $log.info("in_kube_podinventory::enumerate : Getting pods from Kube API @ #{Time.now.utc.iso8601}") continuationToken, podInventory = KubernetesApiClient.getResourcesAndContinuationToken("pods?limit=#{@PODS_CHUNK_SIZE}") @collection_version = podInventory["metadata"]["resourceVersion"] - $log.info("in_kube_podinventory::enumerate : received collection version: #{@collection_version}") + $log.info("in_kube_podinventory::enumerate : Received collection version: #{@collection_version}") $log.info("in_kube_podinventory::enumerate : Done getting pods from Kube API @ #{Time.now.utc.iso8601}") podsAPIChunkEndTime = (Time.now.to_f * 1000).to_i @podsAPIE2ELatencyMs = (podsAPIChunkEndTime - podsAPIChunkStartTime) if (!podInventory.nil? && !podInventory.empty? && podInventory.key?("items") && !podInventory["items"].nil? && !podInventory["items"].empty?) $log.info("in_kube_podinventory::enumerate : number of pod items :#{podInventory["items"].length} from Kube API @ #{Time.now.utc.iso8601}") - write_to_file(podInventory) + populate_hash(podInventory) parse_and_emit_records(podInventory, @serviceRecords, continuationToken, batchTime) else $log.warn "in_kube_podinventory::enumerate:Received empty podInventory" @@ -434,7 +325,7 @@ def enumerate(podList = nil) @podsAPIE2ELatencyMs = @podsAPIE2ELatencyMs + (podsAPIChunkEndTime - podsAPIChunkStartTime) if (!podInventory.nil? && !podInventory.empty? && podInventory.key?("items") && !podInventory["items"].nil? && !podInventory["items"].empty?) $log.info("in_kube_podinventory::enumerate : number of pod items :#{podInventory["items"].length} from Kube API @ #{Time.now.utc.iso8601}") - append_to_file(podInventory) + populate_hash(podInventory) parse_and_emit_records(podInventory, @serviceRecords, continuationToken, batchTime) else $log.warn "in_kube_podinventory::enumerate:Received empty podInventory" @@ -459,7 +350,7 @@ def enumerate(podList = nil) telemetryProperties["Computer"] = @@hostName telemetryProperties["PODS_CHUNK_SIZE"] = @PODS_CHUNK_SIZE telemetryProperties["PODS_EMIT_STREAM_BATCH_SIZE"] = @PODS_EMIT_STREAM_BATCH_SIZE - telemetryProperties["USE_MMAP"] = @useMmap + telemetryProperties["WatchRestartCount"] = @watchRestartCount ApplicationInsightsUtility.sendCustomEvent("KubePodInventoryHeartBeatEvent", telemetryProperties) ApplicationInsightsUtility.sendMetricTelemetry("PodCount", @podCount, {}) ApplicationInsightsUtility.sendMetricTelemetry("ServiceCount", @serviceCount, {}) @@ -671,16 +562,6 @@ def parse_and_emit_merge_updates(podInventoryRecords) emittedPodCount = 0 begin #begin block start - timeDifference = (DateTime.now.to_time.to_i - @@WatcherTimeTracker).abs - timeDifferenceInMinutes = timeDifference / 60 - if (timeDifferenceInMinutes >= 25) - $log.info("parse_and_emit_merge_updates::resetting watcher to handle api server timeout :#{Time.now.utc.iso8601}") - @@WatcherTimeTracker = DateTime.now.to_time.to_i - if !@Watcher.nil? - @Watcher.finish - end - end - # Getting windows nodes from kubeapi winNodes = KubernetesApiClient.getWindowsNodesArray podInventoryRecords.each do |uid, record| @@ -720,51 +601,17 @@ def parse_and_emit_merge_updates(podInventoryRecords) end def merge_updates - startTime = Time.now - $log.info("merge_updates:: Start time: #{startTime}") - # podInventoryHash = {} - shouldUpdateFile = false - - # begin - # fileContents = "" - # # Read file - # if @useMmap - # fileContents = fileContents.dup if fileContents.frozen? - # fileContents << @mmap - # else - # # define path above instead of hardcoding here - # fileContents = File.read("/var/opt/microsoft/docker-cimprov/log/testing-podinventory.json") - # end - # $log.info("in_kube_podinventory::merge_updates : file contents read") - # if !fileContents.empty? - # podInventoryHash = Yajl::Parser.parse(fileContents) - # $log.info("in_kube_podinventory::merge_updates : parse successful. size of hash: #{podInventoryHash.size()}") - # end - # rescue => error - # $log.info("in_kube_podinventory::merge_updates : something went wrong with reading file. #{error}: #{error.backtrace}") - # end - - $log.info("in_kube_podinventory::merge_updates : before noticeHash loop, number of items in hash: #{@noticeHash.size()}") - + $log.info("in_kube_podinventory::merge_updates: number of updates in notice hash #{@noticeHash.size} @#{Time.now.utc.iso8601}") uidList = [] @mutex.synchronize { - - shouldUpdateFile = @noticeHash.size() == 0 ? false : true - $log.info("merge_updates:: shouldUpdateFile value is #{shouldUpdateFile}") - @noticeHash.each do |uid, record| - $log.info("in_kube_podinventory::merge_updates : looping through noticeHash, type of notice: #{record["NoticeType"]}") - uidList.append(uid) - case record["NoticeType"] when "ADDED" @podInventoryHash[uid] = record - $log.info("in_kube_podinventory::merge_updates : added new record to podInventoryHash") when "MODIFIED" if @podInventoryHash[uid].nil? - $log.info("in_kube_podinventory::merge_updates : modify case where uid for add was overwritten to modify within same minute") @podInventoryHash[uid] = record else # TODO: will need to modify other fields later @@ -773,53 +620,24 @@ def merge_updates val["PodStatus"] = record["PodStatus"] @podInventoryHash[uid] = val end - $log.info("in_kube_podinventory::merge_updates :: modified and changes reflected in podInventoryHash") when "DELETED" if @podInventoryHash.key?(uid) @podInventoryHash.delete(uid) - $log.info("in_kube_podinventory::merge_updates :: deleted from podInventoryHash") else - $log.info("merge_updates:: error: key did not exist in hash so unable to delete (probably add and delete in same min)") + $log.info("in_kube_podinventory::merge_updates: key did not exist in hash so unable to delete.") end else - $log.info("in_kube_podinventory::merge_updates :: something went wrong and didn't enter any cases for switch, notice type was #{record["NoticeType"]}") + $log.info("in_kube_podinventory::merge_updates: something went wrong and didn't enter any cases for switch, notice type was #{record["NoticeType"]}") end end - $log.info("merge_updates:: uid list length: #{uidList.size()}. noticeHash size before deletion: #{@noticeHash.size()}") # remove all looked at uids from the noticeHash uidList.each do |uid| @noticeHash.delete(uid) end # TODO: copy noticeHash to tempHash and use tempHash to loop through so we dont lock on it for a long time - - $log.info("in_kube_podinventory::merge_updates :: removed all visited uids from noticeHash. noticeHash size: #{@noticeHash.size()}. uidList size: #{uidList.size()}") } - - #TODO: Look for a way to replace only necessary contents, rather than everything - $log.info("in_kube_podinventory:: merge_updates : about to replace entire contents of testing-podinventory.json") - if (!@podInventoryHash.nil? && !@podInventoryHash.empty?) - $log.info("in_kube_podinventory:: merge_updates : podInventoryHash not null and not empty. podInventoryHash size after hash loop: #{@podInventoryHash.size()}") - # only write if there is a change - if shouldUpdateFile - $log.info("in_kube_podinventory:: merge_updates : shouldUpdateFile evals to true, therefore writing to file.") - write_to_file(@podInventoryHash) - end - $log.info("merge_updates:: number of items in podInventoryHash: #{@podInventoryHash.length}") - # $log.info("merge_updates:: number of items in podInventoryHash: #{@podInventoryHash.length}. podInventoryHash: #{podInventoryHash}") - parse_and_emit_merge_updates(@podInventoryHash) - - #TODO: bottom two are not necessary - can remove later - # @podInventoryHash.clear - # $log.info("merge_updates:: number of items in podInventoryHash after clear: #{@podInventoryHash.length}") - else - $log.info("in_kube_podinventory:: merge_updates : podInventoryHash was either null or empty, so NOT writing to file - should never be in this case") - end - - $log.info("in_kube_podinventory:: merge_updates : finished replacing contents of testing-podinventory.json") - endTime = Time.now - $log.info("merge_updates:: End time: #{endTime}") - $log.info("merge_updates:: total time taken = #{endTime - startTime}") + parse_and_emit_merge_updates(@podInventoryHash) end def run_periodic @@ -842,7 +660,6 @@ def run_periodic if !done begin $log.info("in_kube_podinventory::run_periodic.merge_updates.start #{Time.now.utc.iso8601}") - # enumerate merge_updates $log.info("in_kube_podinventory::run_periodic.merge_updates.end #{Time.now.utc.iso8601}") rescue => errorStr From 9b88d81f7f88fea551ea33547d6f604dd4f635b1 Mon Sep 17 00:00:00 2001 From: Khushi Chaudhari Date: Tue, 24 Aug 2021 17:28:35 -0700 Subject: [PATCH 71/76] temp changes --- .../linux/dockerbuild/khushi-watch-test | 20 + kubernetes/linux/kubeclient-4.9.2.gem | Bin 35840 -> 36352 bytes kubernetes/omsagent-watch.yaml | 917 ++++++++++++++++++ 3 files changed, 937 insertions(+) create mode 100644 kubernetes/linux/dockerbuild/khushi-watch-test create mode 100644 kubernetes/omsagent-watch.yaml diff --git a/kubernetes/linux/dockerbuild/khushi-watch-test b/kubernetes/linux/dockerbuild/khushi-watch-test new file mode 100644 index 000000000..69729d1ec --- /dev/null +++ b/kubernetes/linux/dockerbuild/khushi-watch-test @@ -0,0 +1,20 @@ +apiVersion: v1 +clusters: +- cluster: + certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUU2RENDQXRDZ0F3SUJBZ0lRWDdub2hxOWthLzY4OHBzSHh0b1RHekFOQmdrcWhraUc5dzBCQVFzRkFEQU4KTVFzd0NRWURWUVFERXdKallUQWdGdzB5TVRBM01UWXlNekl4TVRoYUdBOHlNRFV4TURjeE5qSXpNekV4T0ZvdwpEVEVMTUFrR0ExVUVBeE1DWTJFd2dnSWlNQTBHQ1NxR1NJYjNEUUVCQVFVQUE0SUNEd0F3Z2dJS0FvSUNBUUMvCmR1aE1wZkxhWURuTHoyUjNrVW55M2hwbHREdGQ4cmlvakE1ZUZteXUveUJZcUl5Qi9LUDNMQWd4ajVwLzRtYlQKT2RqT0pseElKcW8vRlVGdU51b1E3YkRpSlVpZ0I3UVVQVVl0dVZCTUtFUHpuZk8yUFB1VXY3UVloZjNXMmpQegp1cUNGWkFRVnZUeDZmVTZLRjNhNkt1aC9pSkZLY2c1YnZGUVpnOWVNSy91QjlBdXZUSEx0aS9VRUJjWmM5UXBICm9hbklsUGo1N3dVV01zV25veWxRTndwSk1WczE4aDhaMStYS1pYakkvN0dVRTZhaEZwTVdmaU5GWDgybGNlbEQKQ0NwdzZSL3NiRyt1dXRuRURXQVpPRGFSOERpdFRBc3hyUHJMNHdDWXI5d05tY0tSQ3JwUXFsb3p6WFNGZGN5bApwbXZDMExYOVUwSGdZOURMMTZHaDUxbHRrSnN0dS9velhCd1BSM1JrdVVJZ3Q3WmpkOERSTmtlbVVvdGlyZHkvCm16VTJmb2hpSWE0a3VKR1JjVnR0RHZ0eVFSYTNPMzNHMzNQRVJTbkxCblpHSzRHMkVMbTBxTDh1WVFOZ1IveHoKeWlTdmozSlUrOVZoRFplaFlyOWNMWXFuSXRLci84anhVSFozU3RkNjhpSkFvQUJwZzFFWFRDc3pxamdxckJnagpLaG5YNHFCTlpQVmVUZjNrQjVkZ0VsZlJtMXIvcGsxblpFQmMrQllkcjBoSmNKTGgzSEYyR2hCM1BLc1NwUXE0CmVibVhQalZmanNxeFhSYVN2MTdzUTloSmhHRXN6WlBHU29SUjA1b3UwaU04ZHF3M0hjTnI4NzJuYW5uK2pXK24KTS9EaWxrSnJRY25WUVJLc1IxVkJ2ZnhKNjJzOFA2Ynl6V2FWL3NMNm1RSURBUUFCbzBJd1FEQU9CZ05WSFE4QgpBZjhFQkFNQ0FxUXdEd1lEVlIwVEFRSC9CQVV3QXdFQi96QWRCZ05WSFE0RUZnUVVCRG1ORE9VczRrVE9OLzlVClI3bkQ0MGNnR1VVd0RRWUpLb1pJaHZjTkFRRUxCUUFEZ2dJQkFEdGZ6M2pjV2lNeE1zaDIyT3I1RWV2R1pTRU0Kd0JDZmQ0QWMwUXlENkRmVkdaUFlwZmU3aDIzV1dKUWs5T05ncUszdFJhY1FUTTA0N2R3WUJGL3FTOEU2M3pKSQpINmZxL1lwQW9adHpZRkdaZTU1bUFaWWFQTlZWeDdvaHdoMzV2RTdFY0JaZ3JuS1pUN0EzOXlOWHRka2F2cThYCmhHQzdMV1lIMUxjQlhxckxHZlkvL1FpNnYraXdzZWlFdDkvUnRrSjhJK0VFd2JTWU9EazY5Z1lhSUtyK2RJaDgKbXdZM0d3VTZJUXl6MkFqb3pCaEcxbzQ1dzh1K1dKOTVyZEtrS0lEVG0ySTBNMEQ3dWhnZ0ZDT2srY25sOWNtSQozNkdqM2hIMlNFWC9Ea0tKN0orZ1o4bG1rc01HbGh3bGZ3K3FqRXVkVEQ4bCs3a1QrWWpMbEl3dTZRVkhvOGxWCkwyZ3diTG9tTzNJdU9IdHo4VUkxNVRJYSt4QmNsNDVSek1ZWWo0NHcvY2JiZStxUDE5THFQSUZXaXl1OG51UE4KUVhtcUtVR2wyOE83RmtteWFOckpuQURKNmZ5YU5nY21Bb1RrSU1oSk5IMHd2UkNjcTBGaXdJNkFYSGhEczJXYQpDNUkvS0c4NDg3dnlIRUJON3RhcldLN2RMMEhjcWt5cGl4cVkyNERRSnZLVUlIdUlHVk5sMGtza1Z4OWRKekxOCnkrd25SWndUTEVxRkNLSlMzU1F4eFl5Z0xLd09uZGVGTG1TSG1vbGJYM3IzYllDdkZLQ3BlNFA2V1Iwb0M1b20KOGZvMUpwLzI4SnF3K091RTJQZkFhRGFBaFNtK2tmb3V3WnIyWUNHSm50RjFTOEdlSFQxanFFRkJSWnp1UWd3eQp3R3cySnB0cjFEdmJvWmhaCi0tLS0tRU5EIENFUlRJRklDQVRFLS0tLS0K + server: https://khushi-watch-test-dns-9d093a4f.hcp.westus2.azmk8s.io:443 + name: khushi-watch-test +contexts: +- context: + cluster: khushi-watch-test + user: clusterUser_khushi-watch-test_khushi-watch-test + name: khushi-watch-test +current-context: khushi-watch-test +kind: Config +preferences: {} +users: +- name: clusterUser_khushi-watch-test_khushi-watch-test + user: + client-certificate-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUZIakNDQXdhZ0F3SUJBZ0lSQVArdktyZDVaRms2ajFycDVCS01adjh3RFFZSktvWklodmNOQVFFTEJRQXcKRFRFTE1Ba0dBMVVFQXhNQ1kyRXdIaGNOTWpFd056RTJNak15TVRFNFdoY05Nak13TnpFMk1qTXpNVEU0V2pBdwpNUmN3RlFZRFZRUUtFdzV6ZVhOMFpXMDZiV0Z6ZEdWeWN6RVZNQk1HQTFVRUF4TU1iV0Z6ZEdWeVkyeHBaVzUwCk1JSUNJakFOQmdrcWhraUc5dzBCQVFFRkFBT0NBZzhBTUlJQ0NnS0NBZ0VBcDhoV0piY3F4emFrejkrZjdRdG0KV0xvVzB3RFo0czlacmlDM25xZ2ZzaFBiYkMwSWlCdGxtQ05lYU1pRTVXeDBnby90VXQ5VUtvUlAwbzQ0Rlh2SApEZGVhU1J1OEllZnh2K1VUOXpkR0VwZDc4ejFBdlF3djhKYysyUjk5b2RnbWtYS0ZXeVhZb0UxRXpSMk9mbVVJCnVOVnNlNldoUHdCbFhNWXZmL0IwYU0rcHg0QzNZOGxnT2p4QTdybFRYYW1pNUlIemxZclQ0aVdtbHI1c05LczAKbkJkVWpicGtsdngxZk5DalR4S0tRaGFNRGdMc1dUUXBaVjI1b0FzYXhIcUx0Z0g3TytZaGZ2bnBJNUxLV3JNego1by9TQ3Vya0phK1RyWnVTbnJKWmgvR21NK09wczhXT1NJYVJpcEtLZlVtajNONzlUeEhkWTJVdThpNnZiTS9MCjhHMVZVYThRV3BnZ0wrbmtSWWRZQmdjSENKVjJZRUovSnlmSm9oK3c0RFF1aUV5RmZyNndmWlpKZGNnRlVRcTgKQllJZ29HNnZGQ3hMNEZwRW5ueWJHcFBLemczVUtmVmxCMGJ0Zkw0TjRZM3ZPYzhPeDgrWHZBWFMzSDdXV1I2TQpUeTFvMGxua1RRYW1uUVo1U2Y4TFU0OERHU2R5NFIzUTdiNW9UTU5TYmVHc0t0WXlxYURPQjE4WW0zTE9CQ0dvCnU4VE4vTFlsaFRHY0NSVklTOFE2SEoyYUZ0L1Z6b04zbXZlYitsUDF3LzNNaTltYjIyOTFsQ25hUG9pcE1memUKdFpoVUNOUURmUGpVRlFyS0dKODNhVzZkTWRRTVY0dXNzWG1Dbno5NXlWeEx6eEtRSVEzM0RKYWhVNlQ1VUtxRQpqY3ZqY09MWmFTdDE5Z21DMXh5QlF4OENBd0VBQWFOV01GUXdEZ1lEVlIwUEFRSC9CQVFEQWdXZ01CTUdBMVVkCkpRUU1NQW9HQ0NzR0FRVUZCd01DTUF3R0ExVWRFd0VCL3dRQ01BQXdId1lEVlIwakJCZ3dGb0FVQkRtTkRPVXMKNGtUT04vOVVSN25ENDBjZ0dVVXdEUVlKS29aSWh2Y05BUUVMQlFBRGdnSUJBRS9XNDdtU21PS0ZSYmJQV01aVQppTmROdmpzOVQ5QTVSc25WYk9uKzN5WFdGSk9pM0JucFBlOHNRc094MTlva0c4K2pIN2lpZWFmOW1SNEtUKzFJCjNMSy9uMWRNTVUzZXRTdE9zcmpob2ZYS3dNbVB3YklDYXAwaGs1T2w5Wk0xaFVIWEtGWGVMME9qQ2piVG5wa0gKYzdTd0JEd1NLcS83bDhYaWdYN29wclFxRTZUNFRLMWI5SDFQQ1JjZWw0UFJZZ0lwTXdHTWZOckFuK2F3dkJpVApZTmpudXljdEpHdXlXT2xGYVl6V281Skh3b2JSQUpJaG9sN1RFckM5Z28yUlpTL1dwSXNYbHhLREFaSnNmUGFkCmsrSmQzNlV6eU1QRnNvUHZqUWhlajlIVnl0Wk8rY1piYVlYbkd1eWx0U3Y2dVErUTd2cEtJVmoxTEN5cVJUL0kKWkxjM2ovMTNUNGFwLzExKytOaHRYNmtRM0JJRDdQS2VXd2YxeFF2NVRPTExIbG5VMTQ0WXI3b1NqK1QzWEdCSAp0bENqdHF5clFxMERZaitGaVhyb3JuUGJRMFFybG0zVE5lTVRLbktUSzdWMDl4a2FMRW8xalgrOHkxMzkwUzVMCk03bWRVTmlzS1A4T09EUTNMSVVETWdYRWEwWFFxdnJmMnJlVG9pVFhwZlJaK3JZU0RMVitsT3VoYzhlSFN6aVQKWXE0dDhJMlplMHdUUzdrTkhiMzlPYVVHYTc1Y2ZEdHZBcWxNWUk3UXhkU3hVYXNFd2Y0NytwUXdMZEl0UWFXcgp3cFZ2dk5nNnBIZEdVbG1VQVJ0dEhNMnR5YWhRNERXVmFoM0lubFlDYzRqVEtuNGc1aldxZkhBNWFsOEtNTWhQCmpNVWFVRGMzVkN5RGFacHRDeWRHQUgrLwotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0tCg== + client-key-data: LS0tLS1CRUdJTiBSU0EgUFJJVkFURSBLRVktLS0tLQpNSUlKS0FJQkFBS0NBZ0VBcDhoV0piY3F4emFrejkrZjdRdG1XTG9XMHdEWjRzOVpyaUMzbnFnZnNoUGJiQzBJCmlCdGxtQ05lYU1pRTVXeDBnby90VXQ5VUtvUlAwbzQ0Rlh2SERkZWFTUnU4SWVmeHYrVVQ5emRHRXBkNzh6MUEKdlF3djhKYysyUjk5b2RnbWtYS0ZXeVhZb0UxRXpSMk9mbVVJdU5Wc2U2V2hQd0JsWE1ZdmYvQjBhTStweDRDMwpZOGxnT2p4QTdybFRYYW1pNUlIemxZclQ0aVdtbHI1c05LczBuQmRVamJwa2x2eDFmTkNqVHhLS1FoYU1EZ0xzCldUUXBaVjI1b0FzYXhIcUx0Z0g3TytZaGZ2bnBJNUxLV3JNejVvL1NDdXJrSmErVHJadVNuckpaaC9HbU0rT3AKczhXT1NJYVJpcEtLZlVtajNONzlUeEhkWTJVdThpNnZiTS9MOEcxVlVhOFFXcGdnTCtua1JZZFlCZ2NIQ0pWMgpZRUovSnlmSm9oK3c0RFF1aUV5RmZyNndmWlpKZGNnRlVRcThCWUlnb0c2dkZDeEw0RnBFbm55YkdwUEt6ZzNVCktmVmxCMGJ0Zkw0TjRZM3ZPYzhPeDgrWHZBWFMzSDdXV1I2TVR5MW8wbG5rVFFhbW5RWjVTZjhMVTQ4REdTZHkKNFIzUTdiNW9UTU5TYmVHc0t0WXlxYURPQjE4WW0zTE9CQ0dvdThUTi9MWWxoVEdjQ1JWSVM4UTZISjJhRnQvVgp6b04zbXZlYitsUDF3LzNNaTltYjIyOTFsQ25hUG9pcE1memV0WmhVQ05RRGZQalVGUXJLR0o4M2FXNmRNZFFNClY0dXNzWG1Dbno5NXlWeEx6eEtRSVEzM0RKYWhVNlQ1VUtxRWpjdmpjT0xaYVN0MTlnbUMxeHlCUXg4Q0F3RUEKQVFLQ0FnQUhyaXdEWGZjZlYya0Qrd0NmSVQ1MklVNmFLaGZQUWg2ZzBlNlYzS3hXd29IdzJiN2lQQjdTY0F3SwpUK09GZlFsNFVJNVVsTlNOZmJFSnVtam0wdHV4em9USmcvT0F1ZFZmSzJWV2s3a3BjTFhEMUxINTlXemNYcEFKCjhGOFg0WVVpYzFPWGNJd1NDbmR6ekQ2Um1wNWpsNkYzcDRWU0ZQcU8zS09mLzZuVWdtMExMT0U0T0NlbmdzcVcKSXZXbCsvWHc3K0h1bm9SRWZlUzZVYzB5UEFRVWdSemx2L3FLenRPeCt2cit5Nko0ay8rbFJJejlLRjRjdmNXWgpoZWlieGVCUFhKZmJqaFZLY3JZeFlxN1FxQk5nSW9WQzZxMjI2K3FlcjlodVcwdXQ1V251UmlXbWpReU9WVnFrCk5VRVJxeVZOSHdnSmJvL3IxTjNwU0NuUG1WT0Vnd0t2Qk1iN24zZFE2ME1TZHIxVkNKOVl5ZkI3TGlVODVOdHYKY01CWWt3Y1FocWxPR1o0WlZJa2VjaWxvZnBmY2lJVWlzWmJpMTlsUnB4SHBoRzUvR0N3K3p4VVJ2U0xud0Rqcgo2Zy81ZCthWjQ0bFlSSzIwS09OQ003VnA1dHRnZHJwcmtnK211cldBZ0tIcThJQTB6KzhzZlVYWDNXUFoyelVTCnppb2NGcmlmdXhENm1aMEFHZEhRaFdqK1dMSEpYdnJtaENtYmFsUEk5NllVVWRxZlo0RHdkL1ZZcXlLOEI2TjkKM0RFVnI4SlhKU1piUFV1MEZDcS80amV2R29JekJPVHBrTVU1RGtSUlA4ODgwY1ZLMGoxNmwxYXhiajNYb3hVVQpuWmQ1UDhSMWx6dTNONzdYUHNydXBPRDloUmJXRDFiQnVKN0E4aER0bWlQZ3hwTUxRUUtDQVFFQXkrWmZHMWJqClpnNUZqcWhOclR2dDVENmlmWmorUkkrL0YyYlpLdjdsdVRRRkgwemkvazhMSk44VFQ2b3ZYQTFHM29vQ1ZNZnAKNUZEYmphL3l1azJKejFwcCtiZ1pYeHdwRUdpSm0vRjZRVGQvM2Q0RzhGL1hnQnZIb3FHSXZCTzRFRlBCcTFlTgppRGNCMEUzdnZTWEM5bTFsWGgvSEVHczYySVg2bG5Nd2tYbUcrRTZMT0NTRTNQcTFUMDQxaWdUOTNYTmIzQnZxCm1YWWNBeGlzZVJMUGY3aExpNVVaNTJ1ekJLUDJ1OHg0R2lEOVlPQm5oc1g2dnBIUVExOWozUjlLMm52U3ppRjMKMWlTWVVLbWhIU0xwY3RHY0ZzYTdRYWJvVklpdGYzOWEzRHl4dHZXNndPOEx1TWVGRnMzdytqR3JrbTVIZ1BjZgpMY1A0T203T2xOa2JmUUtDQVFFQTBxZHZqUkYrMWxlRHZJUFhuSy9wTk5kTUNza21DTko1b25MTllKTWxNMVJTCk4velBZbjg3bXRsbnBzb3QrR0ZXS0pEaHkwTHlhUk4zZERacHJvRWxzOFVnNUt5NFZiZTV3OFloKzR5K0FrMFQKM3lwSTVDdUllNFpaTndpbEF5RkVWTkNhTXJaU3Y1eGpaR0pZQU1aa05mRE5XU3NSbFowUEkycFhYRDFCVnA5UQo4djVDWG5UTVZrTSs4NVZVN3dKMHl3QU1IRDdqVGM1RGVnaGZJT3NlZzVIc3RBTTJSbXc3dHU5YVVlL0t5eGVDCjZPY2NOMGhtT3VuZ2lPakE5MXYxWm5nV0lxaUZMZFVUUG9YS0JsWlRwbSt5QUJCTC92M0dWYm5NcUdzcnFlV3kKaS9ZQ1ZoSlVXcGZPa2t2djVUTTJXU0hKTGZYejU3Q25oL1Jodi80RHl3S0NBUUVBbGVlYWlJa0t5ZDg0T0RkNApWQ0k1TlpMdTk1UGhiQnFhak9QcVNYZjVBSDVFUGN6VEhkQ1RDdHFPWWdWbXFEQ0NwOTJpOVIyODBVUzVCYVFUCnVmQ0RudFNFRVRuT1BXU0F0RFdHNWdWVXNsblJRaGFYMTJVL0ZFcFlMVExCU2pUZEgvUFQ4TnAvaldPVGk4ZWcKdDlqcFN5OEdWWHJiYVRETXBKOXJxZXlxQ21ua1Z1MjM0T0RJWllaVWdpZW5xUDhlZlE3d3ZCUXlGLzBEZnl0RwpzZ1NvVk9memNuMTkrK3ZzUXo4Z2lOVThmMGs5djFsOEExUE9rd2kwcXJPWXdkcmR0MTlOam9xQWhnbUZpZGdNClFWV0hlQWl1enZmd2Q4WDBEU0luSEJOUGc1ZUR1RlZVcGIrWlNKSTNRSnJMemNWeXRFY3JmcDh3WnY4cW9oc1EKK2RIY0tRS0NBUUIydGF5RFRzeXZkWG5qamxpL0Q5TFgyRXdkOStUYksrRW56cEkyVGpXMGkxd1orUG5WZytYUApDWjBEdlFQUzZPZG96Tnl4Y3ZTd2lpdlM3YWI0bEFidkc3UEJxaVBuQ2paQlFUSjlVMzd3UlFkaHg2NTBCcVJQCkdCTEdsTFNJNHdKaTJYdE1BTHI1QitScStaQ21QeWJSenZXcHZqK1dsSCtuY1pIeGhRT0JFUjdKRU1mTit2djcKME1GMCt6NTF3bWlXelZ3RnZ0clJTZDkwc2FzS0hmV0FKNGZBbWQ3SUtNNHQ5UXh0Q0RpNzRtLy9WOTNxdVg0Qwp6VEdmZGFyb3VvVzhUUWdNVi9OVk5MSTNsYVdYbEtabk1LS2FycFJsQ2hYdTBWbW52MTFIelEybEdlbVBINTB4CndFTEU1czQxMTNwVkdGa0s4WU9WbVZPUHBnUS84YUdwQW9JQkFFa256SzNMeElSRmowVHB4TlJaSEp1NVNVdHkKTGRGaDQwd3kyUjNJWS9SSGpBbmx1TTBmUUQydjN1ZTY1VHhaenY4Y2tiZ1pKTkE0eGxTd2M0a3U5M3pkSkxYZwpLZzZPTU52cWZBbW9nMzkvekV3cEZ2MEZsdXpYTWdEdjVaSk5EYm0yMjhMaFJaQ1E3TFdKNWZMZXREY1BVaFRVCmd0UW9WVGZ6Zm5jTVMwa3IzQzdFSnQxam91UXMvNFhrVlEvZTkweTVkNTkwRGV5YmRSNnlDMWt5bU9sWVZJWDgKK0drdkxuZnFpMnZRZzdVR1hNTEY1LzRRNjZYRWl3NzVxcEdDTUFTdnNSa216L2FWYkpFNzNvR2dnZ1BGc3ZOOQpVcEFuR1NqaUFpd2Z5RmJ5d0NRV2hnUG1SbDhXV2xjTStZM0QxL1lLTElTVU5xQURnMGZXVXdkYVhFMD0KLS0tLS1FTkQgUlNBIFBSSVZBVEUgS0VZLS0tLS0K + token: 71b6ec454624539f8cf4a07c38d4c0efb015da0d5a9720766ab1e1d7c68c5920906350d2f4745a395a55abc5620c5f0002124c2f870358b61670078f137b5aa9 diff --git a/kubernetes/linux/kubeclient-4.9.2.gem b/kubernetes/linux/kubeclient-4.9.2.gem index 84f2aaba171904dd94cb52b8654ee08d8abab947..dc158c6e9160d9c43be0ecdd3928fa9549dd1dcc 100644 GIT binary patch delta 32753 zcmV)6K*+yuwI`D$SDUxE4T6jYX7R=`9d81=&FNK*w_yr^?vQh@f+ z68o@cRg(I`Se3=WC!)2V8|$(ti6>GxN8|5|rY`gw3%}LE zV#p{LnhTr7_#QE^p0GI;IOWg3M4WoG7!SvzWcXDwK8c6Jtocue>sS0!#GwMsK{EkO zth<2Ut}RxGxm{^qYc9cJyF;RI(YqEqyBS8~Epv<(dJ_CQdi{URQPT*`hZt*L8{S@w z(l1|Nw_jBVD&uek(o|ir%7O(6)ZtJ;rGp#B$rg;BVDCw_r3?6MJpcLnBLR=oVV^{Q z8ty%rRzeF8&iB5nhljYQpU!h)%m1nVXQ^I7t_)lBt{=yTDIdpAP!F^*%v60-JxUSu z;Zfbf-jnLmm=b?if#;*%H})`nd59akfxWL>8$43>e*!s7qDN04+F+01gc+^T>=`gV zI!wk#dox~8OF3OVGpcQ^y-@I-d(a`$ zrfDg$+%QszwN3j8r#ni43vVo*ObSsRH9e*9^Ew|aja?MdEZu;zfYPNaB{vqL)W$;F z?CwYF@i%|pPcBZU=kHHb#j%NGgmC!j{Ory2;&Ph$Ymcqjbn^Dyw6&Q{&!>~ivy0AU zMiX< zoxfxi3D+4?S#u>g*n6$;St_de<1;o_Btol7b~S%$!l9tG^pCm*=^$XO*^NTzP@}#F zq7u#tU6O?e7>%khbS@+^wA0)O&h}Wi1!ro@;K4ycr3K1Y1g`N}xCd6S@zBPT!qf=$ ze&a47caANoXLEvIu0T~Vm>UHZEj#&j*dY#=_1JVyqO6xt~)sidP5%gEn-^;|1N`D zBKWv^ZQ_hm6t8U%Aua!XAfZyW{ap|ky8{RHIz5gYs#UXhVTDAy+rRJJn4Uzg#tgKp zJ>)plIB&`6Uq3EMt4JCE0000000000005Kx1&10oFfcR#F*GqTF)}kXG&L{)FflVT zH8%htFq2^cWU~PQ0|9?0J0)QP1MI!~S{q5SFFJqgDRN+@C3$q^W?O6AZVa-U)fm`- zyQeq9k_xGS8YGo=l>}DXtnWO;dBXD~JAM(FSy`o$z;1WX%sS@lHdIyKGcq#nQSkHXQsu=fC;1*~pxKdeiuN{(q03 zkvH^PYGq!#?$y3zm>X(yixDkkL9VVicg*t`Ok@Wm3=v$4hEgb|36bdo-C>T zkMM$DTug&OU%o`sQCy|2)t^3mP(zPU#rs+qtNVNY8WVrknD|#SDXVk=^Mu+Lvz9tj z#kJaIv4US)^6M#nt=Ag(ML6d2a^RzE_1gFPMRjf)2xF>VoL)>vlWBE8eZ>=08G5}i zHqS}vi5K5cdAv3AGQ6Ti=6w{@yK zlGHftH#1XBo}}|uMx*=T_p+DSm2_XACA^Bf8{d78wvt8n$9CVQG3}w!D4|X}+Pev- zlkaZV3NlsmCH5!Nah03z2YodP?NpUse-%cw@uNQNgz9;{Yr2zD3XKNRf6c8d{gdVj zVSh>r{{HlzzCgYpx%a8(e{FqhBYXb0zS4ic&(HQ?uoKFCf9ge7{^XeM(I3tIN+bu} znTLZuXcv9!s+;MB-x~zP1e!ZOrrHPo9XF)>5iZ!mTc0kdUR~c-saDJ zeIwgqI|DC{A1t+f(K{hNG^fPAHzUHNzJE`AXVS`B?|n-V@Y(uDUPLs~#EA53OWA4wN})boQ|zrVcL+ps?! z_zNd|0`6L`-44+MKM}csQPBiw-HEUk(%RNyMp~&ozRCMi540_a@*wgnL^uR7{eRNb zYDIaINkk;O*Y_j(=wD7Jco~C>$1~TW??j8zTtMx%-wU0q&NR-hsAD2Vw1oAyUKC$@gZhV!%Y{r16|_Wt3U+OYr6x&PPJ zwzjhJpY`U(*Zco>`B_yv*WT#L2UT7;2q(-O_xM$;CZU>K2eBGQ;XnM|gjjK6U@v?{ z)c+L7fEeRoBt;mJ7EZ5yMF*O$Nk#82yqHQ5#c=k6?~j$2bah?=w@hsG4Sn!><~6g^ z{-}T6tTk$ldbw8kAt5#BF7l~8Lxq4nzYGVESgJER_e8QtqhQWse@Mk@VRVH?3ahK? z-ALE()03HVzfe`(q>EN-M{b{yf7ex`@J@MyJ8u@7>o8W{NO|MH450$a1*S`^xre*H zwWzfc-C-3~ylocCT^`Ox?RjlY{eS<@|EpWpi2JQJHmmE60*`FF-{*0G3{&g1hN2}I z(!D$Vh_LRkCmI_O;kc<5M;`gEyEi^nWZjA|e>8AxdY-5${|6Tefd)w*gZB#*kI zJC3dqx+!SR7hWG@(c~%g2n~M*GlJ&#wB4)g z&1baM8_j22)e-#+JQ4lKpH%>w{8_IW5zyLNV`IH+7qOwLb_pBnsjWWIf-*o-mo_A7 zv8<=7Xns(&1l~Z62h*!yg#E(cPxnuhJ{I1TRtMv^eLbVTk_~%`-WGNzyHt&d8>fXD z4**~r8_x`AnN8j$m?nS1vyVDg;bh{Eu(3i$%y7W}5r-q*K{fJe%zc^5Rd3t{?zpAq zBiYApZ`kL(?((1;7R;WeM!i*SZls66=V7h(G<_Z*92<_QKca_%;dDqxmFUHoNY*Ot z<)|7HT}*h3@K`wV2~iOQ0!m|y8I92x)}PZDoc8a$2*@SoZ|i^9{`F^EVsD74P+xc6 zgh15bKySjRKnQCFI{(_c4QT%3fj1gadFov-9FXNEP#d?sWmeg2%$qR0BILVG`w;ZF zSz7o@Z#tN$9SaFW)N^@(lz_;mnomb*&_y>|1W#XMV(I<3k{qXz58}&Sz*`2eNnI!BsXGXU!Q|D95N%h{KG0H+4QZ~=gFoqv!+wWux~{aaWe)A8 zyXcMQ)wQSjhql|IdnlX|$*hrD*Xe}7i~C5=wnP&Q==frS#E!;{{)})QRdSZ7EW>N? z1doXee4l?g@LeFoE(^A)S&!HJBBW(quf69b+^D_hHkuYMZKS5SIfs|t>>MdEL(n_K z5Pk+&sTw45eg+-Kxq9(J6$!f(yF!uxT@nL3k1$nf1Zo5*9UDj(X%J5Px*PqgdhHL! ze$>S^M95`Gqc79B@keIQQ4|~$n6{X%O56$fUcVh)S6V${cx9}(InaJEx}~nL&h#lF`s9+<;OB5!G-ZHilt*+Q zD`gm|ICNsCANt&P#|S%k$_>UAjV+BW5(4m$zE>92E$Pe$+t>eShxM(X9~XL8ep{6aeV7_l%dUIks*6`9|hi%79tV zy*trI{$@OG1Hz!+`vN=gX)6Z$D(H9go%n@TZ7X%7@}&vVvNPr@YV`w_Y%;s=@`>pV z{piY{`&Lukx#Bz^tVb}K`sSwB)&+k-PjN$vYsoko=FqKKHVuc4IJM{LZ9VYsv}O@=tLicehiU^uZW7-|zlK~O z#GL>}^dffq;1L^st@#Y(+uBx|e0v>&&Jr3zV^1s$xUs%R_$S6L0k$gGaKe9K25*7+ zCDMO5o(QQKhl5)I3LzS}Qo{3tYMSB>Q5JrA;OTt>(e2SJd|ndof_4N{5UzhqGK0Dxg1I0hHMdJR zCp2S=b7&@^rf?<;5TjBtE9Du6&>aX9swpD|(CLU6Enz$~VCnmx0Gog1#(j*Tz?6Xz zIm2lZ$rPSY`fYXftx@N5qbPx*08U&)xWHZCGCyS!AemWfnJ0H2YTK{zo0kD$&1 zPcu2p|6`8Enlrof!EZjz%nrZZ0SUqxg~TRLFEIq#{|FaSJf(%Eb)Ko}w+b{bgX&YK zC-)FGt835r@wolwxP5fJg)|o~FEF|p(9rj2sCuZ?iR9nLU*VeP! zny!22)N;l^z~6shqCleqx1r?PJt0V7#Nn{74qwZtY^^rKMAmCFZpFw&*3;Mn5~^n2 z`O3Q&pl0pTPF)d5ir_d|gI*+Ml$fP2q0(PaA1Eqa*YTzFK7l?amC+?7y;$6 zlCLV`zbC!%&Q)^u;Pc8N?{mvltk%7&F|e$KL$LmbBk%I0 z22lnxb?txMg72gI$G^Pf16b9!S~3o1{_QnO1z6m`rhmAJC&Cie*dm+qA-|G~^5A+O1s@ zl*GhFCCkGMGi~wrr>94}xmt?WhWx!t3(@n1TIPRo37g^ZH9)k|7#rQ(%;Le6FpcrT zH6hS$a(sYwnS$!z9fM2rd07|-g5&{vIK=#ZZL2I&w752>5#j~o-@xswe0#xpLHdmS zNd-(A-Qh7PJ&nN4GK1Ws_8S=ivZ42w>}Sl0Q{>21Fm-G=SYj8;896EDqHiE~u#aN@ z9r1sT7C1XVO(#JZ6PG_Nt*$i*nG&z#(;1l)P#G@Bi(9G%AoTm1KXcx6;@R*b#BrpL zq9J4i*N|z5&^Fe(88ZkpU`q3V0UCGLAvBbRUSE6u9P~P?!?dUppq8r#=(@<>iY*DT z{sousolnFZR(9Ru+=c`8Yt_a^=6>9TZVi889A+~u)+nVmc=4@Tq2a9Puk>p=?t}1oLJKqS`44QIu+L9;EJR;SBY@$tSy4zbPDHQ53>1F` zg~Ry72#?V)P-~7dw5t)NzJXFDzNaf$V+3fXrWa;CX&5!NgE6s0A9dOWWe}Q~fCv|& z01N{mG1_I?f1puDhB*g%RnuzSWF-DfcMdd6=m%@uJtibc*h7CYwmiZ^G$!G>OQAd-B1ZWt9r^?;n&PCv zXbM=1j!AVx1_>ro0#ny4(Zy4%zTcf z?P}-~nF-E(qSJr`-2kUmPq60waysB0A)3Aij$BId0G|qOWc(9(cUBsc2v~op)H6XD zV2hW7BT));OncK$hd%12`FvBKkGX%p|pm-Ra1LaByWrt|am!4K`-a~Cm~ zBb+_KjOY?%pm$qK4cKInk-(a|4LsVar%k(68%fVZ#x_NExJpBHtecyVF&2(l4&z%c zR<)fw9=aMoqV<9WWr2ULQugI?&%lWv-3C3sz}GC3?s7|xh$-piS22elQbeiu6U#_^ zn9+w(hU*bc$xY}=T)D`nJALzLorTc?O>E+jb1RnzZ(2ze)jiv+6O1?+T{J@ zpLPD%8k?K*_P<+S&;RdY{~eBJQE+ulymYUOt8s&vapFNd^(q{B#LK1KsK-x!k=JfgwIGppc_ z!^_DXh$N7qbiIZ>5rOqH48{PwHQl3yl@ncKrOci5eJBw`LT5f%gBqg3;CGU-(kqzH zY_ugkFrkE{h_%eI!sDz!>qQu+B5LA_8c#21g{~`wet>_bFW7Y^#!FtMijm#LJ}aHa zz)ITK&5VxAP%nis7IdPQi2J>})-;=)3EMzKp#4Ol-VbTXxW|8p#Y%y#9@rRK3;D!P z0@Mi@aqogSGMF@~ z{Jw9ilf!@4r$22Ux7FT>Iyyf5ac{T1t5&v8=y|20e%d?z{_x$Yq7uj32d6))!`Ev2 z;Ai#2-ob9A(Ei&IB;4xoSna(%+TUyMR@C0X&i=dIy@NOE71e`=aDR`$lo~xfROnDQ zwb!P4uM2P6$MAAE-F~&Vzjyj`MZMlTJwThUsnLIJb+mnay0`OgfBRS+y*oZSJZV$+ zyM=?pgS~^-$J9&vZTsM~M!iy3s{JE9sFUxv_xHK0?RPZ#V;dvCr!Ro@@( z@3!gXt9D_3Z~N7LTe_l2?d)&wy{)L-?YG-++Fa|9S~*rIrbqYF_icVf{cY2KJExHD z)0%(m93Gq=(_@8Zd3*GUe9?MBJ4!IGkchHtru+%EGBUA*> z?@rpuSa#dn`_$G6D!YYiUvJOq(h9Tv$NUc;bvO-4=Y!V6+>F9sIh?&$Zt0UNfl|V7sp`u7_bI_X`fJK7$J6{8mI1+jlvi{{Ky zwBPTJ_q0)qGp1=2(17Sus|EFPjblrdxO24vNh$t>yqPFXKxj*0OyvjXl6dTyeOOf| zC;PMw6J>4av{cTRC@@4y03nuvPhkNb4|D`0J7jugR2|g+hl0|x?DTw$kA=f0`>odB zHXF}rO!SiNXeKNgEU|0^^4tVcu?``VN+K}@Y@aQ|p{h0^xUk|l9&CjEz#(~fzQnJ!)$cZ6)>1eI8@OHeE_sSz z*~QxC!i$4m%B}-qkcjwdDiax36AuE+xEBQ%e!p-AnMy7tBgPs#@LxKj`wUhQ&ed1H z7=#yfA!hU)nCg}Y{IXnEW`7rAEk_MUenGSCR)pZ}1DeSIB}}U&)Lha0$(Ug5#WA9% z(0lV|@ykJ=JCIiD$?QoOEgiP3dWTn?hWK{3{r>Gw|9SKFAA6I-pMGv0%o>fuH*Yuh zfBMhC!ESr<_8+f;x4&GyKis`}|MM?D|MIUo6^)-0+?>4qU5AVl@qaex6X65Nj|iS# zdy&`oW(qg6xyp@`krPG86E}WyX;R(R*tZJc(wSYQ=&5sCvZJ~9IfsZ}C98=pI zMIoU6p@r?t?KYPTVx&ze726EkOH41RYev`ARFSa;v?P>v ztdSehE?QPt2<3BM)TqhIu+y>$yukB{zbvWAy`?bx5D}8rD(om2-9iP`s5Lf|=MFJ! z{ct#(P8V6Zkewc?c-sEw4#Q0U^+;YV-qDJ?jSNA>Qm;ZjC&oJJ=h}-W;LxA-_w=} zE>EW|5aoEEIe(U-bOza`4qHBvyM6m;PX`!S%B&%TC3t$a8+LfHs_x5tGLNhBOu$>E zUZ_S;g`GpytByl>?uy+cd%}f%tbore#as>o%*pO#(nh#?^R6>pM&|Fu;UZLOqtuP()jkqGhy=CV_FbEHE4FDf$Vz zWPOhGIMI3|--FMs8L?<|s4=kWtvLe9oe*z&y@r`K+b6H5j{pUWa`=Q&aPy&zl~OeIbqF6o0%DM6>zDr-P$o%ML@q6m-{=$T?i( z#fK3jm$1w57O+DX2ZlIiBT0-_M^Zr80irZ&32-KLw84lLL44lOwRq-d&D2~5qduEH z!epF7{;+^ib6PaZGamQVlrFa4^HTD7Sh{y>J(fhl$`N7qhruYr#MO=ECVv!e>VqnY3Y-beJT9~B_^pMw|_Fv1|u-h?`cONaSy278FN;>PwWE;KD`Qm zSFOJg+xu=!Y#}T3S7Id%Zvz38qbPhI`0*nEguY=?ijBVwAz}c8IG`DSV6>p!4dAI9 zNw^AYcL`q0k<-@F7nIp#Yhb=uayyj%@RSH{HxN+k3vEx@(!HhZpDsUpY+Un`1%H(N zW2Z(AX~r%j#)TgkhgZ{4k1YW7FFB1==mFx5+$o#}L!VF}&EO~FL18JuSZED~GF(0* zwl(EY8o-*g7k79ASPnt)&_u+&5{e3jQZxbMkrBNHayooGkg}hgmoWV-5tP9ukfL9h z6AxdW76(F5t7$BkhzB_F$n7HiveD=nY(?Vpwhb3g)<#%0e$5%N~Jm6UW^2V%VM1=C0aswCVO*9}J07VSS{Ay(~^TI zM69do5oH78JvKt%V3f(KShXTYYF3#>{l3+$B^`myu;_N)5yl+W;w{#Oc29HzEQ{z( z27(DRGVIpyRcYmt=nv6$F=;CBBXB-!!G7XMq(SZz0p>(|K?i)< zc1MWqsa2+NDuR&6kqjKU;RF~DB(W4Z>rlY4EfeQ|u$oe(Z5s(uE?StTSdk%XX#^Pr zBQ}_M7|T?#o84Nu06QOhF=^~>iiEJm%sL^U z!eWAd(gb6pE6WCgaV1MB*^w#iyquRl*%ot@aU#G^f-<@RWI?67pAaP4(5o;OPphl- zl}`}AxBZr2<|hy!CV7^|;%eXx zt4@zljuSIM%_MQ4eXZL|$;_`=P8Q*``06Zwx20U^MS?P`F&&P2Zp|{AsUZ}T=?(D* z+%SY9!Asqd`N1^BU$CdOd1H=t?Q;Zd#KCo=EO?Ms%F$yLq=aeE%TMz+%bB-HiS(UN zZ4zh8L@hU6a=c7UL=zF-aijY|K2_)@d}}xDRaNR@rD(}+H@amL=1+^TJ+`WoNU&^w zP$yFL8s8ui+z?lY5$*T^(=NP#+-U!%t8UkDP|b4{hKKZWU5EJ0%svT1xP2G5QtKTW zp9r=_umS948EcG|#1F+pG|iJexuYl+V1UPQJ>!DAUoR9ocYEifF4rzBuOLW+C=-9< zCOU*{WE_P+;dw2cTqDGGwmn^;qYW;9^hqt2Yd!ZwKWOgo5Tw?)T*+y)qWqh;``FdIAd`5jJ<|MvFziM-V*(VH{0a21K`!e|KJ5y$vh)$v^&@+vL^Ds*r?M$$S63r9cBl7h@TWCO}@oEr&m_#qV%l%54ju6pB&G69M? z{GmL0oYsgRc^#{4A4t#l_uYy%_+_cRC5jF4$r033=cZOD9VDTZL2?^)miIuIW!(pe zghkt|BS*=tkIo*wxgKvX3tsYn_Er7f`$~+%1M%GXI@Umdlu?M_*dEd5BSDdk2yb5{ zVSCsVOug3E5~x8P!v^pnJJ79KSzliXtrCkn`q=|Pf0D~nTk;XrT9e2>e2+@Zf>dF0UN^oHH>-K)hoAPTTp3#uRp0G&UL} zLnd+7K@TyFRULohM)&&`EcYgA$e}zqd^Cy#F*n1s&vZmsiojeBS&Vg+HMM4Q>?+ zkq;1eZXE)wFtUX!Vk~09y`Guq>$WDRZJT9({K6j)cGod$7|hI4j$)V)Z(tb3M7kkc z5m5X%j0qXmb)trPl5w*(?62C`kX8LXuF`T=k(Q50@6n_h-}rYrb*Bx^zQ%{2H&1xT zn-K_Hx%dPzfI%49#xwQTIl}UED2P1?x(Po|B2=zm;CK=CX98rx(`s)-m>qhV? zJvq}hh89E&Tt&=lfAB-)!}Y@ox#BXB!iDQr8@viS>QlO0C;JtU9p@L-=u#KqeOqnaj6 z$T5{8AjZh~EIZZtKhuy1VDp-PL*2oP@+xTwa(x|~`8FKT1-tkQOy`54{cbP(ju_JU z*`ar|vqs!?s42wjeW7e{PFVuke6x%;LQQ%VIqtoPA}owM;R5t>r5iVIQ@67dn!wnM z8A_1qB;5-qEa^PVAc*<)iA{t-gV@xH6=xEz932s}%;h@BtqtR#t}3}G?@#P&nvO%} zh}mVi_0V#a(yi;G_W~B+aaAb0?8J-(aAJlh6Dcvdk~Sn&s571Y9Co?EfH|v~dIIU> z6K#~V)R~8l2y8FnI_x#UcyTN#`0B8yiWZjCxuvQZatyyn^%GBD8$N+XO+uqXFX zEiteJtsK)+~(~AOY{{#-(Xgy0hPe2%V%Le?#fvlpfP4EhqYr+qE5UjC09P5s- z?Z!0|)_=D_f65Utk)=S=^~GTl9t$?Ua~YhibqbHGn@*y#B!Nn1d~8W6R)3lhwhy&~ zhQp8|H&{}lC7a_xJbvvactU1Px!v^`0w|7Ilc@U~cr4#=>g3GdYQi6f*d<9>%Jx&z zb-n;KT;S04{SXEwrkxzD1BB61%}%4ysMv&`E&EqlTjJ?yuSdrnw)J_&e>MgzJCIQd+48FRMq*~E=~9W$D>h8LY-WuEF>4PwDK<5JxBBl z${VutHK3F2ApnR;VrtLi8g&)C#?6CMPQ-Wsu>)bS$wvf7shZlsY7YlERXu z*FvUoX*Oyqn^dN|(QK$UL`HPA1ByAVT4FXtNGkDijwLYR}*CkTI*u^TK44#J1ov|9HybaOQAY`v(j99iUd`)=8B-D3@MyT39FSWjPxQ5 zZ&oU4QrNLusJc`s^jW|REN-((lWEYG>n-wF>tjC%8wd>@ZOUX;EBvpiAI(fcGt`r; zC9$|Z*;B!?y2yhsf2D-aiP>1wyA7!Z=(cM}Y5C;qM0X-mgwrUi?{br?7(E!1Mjl7@ z7(?DI#-VJ1n#~cUQ}O8#I`ghDMv)XrNt4Q&A*ZYAsw>hYo1UbmgMe_17$y!;?E)Ax z5UPJ-?XKOqH z7L(?6{7Z3E(iz`~f^HQEzPe0qsgkC_%97V5ZwsZgp@_IxvJ9!OSqdq&UTpMe3$QtB zEIBU9+=$8GXiOw}*CCEeX=1&Joa~HRN9iJOaaZIf>`?$TbDlI z#m7QN7$Oi_fm`^cnt9B*`Dq7thOB<5K1wqm3y=m8-r%5DIu1Q((1h^9K{9=cC0Y%{ zqxJ&PtSVIAbh^65?cLq>ZV^bm`1Wvj@AaN}lAyT!e}L=X>ZZHJ_VMxIaZv|op%v-Y z1c8_yGW4$2O)K@}$-ASS!?y^)j!5%w+E1Re)ZR#Fj-mIDM7>i{cfK`76{j5QkrTV= zuYjSn`hrK5M5f^|4m$RWBhog@oenNyytKU@9k@6nT9)GoppiC$OtsEk>P-KGB_G6^IVb#)e6rNESvUM zlQDT|`Fw``WlbhLr#QS;YSw%=8)d*14a*x`V#+0OvaTK>)bc1C7k>mzRGA$a<{*7; z0j;Lu!i%J7+iXQl7+BsSlTTuVrBjN-KZccIf0EJfkDPc>WGJK1Tt(D<*W{i|2GrFF zaA#O<>TLx_X@!pOkUdbWW)-MOR#P9nim;8Gt(%QK*ujGx&25-wTicp%{Op+ZNnTH$ znYIrDQK|#eak5X1FwNL>&xTyscES%34d?5~TZAY6h^$$#^cjUiiM1m+LwWpsaMTlc zf6v8FEC)}#6>fX>svB1rk<;v^Bd8IHrVM(tP6z&k$)BE$xfdfZ*JvV^K@eobKKlpD zk#bn>lsmO(J+VF}Xo8ssP3;|+yvY=AMJ3OvQB0~goh#4v;1dy(Ey3&&m}K4=|9vot zKM^&J*m8RiTxeGtJxD7XG`l?KOZXoNe`1o73~U$z9l!yt(@gB*BwcFKMG|Xn*Lf!e zML@;NlYt&)(3t}33d3cS=wGQ%8r%Z+A&BS*OUfqf3k46 zSR1|H^U`4>C1e#lIBTp*g)Js{Lpd@P;rW$)Iv9Up%4G%mEI7NeoUp*``Mi8T;e(Bo zkP4w}Ru(4ynvCORdMSAvkrMeGl$=b%<;A_76^&f0cz#yQbX`R5L2MC=`-P~_$~M() ztTNSV^F&RKNOXQk=(22Oe}!R*AwWu;~rr~)gQw((eMe7*{wimx1xo1+VWoC zb4l7`n0sA~h~ENq7)&n=e~&DU4$sDV2wF|fcdSh*+PVOt!ZXVe1tx+iu2Ifq^+-^* zzR0cvFd+N3S!+He1_`}YZ=3A$S+1q-8x|9!iBc;p@M1ATO~H6VlIHR~GWCTjNuxA( zR6xP#mP1n<3Ne+V?Rx|=@za4yb|qVKp3pHBw997N&5Z|RS4gb1RN{7^t>w-ttd0pigL!MobgH0Zo5Y^jDB*t*l}^Ir4?Cm{kWoq zCvG4nkg};<)M9gO0%3UY{TPHZ>~dw^%_W@!+ev|)8l!1|B^Wf;qKqo}P)r3_wEn_{ zbn_WeQOdM~jeO~BfBb0mdAK#}*8cY@qPCCN3E)HmQnge=J|WLMl^57Ll&*9w2m}cW z;nfivFj6^!GatEMM*X6KfAsj;Bh-_;W&Zq?*SnF&LK5UhCX>v%lvx?a5T>Fs*~K%@ zPOP*YZGmpiMCKr9gD3zN{om1|fr|aG;2cCf6m`mJ2-n`>e>u&hWbNt4`Qom)-T8aI z;JmGz&p&4$Qd4lqVd6;#PnxPvd;BEG8O&`V;7b&mg7qcf5E*6BfinbUqS-O2z{fplC zg*t;5WD!qAfA!*9wYC2AImZu~K!Z3rGVhBCnxc!LaDSd|&D<(E9hpz**{m!mL~G53 z7EV@MnB*^*e5`rnvBhD?#dzS|`b8VzJq2GS{o6Us??gzs=rn69K@pCar#JyI z{U-lVr>fu>r)szd*7@L^PUr31Nm}+zPTBl`&2f6ve>u{J$2m^<=Sas9ebiZ!;W)Qv zdVFSkn$G?l31yl?fTTBS>+gR@OzM@pGI&+mb5Wp1(O7A#&yV%Q97Gec$YBucQL zC~KQV_SzIfbuIMgp;T<>&{jq(XnT^iKH5Ir`TjT3DL!g1c~0(L+8rpZ^bssHs?rUO z+_$rpe~NM~LA?>to1VUAKX7!3B5Od3_xJY|wW6sJLpJYl+T=N(c@SB|`FPZ(D=mk!eP224vWWdb1AFNlAV?dhcldJ8 z@!jYcnWIpfQp}D4Y@Pnm`K2|OK-7((^g}2^~AKQFBi44uD zEb?m`?vyz?RbshIV8Wpyrp?iwWbG`o7Rg4~lE{bzqqqJj0QB1TiDuJSOmJ6nfAoqm z>riyr7~&?Q9XPg;GuTQ|qHFuJSeuvuV4mQBKz8)R+|3v@5(m){yz?W(7uNDpPG}lW zW{7x@ic-m^6d&!(Sdpy7&)i{`HF{E<&L$~Oxrrl?D-$UBEt zlo!SmS3g09qN>=DggWRtO0w6%Ga_X91Y1qlOy zn-_Scz%LE7I?(*P1`3E8X{;3)bkX;G|aO0>RYyuYL#(sSOHKzirmJlO!ocTpz_Ie`5vxA$_%+ zj#g4ZV%YsP>L=J)0^V2TF!9yRWa1?o3TZzFZJ_{d)|D98u+432*#b}jOp$)+*={x( z&Ci{4dba4ht{ZNlSy_{n^sbAF@=99(Q}NW^r@xvQlT@mbCBb<)jNxh6H&&yCln1dk z&(ukvbaV6Tqjg9f>X}!le`+AXhT79hb1W?o^8Wj1(cddpDrhqoA@;~nY(-&XGxC!p zD~_1msNjqND}Vot!B(8wbV*TFJAN8qGCVJxXGs!0SxpHnLrfS1Qz#alpiyx!AJx#e=DIjpAi0MyykN( zbm~EpLvu7=<7y(208nk6^LJl={N!V^u^XF>(b#Hi+cs|O#%5#Nwrw`H-T3|e?Aaf7 z&z}7w?(3eJ=VNAip72Cv-xB)OC=_+VH~$NoN<3&a*DLAycyIWwXgf4zD%udt;$PTI zpUiT8{rh-!OWM%GXj=_tOSoA;x!@D|*gbK7t0lg4q6?|g7JBV+4;zv83(NRx5eNE$wXH-VkLhwCu?>AN(J z&Way?nBfyT70MC1B!8c%w$Tu0#%ljjQD}r{m2P5PFC0L26K|#c00kcLb(%TtW!$fb z!BW!B`Ae2j($y>r4PJIHg~Y{u%J7@cfG*Ou?RZQdMXRW~b*7OFF`o*ZQ38T8nm?MD zr6ZhsN{AcAaQSbsEF|6>Y7y9}`~qoi)6|HzHh?as%y-UvL5}?OP(-)Ag+jJa@wCFP`N%G6Bm%I?IC(9OWhP~W)Ydq08;$H{~{=3oKx@(wk9;f93 z6}p*9$$7`q#F&sk@C^iA=}1Ydce+Cm(PT)=Lh!^Q%*^`3^a7$1s{+e?>MpWYYh|CV zhcY$l&KuJtqwSgUo8pq)aiq{J>ta+!hw0mK8WynfXM`S0{db%<+@+24cG+=>cR>q2 zR<@t{zvovj4MCp20=^#F3U3v6W?O-3_}HsG#~i6>2pe@N;{!zj43n^xj8s39Cu~%p zp2N_yb{9BEn#N9IL$;Hb7*an~`KWy1$cC^o3IBm$vNR?hYJU>URNQNFSQXCdIb6u9 zGT{I#))J_rVQXoR)LrXLa7v;J8^u% zaICdxxKPqA1Nz)y<;+SDgj+FQ`xLlJMuS79OA5jw=w<@})WSk>o12{uo1yX%(_xO@ z(oeOcj0P1U1JRp)tlK|+Ff7cqTAy3W=+*%Bk8UUYBjGFvHSt88d?%&u^QOs?w?BEY zCGuJk=)c*(Ya|l5lSvbJI&C*#nN~e`pR*g)zd4t!-T2S}!9=ev^%futDd$ z#n*pHn==0o{cOl|aDi22P56`TX0hky?$!Tg!(>Q?7h$*dt1$s78hCkoSQi|<`g{hS z$G7|cSL#*gY5xDpz5jbV)_y>p{%zja-qO=vf0sN_%f-b7`h(v)3LzJ`Ye_T2`W8%i zMEsVUv(-C}nz*%xb`Ve5igOS{Vz))j)t4XpR+L!wi^@8cm+vlIuwy;f1c=~o{x8A0 zl(LD@Pn0;v&l1S!mU<58{~63P=lykK)40LxY-WIlu89_5=b=yJKyv8|8nFH|UAz9laZp{Qx&C$pFatL-EE; zg3Pg43Z^-o;{d3+>o7G12i}(%<&r)i?kt-)+10`jneI=LXM*s7p1}%r@*D~!i`}?|bpnbH2f5dGf*H=|ow*0xy(~g^QX}B9T?g=nQ zhZ^~6XUl=llZOb6@45Z1=0;`#8B4r>Oe)N$V~$Jo*F#nypTs9S(s3(QjU6_JgGy`+ zjl<=Hyo?rT1m9g|qUH^S4~De+#F#HfSQ3z)5kWX@G@NFkni;F4JF_uiVAi-6-1n>Aqn?{WT8WEP3+>ikRC{^6QQ0GrA8PI?aK6iD6<9A{BdFL{|ARoVC>y z{=!qJtr^T;7Z#||+vl?PbDF9Gss6;aWjeozGmSjJZ7;CZG0)^MV)6;=fvlIpJa^ZAQoT(j~G@dtw!GzP+~P z0j~FF`q8v6CpI<6R`iXJ&)43O5%1#|3GoS^HTz7V7?J4NZnc=K zn^|48bP=^cp;Y%}@oVqd{>1YA@qNP;Q&blCRq)NMc=41sk0b3Nv6sMguAY`9`c+pF zeGfdmdL?q@MT&+3F6K`=*pmMeC#P|@Eudd!yr4aw%-YmNdk|%Z{8<|<1ChFhEkJLN z3SqpnM306v?4p8VzX(~n*;o*dxr1)zeI+ksN)c*8q`4OBv)h^nQp*(MF`> zwG-!~^nFG$PyHcN(;;@+^vxFu%hU!nC^v2ltL1sKdy%ZbOzA4Nh}1t{Q-LR<@?Q6| zSSNZEO0hVlx7VR6SzhIyLyLA(!_ny(z=Rz}(V&Q8TKeuuYMPCoXYzw(~7>r>gi$Ja2+<=<1*YC&hIH!7|l z<@v;a&u?e2>%*V~<3#NMdzM(Y?+u=4F$qE^=ziSgg+*Bsb`R*m^jKCuS96K}YtQ&d z-(DyOY_H6*aTD>=^bgSuuCev>8-5_YL9uxZfp5tC4oxA$;3$pCSEKC8Tg(XTG$Vq< zISpDPeY>cQ0w<)~trc4U?dCoD=|nU6dweq)u==wR#dgAVk#O5G>-As4(+j&-1wCRnu1uJG^Y|{WJW6*?Vm3AXn!Y_xondk2 zdl)Uvf-)6A?U$9g^dd0K#Czj=GpJXK*{mbq<*13aAKRft8!BA`kGiw3yX{DN_}n%P zmVfFFwGa~m--1V!bf8@6qzMHg=Shun2F{8~=!1J~NO9FFpFYpd5^!nO=c0$5k)dOR zl|tXXePfnDb-BJ-eLzy~{;e<=5D~dfwq3z53gI z@QhL`)tEr{Alm7-w7$LbUT>HBd%~8Zs5I&M`EKSPkaKmwb+f&2=YrFej!3upjl& zz-tDd%@r-8peftmYSOz7ZFi2jcHq|hRGh26DffZYf-&B*3L}!Y^N6Y>3v^C3oK<^% zT$J?=Otd!MrNLqQl6c?nIetYM%JCW;kNk_I5K5j}8kcSz>~*1|j@4kh>H#uB3Z7(P z;v@otEB-R>HPN}-o2lrBI7$SP3=!QFJI&JL8Ycg--<-#y{;|6>i-j`s&DojB%T5$R z*pDBjN+CO?iX}{U{wmNexi@jornO)~Qz1kCQw@#Ft;P#oQ%y;;zLW{O zyYh{~ZdI)GmlQa0X`eVIn8&cpbucY$q!?@Yx92F2a@{$SUKfZjppA=}hS~ z*BhH8$xCv+7uoIdsXSM?;f0-?L+`Zm_OWjuWv8chRFQtS)x$Ty9kl8_rVhbq8_FMD&T36kqRcY# zI_;$P=qL{|ga}5YaZbBhEcAqawzIv88iI3_#(ZAq|C~5NN|h%S&a&$)#Cu%eB1BrW zFWcL!KI@czO!dyINI4hka^=f>VmKT1?LB`Up3-xRufaH{Ng6w1Xc1)Mb;-HqKdC^?_A_d7{nqjSG);_6bARz_^qxxp zIxU!T=o!DQ=Wt;6fj1j}kE1J}3}1i2M^pCJ@@B3`k^|ubn*ntlvavflbB#Cheoxwl zRm;W@kWhazD4gDMNH581>y8P5mEI&N*{RY*JrWGr?!~0|M{a)JjyG9T5iUiG|Bki$ zLzCkd4%T0%&smwi^fs<3NTJ?cMm->CcmBmy#3Ck|!;@3?c}SFjaT_T?&)jx1=)6Pr zPr|GN1~8Pz@28~j;W**s+7QX~_R&Q}(?N;*kb}}#`n*(tuYu z#$<-ayMElD8Tb3GNiXxq@JT{d1T(Uec7P(}dLF-15@Gvo; zm|pW4F^^@^EP93NZkPzCmC*qfdq*}mGIj~3sAkZA6Hz^oVRjJE=l-1|TyBH!r+u$i zYp>r{VEy$tUWXJMZX{bZQ+}#=s)bV{6k!Ynos>ZHvEw*J--JX_Ezi~GApsygG;r9~ zeHmAYv#2GkphgMI8w5!&g!H6&{)X>L`z;tcuD*P>cWpEYr6LoT)o z;EF{Pc?turacBw3S3mSjcH?TOR-3h(;e$t0rsh|l+AT-dS*cY_)GJ8%cg?Vl&5E_rw7azZ0b62H^*najNpwDGhqGzLRF+)B4|ERx^R5|U1o)9 zw_IrB-96~qNB{N`-BU@+Cqqr9kogjkqbSH4L}CGHv2)t2Jvp8f?V8mo{<;k3`?4R= zQ@rbiJ;G8*8%brVRRiCORM^GsKEwOw#MSzbbaBc}l2$Dn$Su6`=v^3@gx-fbkauZO zE)7{p?{(90Rq8H-_Xtb1ozirt^Sj%n4Fh>K{}{je;yI);Z)%gdG4^1MU>Z*rBy8gc zHl+d%c=!>iB-PgW*PRF5Z1sX@lTDZcmey&xlj@l*bfnPSgXszvtmk^y)&C{J(iQ?v`n}$C=Z;WM@*^e*N=)NjwKq^jr3OSI;;;r{&Q&K|Y@!%t!aK11F-5 z=34>>k@z4j8AdV)UHw%{^socvdVRT-D2a(-qIh26TPzA<;mZ_O0;&p*NL&d=o+V+} zL{nN{f?5niHV*90LoDn#nsdG#-Sx3v2(4TSob5D*8zn-w?lf0ciu|fdtcR56g%Uw48*%3E!itbo2%KV@2HF>2(k?x+p`LzIRlk9$dy}Xl zH%Mw{d!iw~0Lj0)+jz{J;~o9@#rO$*F>Rn?%kNWFH*q{t6FE3fJhJM>_=l<f~X8 z&jtp>rc#XWxc=$Ou3F^Xf2;)p?_qBK?IcmfN08?P3Nes*-pbDO+4d)&Ea3ggfW|Q3 z_RqQBZOsU>|5!BdKi^W!4OdrV$dU{K=PN*P3a_{*4S$16PO`pLfvpzJ2cG}9eHLH5 zSHf`X2hCzbhNF<=1fqNF9vn(Bha>(~W#u||pFcZLu}H#% zS9@7yqjC3GpvqkDddUo!z4{%i0X)pBRvw`3%MZV5xWI+y%AB7euA8;Ie4nr9x}w$= z^!ktP?fc@jjajNq5VKy$?lnL0m8fvFt9OG8&_t7r!LBdfLQ9OVH8dg6k<*dsrV<1~ z#Z0k!@|qi-QEHIF|5Y8SqDM~+$&Ij1_k}-1sVp=X%cTs3i!A|ec}qs`I;}r!MlbNI zvYQ_-R2^!p97>u#6Ki%iFfFVp^6o4CI}q6;za>_D|L^xa5ZPSchyeYd1Im9Ik!Eul zgk({dT4?ZhEk3~$MIdlRd0U?1GV#5fAWTx<_q9?gObL}gHp2ZLWfMZZ$CBuW zGExEkiho&66n+AnE;dVaCnHtIM6xk%87<2d^g4zq1~Z&t`P$@gXjiWiww$e{@0=PI zSM&SiIO{h`glb#Y6_{bX(MfAeSEkf}NnA21$v93orPOSOm0kimu|c8C0*pzE{nC_#0)H8) zQfZAbWD|>u!kg@Fa83gWTz%3_6mx!}vC+1WR&j4wtX&A(4wJe#(xE{tQ`8?y?NA+v zEI{6M22^y$)zOMI))cI4;MK4}{UI!E-2L^LEgYn;X@gewFBC`>F0l@CQeJdvXfie& z9jRqQf6W5a#=-FSy)U+YI%8A*Zb~sfPv)$`bZ;P5b^Vcj_KXs=J0>x27JZ8GtX^MA zJxF@eI6~_bjHNfcQev(!V}_pk>1=_e=?4=e;3%!f+&+lY=Ao#N6T3rgwJ$94@w!dBdj=Zh~nXniDuB>ECMNq;ndSj)u z&qyAHfO_PWxmZlO<7YN(kB0!!U&l9`)uq!N7z{|BKjQ3C8w zYt#2gO>Vn0xU9373`(Wm@mED9$q0D^nYW}beQcl-3S=8{>j3N6xZtc+O3s%9*(E77 zk}Y=huxA9#)n@N;O}1)X`JNJm^iNh~gpkFw0ew(TN+qmIkC$;9={SyRztg>H*|x2a z)FpH_ZZpzWWY<)1!GJg0`CR;%aS3?2rEi&&yAszAwGmNdSmVsc2B~CzXYb< zWFIPx+sa6(@zMrU?u=>j4q^behJIr*`e z!=i$GT4WDH2E2V*Y9pif;-OYrYKmn7lcTC!L|FuzV(SLnTlS2O5PLH^Tib17jeKsfxGb8}2 z#Uz0Y24Cv8ELi(S)n0qWlDJ+NTya+0GUxa^q55u-)_?XoNi#C96P*~Qm%}2|As-*4L08=lN&mFYTK-4EOJeD{P zooBSz#S0D!r_#9H$AWkDrPk*5l*@(`GSQI6VJPf!K77FQb>{zqAktzCAffN)Xfk0 zm8oL0@NC>R%&Lx|Lv524a%q(@`{J65UTJh15@jBU_KGZuID;bRNj#5>VKQXK#=0}? z+2(NEX!@s4IVC>tWH)DFeJTJo1Akns?UiDXY*{*&Eyc59Of!P_%{ItcI8?a-g+WuE zxKUv7{?f#)&7@O(tKM&7X%EXc6k3eZLO^NKi-BJx_NT|HKZ=Y0te2{WAZ|oQEk=6v zVT28?G$Z7qD^(vqTu$(G(~O~-U9|qUbR%&JPG&G>dKuGQekx-m7)Sv9oO?S+s-WsO z8e(Kl!3_;6HLw!jOX)bZT}y@A1OSY>rr>?ex-ReDgM*`8C$SXe0I@Xg2!Y0N<4Fi z1f+Ap@e*zhCiI}Y1*1_Me1DqYND3R5me$Y1db%Fo|K{(mpSlJhet*}5xnmtgC3E76 zmqZ+3;{P5C4z&1T^8A#G6(j#QBQ?Fts*~(co-&f`+a*R6C`-ktGt)WZ+)Cbv!K)d@ zZ~76SGXG3`YoveOluaM6ok{bg)!1$&lB=chZ{N2I%Gd&qM~j;2V&49$IEwn0@SK%7 z=jmsNe;{(|agjSvB`YKmMD|iW$q=y@|Ik>J@WM}xrb9(B=xd9>g&+bU4nknXv%L_< zv%TZ5O!1s<&1WmJ@Gr2nnM}@CIl6Bma7!;~{-+Wo)%{so&uFCarmg|h;bqhXF3H9&W;T+D$jMKK zpF6XmZn6Y!X6aV93k4`4RN3OSR)g~Tgv{>LFyi_UWO9^Xeg;G5(b4Kx#h|TbU()Y|UPUEffY}&dgOvgt6izhxG zv!>QM3(A-9JD{(2jH_Xi_XRJZ#;Fvsc$~qOi4&)dO5mr$EN1<3Y;Mkn*zqI0f0<*H zhmf)%Sg{)+^Wyi3PWhcS@`fon9m49)eVA38Cx_+H*7N3W4)%5eI&8kN zRs&F`cd8z0P^4Ko-_z2mN1|Q!P;HY)1baNee47zJP*wd$pY-Hj7}(V&P>Qo7NBf+I z`-^oQAB5u?AS=s;k*RcdjPXjY*;$NNidW4MCXQ)$cskxiYa(YMGPVDn3@wQR{8U}{Q9Ai5r<@*ng9gW=5N-mF>dR=R)SnJS8jJpw z)k8kyW;37xFB99ksltPC(8=x;(V_30Bf+1K?AYn#G3Ruj*T%R7;x4?U?l*OgMX3q3 zprX5!hx|6Kedyhngq2fipTZf;lm9+nN!xQv7$q(gsFZ?(332&im>GD_Mhx1S%Aee8 zgw+97f5UVPI5l`zr;l*kUl4N9+31NjX;TgHTM8X)HHOVb_xe3E2sh=cvt=^1ZE-?K zQ9~xW6szdD4~Hfi+6YdCCo{S6skJ>{BOJ3sAK^T@(BFa$;-o9y*XI2fZs)*(u6bgP zp=|E-JgTW5`-1d=OQO=RxtRCq{Ca`4Gja^d9cd*Af*Rjmc=YibjTDmGhwAB_w$H$5 zbPzv-=)BYto1mgUdS$~9WDjXh%+M9a`XW@iRbbHJZK#j99?!*sD$+)*d zp|2SY&eApW=Ur+%TY~BopM-sYF6#A)q8~zNHs3SXimGtQBB<^$4=&M}D^LZFlJ&ffsWv~oSguwWh%F5?Z#ZM1lsxJ)!j<^ljMlGDqk7Rfd+B2Bk_o zI#KX!b4GMeNLPWU2;upvSor%>bcA=hnR?htf&@=)bd0u`9ZA~Xg5Mnz%(Jn#pwYBb z2@1qL7k|n#LSWfbW<**=KAy6o%IzL;P7dkMabEV7b8cKj7ab8Wp7__VSaMeWuC{jh zvCLvhw&Z-rdU2k5w?m#Txdm@&(Sm=CBYO!*1=`&?ERz7a!)n^g=lXfH%&-~Uls6}y zt`PqSD&Wz&>{iqTyNn5y3MnJIWi_1!SxjvTlUpyfCnxH;_4!{_atf7?qeL5v^p3w) zpH~bxkx%d6d)x1t)~60ZTh{Eo+aQqgtKj-r3yVkKIu1*qal#q7GyS;@71o|e%?ZU0 z89+ZZoDa;_(siZV4Ol_l$VU|v^9(unl>R3WsqlPgrZtZ{nnXBP*3lYIrGaa$XBlDBsBIh zLvKd1-by^YmW^cjNB`kTsYcFG#F*RCGb$xHX#neD`Ran>&*4yP@o>n=6v?gng+39@ zLtbE-yEEWa^7F#D+j67zZ!lyR`Qb&;j@CeKf6h+@ zYgUnj8y(8ZjiLv?$Bs<&JjYE2AgbplPVLA%+`;?|dtCQho;M=8+@UQu`HyQf$K3wC zWd?Arh!Lv6F3&_oLaN>Ae%?V_0F&3-^te&!ja5i^;6&J}gti{jCES`8IY0WV3jgA3 z+hJRcjx((^DAjfQ!<%jW(lCaJ=Y-|=|95^e^S_A?$R|c*uj-bg#9cv$-oLeXdWdTI zNeSbe`bs8;nnKrb<6sDtfj`pSBBV!wSCQ&7lMVg3_+lLn%z%RAGc_?ZImBS>#FB}2 zt^5E46C9M8sAIdkEtm;Y`RG-a-d z)4AgTJh2dsef54+u2kB52*I`$P+p~V++lQ&C;zHoxHV+{ykwIV(g|^3dC>r zG#!_?L`H8P_&48$b}11xagj|4m;~9OAZ1YF0@MO#z+(MxErROVkKb3rTzjKEqe z6Y{Uv4zVlUrj}5_2~oBea$dV2#sF?IvD=BTu#2+D50lp-;9sqQk%qT^yIucbW3g$% zl5uQ{)BAl^(X_S%(&XKk<+y|12adcmTvMbh$^*5B<6B;W!%F?F1XPyr4=`Ssk2STX5=1ihL)_kOUtf$4vVf5PJ9 zvhwR+yXVy0Ncwfm}u7t2+&P z5DO5@zow)wgS=4bveTk25>uU9-ediKQ_efwv+-5nJopD8!oC&aKzsgl3`X`ndCb$I zg2n&{MP6|&V*6tM7It|iP7wD!=?U+#{RpZR1pT-a%vg9U7|BCD08JmSNZ^BNIq(S# zt^~i_t^`{p8G4o>6dAM4sK3iAW9iB^6{Ga)quK{E{KUQM?Oh_`Q$_(xpAJdPGodx0 zs`Bcntn+7uV?Gx}O^kl?WC76=D3pO5L%vWs_sR9Li~FU4zexl%Kys znESn*z67%Nafojl2GWxW7_>BQaDI1Y?VdsTp}vnW&d~8<)Iu_ML|s|%bMJi2GnV=i zd7RcJV&?Y}axif2N?uhLZzZ%l!4IV&Ewz&0$FV{4G5P$kKVIuug9qwnYCkIZt4o% zRQAP(%QLUX0~{HezrQQWDp0CxVdMT7ad?=f)`_fB##~{0%o|cFz%UZrIm$Yb{2`(n;C$|c`*-W^PN>ZqYtyuU(vpA4#Neym>q@)~u7vnlgZpe|tt2Wsh@XAT=Dq;yzvB)u zmDf~~`J5H(#s{vnZd6UtW5n6*(;6xfsZ=zlgnY zG}6U$)j@=F47CJ?VKOMRwL1c9oDt48#@G6 zkn)t0O)Zzk&=3Q-mSC8_mUVyjtOU%GEjQiZMD2Lz%MI^m?P`unqv-%|PSBNA+1XI7 z)DQFrtzIC$o0yH2%~L;JFpDjadLAeK^{0d02UjmhApKu&3#UPOHu-natagKDN)c2X zT`Uqai=w>F>MF|CDFYUWs5Ok)+XikFrb7B;vlH=uqIFeNWf5TYWDwBUMuezDb_068 z5Jc>DpJcT&_2N(B!ht}&DA6%?(bqs}h*m+YT^9OIG5CMoIvA5}G=TyaJ3k~1sG!Ze zK@RrqH-a_2H($Rs_ik3uXg_%QWfyPRC+xwQb^Z315?}PBh+~5! zjvx?7z+;S>zp-(vigI-pDwszcu6guCj?O_`aL5tI+u^&5Iwze}z zpjJ!}xSZ^q$8TT=pgci&Ll{gD#Hb@OdcF(0LQQ_ZkO)sFx(-}H-pc;T>M6SHEq&`g zgL3c-ZqKkHObW#V<#2aIr>yNLdBhIQLJYgLAETh3(fiEBgwyfF^P-vpr!iWXFV4%9 zzZjvTs|(4@K>{8cMd(ujHkLwB;$F&l7$RL7u|;$ImH|LM2()a?Fm0)qI04y+)MO8to-A4uvh8z_+#BStVUteVQwlB3#gXg9f9b-l^!+X_wf zVt$dSu0EA^P!h7wuT>YX@n7)R5r@`X90@hSy8#F~E~no!g(}HcfL}=3l0}`D{;s!7 z_YT9tn!pP|8RYIDbB1O-Wg?lZ+t^O--;O8c#?0m$U@=EY3HHNE5 za3uUl6YdkUFRbICpsmyT_nB)Bh$*Dk-u1N&$0N|l#|Oym>m#}h!_C`?X>jSJO&4}w zg$+12(T(zDi;%8P{?|tctxpDai5S?2nTkVnH54^iSL2;mgy=jt z37SwTTH3RDT?-a}P=Fj5raKe*lqDX|tz8l7bzYQq^xXa`H-L)L^`3ZyHc z|J>8yDfGxF?tUp8RWND2|4<;Q7jWqmbOBJ~3uHtkmnBddIw;uciKVD=sCKvMPP)?K z_LZv=?zk-ZMtMPlMAOIV7oel9{PW@#=Y23dAiWoR+79ZtZkIG;-Sznx-4x{`t zV{Iv-8;f=vOnTSipmeC+2_yAvOW*4ZwscZXRpqR8ccuRGMKxGP;0WRRw&s38(*dY~ zBjBu&>qqBSdhkxY)l8HpgP5&MAH=TIFT?T5k^3+%okO+s>Xs(;z>~O3g$+lr96ae~ zsOs5bH{LDFm?W{gGZ-2UBE2H4lcL;+73YDMK}s)vO%xrR^*f3AErWev`=i3P%nP_s z{89$$9W!r%fZ)t+XRD?>Htn+pJ0SLj{AFQ8inJKr$!V^A`vYz_#>ksnM)FMqNnP(# z)3?;5u)6XW?mx{KW*UST)GmMP?_e-W5`EaWY-RIP=fWo0lPv*EHCO$Tfw3^uTq;+EEj8%{_d*1HrkxhOTdhhzQ&`*I^*q@0%ldwj7dF(i&qe*kM}1O zM3M^As$j(+n6qq_IKF*}gBvn5V&e|cK)5_McG*$zZ#gjR)+s?wmEhGDTi2377sC9p(_=l8uUc^muFn+PV_a0-yOn*Q zJOL@yOGn=23_z&_&QYh+=sBwDqS?gsP%f0J_`s$rxir)mo;)HDCR!>7-{jdT@^+HhjKsw9*lXiUI>&*LD&G!98yokh@w^XL% zd}BYYicXOI_#+wkKqrnR2Y7$4Vj+=OA)plKgnK0`!<_%jhEm8xN+5YM zY&I|!OIA7HzY+N+Ba@i=Bbqt!MOmgAC%&fWkh7$frTbSzx@?;vx7mLPIc;|p66c>t|q#_iVCG1vN}~AzYqD+?ZMwt zlJ{~P)q$|?Ja4G~)^(;ILK#kvG>G(TT0Ov zdZ6NxLL)vCH@ck&hZ?!;>zlcFe%=gRA=Bhb`EZx^D*2pV}&A9iey} z5MW~>6w){YhD_~=&HclZLE*TNI{6@StRLz5?g~UUQ&WWAL;Pm$qCv@#6(y`@dhjcO zQC+^fg}Px8>WraC9U*i?hr%%bI!;Y7*lZ-J!^&iOU%+)STjZ34x*FTVTXvKir#$L; zvk@x!BDBI(^EVVrBcm(5rytF%;u+^}H$ZH8mqLE=X)Z4~P1qjkr#9bkP-Y@bj(&qk z0c^E2U3JmaV(fWn&m>Z4d&1*ZhzqM+$0C0R$6}eSK1zOFs@`8Riw>Oy^$QN-#mwnX zrtR;h>~dMqri}khbxHub{$m1z4M^Oys?D}cZM6rInoo5Dp_Z0a5RC!e3K$UA;sCcC zbN@C2Jh5mh$z@MS0j8W@ohbviD|;5m_uZz^i>9si)n>sjB{ok-`|~IuppXx0#upz$ z*#@ElCq9TIkqNGPl7{+&&8jOgzAJ_)#B+cMhPdeot!`m&9SPc)ZRM{7h%llDR}B0V zZqbMk-y`4ln_x&%fy>uo)J?4=2x#u$X$5)je)X0W<(hgqgQMHh{DVq4{(>|{tO{`P z3~2Fw7<6aGnjlyz!P#>7nG$_0|HDMSq(rtLvI?A zSFZ~o`YOPx%%vBsuW=O^6adxM&i(_1*L_v9Y*%*=7E+_({??D_8W7&I`H4uEwYM|) zG@%RTD|?YAS0`--IEVm->K`Xh%_1M5r)IB@JXkkSX#md~y{FPP5dv6_*7ywMw%$Bn zIKv-4fhHrUJYzGV!6kMFQq=j^gFD5BqFCY_7{+g1lLhssVoA`uIIm!Ol(JKbQpI^g zzco^2*mik7Dg_svFnW;-;;tB|viCnI+S^cKsVO{JO04Kei?G%f`9vGt-Fnb zM{jehi>szOuP0k10Z&(tlts{bF{q=}V{Y5O9`vsNZV5WtzTlE9aTSmN^Dq3F#?K>} zLrg9PK)J&6M8A1;{o>5Ej9zW|xCsh)6@IwPZyGJ*oIP+NqOP>Z`U4Y`Ppy)dzyKhsbb!Fw3@6;F(C|t}! z5_J3^k^cDn2TtXL+NMvIs@R-|WI`x8hI9N`gu$%di%8N-;IC1sg}~t)$>xF|6oh;> zsK87VQ+2r&-S7MSy3Au#`X4cXa_bU(RBQZz=wsX1ZU#_ZJ}&b2sDROF8g;uqN>AQ8 z)u}J6VeZMR6jG)l@TD`{pf%r@$rKI{t-gNrxlihYd_76qZjW?zoL?uJdcJC0yk(;>3SH zH42H}$DG_zM6k2FrO8vOsPPJ=b_hu?!I zhJ@j;{y)3Z?Eh_1|9|%B&Eol-35DRGU2=?>>zE8*ezTd)_(2%3J~|c{ZQ~7Y0U$+Nh*rNuaPjVqM!+!Hq8~Tvt>4$;0Q+lyeAW+ zht#6~feD!OssDNKFIuvN%F$v;i87P}&m>#+So5ky9}(D4iW^DTw`&+EB#*k*2D}g! zfZSD)m?3vN58S*|nPRbo=)}o&>qA+;Yp$^HNcUfj7Peg$!^@7|VO|C~(irwyVZrui z2kRLd9*S;^&5PHR?-Fw>#h|bDTF}e9UB`LUky5x^DcfTvL?EpdIE5wL&osANY%-j< zCM}VZTS^TCJR%Waw_>Q`h^n-1Uuho-VgzpId E0}`#89smFU delta 32684 zcmV)JK)b(yoC1KD0+5UmFf}$ZH8nHefb5`Sgud^IrnAHaNe0xHWcDqx}zjCx^oq^W`|Ueq}-DM0(^ z60F04Y=`GB?NAiQ8>4^c>9^@PswDM=u_}v$Pef}!H`ZlQ5>KRXj>bP2OUb$?%(Gd>jvlS@WL`*RS~3h(iUMgJuGn zShoScSzD|SbGy>K)?9+cc85gaqIWHJc0G*7Tjm%o^d$Iq^zwiEqoxs>_c7MKHoQF> zrC&e8ZojG!RL0>Fq^Y`Kl?4kDsKcRxN(a}BlPwrM!QPWeJ6BLR=oVV^{Q z8ty%rRzeF8&iB5nhljYQAJ21Q%m1nVvQ#f2SB5Qm*N@}Fl#k;_s0Z2@W~#oa9;FES z@ThKK?@4uOOo@N1!1Gb>8+(|(IK++Jz}{D`4IU}`KY<)3(W6HYZLmjh!i?5v_5>Io z9VX+Wy%{g4rJSyw7}d7cUMTp_(M*)!cxzlz`vfjDTL+Q@Vf+z?5nwg?T|yE>C2`)? zPe)Nc?evS2q@zPjmd~^BJvYGI`>OZLRd?#|X8G>pCV79+U#9lIZVrxFQTe8861T<0 zxG*;Usm{S_z5fw1!W)YxlR}h7O;0KOyv_$pV;6-qOE;h_pmga<$&H05wXx7P zyZh04{Plm2lk=15+0Q4b;@CtoLOA?%cKUjHelbn`mB-d>I(hSU+S<&fXVb~W>3Qce zqYLPKt_!be3CdN#uv3bBaAUycxB3Qj=?E+wVaOD=1#Y9AcJ0D<4h1*AB2u{BAp-Fa z&R?>MgzF5cthp2%?44HlEEU!K;TfAN5}{QkyBdEr;ZV?8`UhQubP%xC>_#DUs8Qbm zQ3>aSF3Ca!j7C)$Iu{Zd+G%bCXL~H%f-|*caPJ_Y(gNix0$2Df+&wGUcxdBEVQPeW zw{e${JI9vPvpK;pSD-2w%#DJImYw{%Y%plEBxK;A&B+_?5k48r!8XO-p1z9|cswR~ zn#F&Ay^kv6JQ3P?DkV|iTr+Rfq^M~Gp-s|Gx(G5oZ)QS_>rPIB-jI8Ki`W*zf6CyN z2tKS{n>gbX#cLZxNXvg8NT`%;e-{MCZoxslPLCspYSrvrSRv8w=I=Y#rq3f+V+PvQ z9&#LNoHyk3Z(dik0Z1AE0000000000005Kx1&10nH#0Nbrd*^hY=LwDD`R;^pDOU>^G zlQ8Oqqs!px55LV%quFR|ZEXCZAwSvQjkU)5ADSD@W@B?}ZF7H<$~D)YK3)5RYW(4M z_?gBNFQTE`UHkrENx6r@)L#OM7>axXzCXVXHU)x{;1yxFFS)^Q7ek*kmd86L7AIno!6`wpQ@}Cp$D*JLg9Sk~=|9_@_JXuov zAK?YRxR?fmzI=(Mqqs_4t3Q4CpoSiyiubiJR`>V(H70+oG4ZcvQda2#<_WbgW-WE5 zifgsaVgb39ni|X7q5XMx!IK7yTCe!MG`idv0GW2?3 zY@Uu0HSsiy^H&98YEdx9v+R`DRK)l3B4VhE$fF7?X}`{xIl;16{&y{fdPV+UAw>uUv-nv-AD-!JGE};hWm9|IeZSYinCuS^dvC zJ$|MCzsJw2+PU^dS3bD%!a+D;;kd`IVl@fX*$=)yR$kK8c?r@oiP1Oo!RwjVEKd8Qdb8H3 zHR|PB;fI9PAX?;8dxi^vczziUps`eEbnb~|kwzh$$NrFt)xzisjTBZ_)w_|d-=`-t z=YFB8x=9zU){ao0k$=}!qwr37gF9~)8*&&ce{ZC`abSi}f#w3!CDGgit#2)Atwc1e zqKdc8V!6x1*{D6Qt*QV2|Neh!jilddW3#&6DDcR(`+Xi4*f6zTYbaWhF%jX5U_fM0 zl~y+!jf$dwH)Uq4>ewHWT*h_${e7ZpE!OpP70nM< zErB;sO)P zT4s}X38smE@$93{RXCaWBW$d&5i=a{f5hR4cTkOd8gpOfa@8Amfje%g`AGJ$+Z*W}!LU^pGpQ6+vcCYH5IdpWAc#1|9Z zB0Lt3d_q(Nfq>E&V@6{%hV|z(2B-ZyF9LFj`P({wwtxK@m)IL(D%97VHz5!;IMACg zDiFq+fzH47ZUdVCc;JmjRGxYl3??}Zk87Q(whz@ zYR5tXG4)(tASEF7spivB8g$W(R>9NPm_&L%t|Z54A1d$41O5pB)2vTfk#eJk_TcQaDbbK*E5=Uc3e?~cvDmhD3mfLsHd0gEoWo0Rc8-*oA?O`q2tNa?R1Fe2 zKf@izxq9(J6$!f(yTXzHU6KGhk1$nf1a1T<9UDj(X%J5Px*PqgdhHL!e$)jSBIGip z(UyvkqyGoXWAvV025S0eJcr=w8EJ?_R7lo;R-(GJ?Q#i~unIBLNk$S?$P*~7^`UiBL zu2{C?_1ocfrPU*WSH=p;f%b#ZEp>%;rcV*^CzrGaKZnzzDFZyCJmUNKQihR=!zXt7 zq0fDHjIx8L++b|c*wWY{VE_*atoL@+j+||oW~}jMj}SObf>!a?8w7n1c$YeYluwL? z`5f@tT?&ZZ`22gO84;4*QhEI=BD?wBE5 zPA5|U`ck0alA5va&z(A=3Sk~(N>xDtfKE$iykyO>ZR^iBGGr+OW<8g7qL2K|c-#hr zLBIC}cHq-i4E$Bl?`WF%g;s4VMNye(g0<|7nMAFAz>-a7_gy|Q-Ju^{`E%cDsykPl z2ZZ$qMpNHVdTm`0^b|LLq_~!hqhSu+nibP<=txp~p5E32|4wLOXXw4}(892MMW-wr zGZ0f_Pd5ysmz-3?*HI;9EAf1Vxui9VSXxz=Q8-i^7;=;3KKeD}0wHJu1ksDw?Sn^b z-FYomc?Mu-pRkmx{0(r0|hl78Qt zcq9b{gEB-P{}!!((joLC!e6W;QG+nNnT}%yw5Mxj5W7CtYDcR@P>E(qk`vdrKvh+!@WOU>;P&I!%f;vAkyxG9{; z0@SEf%xZatA?^-D2-Tbs1L$-_f|dv#8n86|C%|UCaUWxUC@^PW6ge6%^uaw^jni}t z;M%~PD?HnS!&81-(0bRyzWgf>Ua69H(9bbilkT{^^T^T_zfaKtgGk)g;htzEvX1; zYe}uR#$8pUAf1A7n7maUtpHlptKczs;SwwwkvNxs^ph5E`+Y0CT{66a!DnfEfcslQ z2Qi&$VxGba7N*f??|M3tfp2cvU2M3!*mzo9ThHnfUx{voP8o4!jQSGuHD{c9n^|4_ z?F-c;P8GW$5|i*VU1Cb!&pOj+fQ6*Jpu-;N<-{!hifU^Z;M#h2Thn##oLbHp2>2V! z6lipR;5L+y-4lWYMjVd#>d3W>%GPQ#Ok}+_<5rAbWIc^NprLB!ov%E)fNRz+?bH>4 zq$rM)HRwfxS?HMjy#02cHL&r-8;*Iq)}J}uH(7-!oj00uM{ZB!Xmje8j0AVVu{w%p zz3WzMH;7q&_xpSYkC6=gD+0)wHyl`rTq8n%Eeo1KR$TSBB0)fTe92c8X_4idnK`lR zPjhIh-kc?))!bSa%TtTJOTRM+JvqPY%=RdCM8qrzr!lm`J=R6{eQ3u>WcY=!b952V zd8C1`jbIP6^kWvpiJ=YR8w~?Ib+?G}Zaos?F9ue$a2VGAaP(cC)F94armnqP$bCeA zfBef!K7dtC)sk_r@Nda16<~D(oBn|kPedfFv&CdUY$DYcjmY~mk}rwrhdesNK zPfw3{bF~(&4gGtU7NX}1x6I=bF~ehjGC;i27#rQ(%;Le6GL7-VH6hS$a(sYwnS<)! z9fM2rd07Mpg5&{vIK=#ZZL2I=w752RBg6~Fzk%CVnR>x_LHdmSNd-b0(eN0Yo<`ti znZfQ+`;CkO+3rg6&zv`%cs9HUaUAKR+z>K?YsfT2X&dXi z88ZkpU`q3V0UCGLA#NxQzrObTIrw$H4%4bifLiW7K-Wd~R&7a$^)Dd4cRn$5SlM-p za~lrWuT>ix8Tz=3yEUkBSj@D4SfiBMl%tSl>`!{Na)s~|n zU+LF$+y~?HgcfGt^B>qa5ucy%Sct!vMgYTOv!c*soXEWj3s4vo4&xJlqdZ2#K&?5K zpugP1XYwUrSb;xP%&T?*&%5Hrf8bm$YXXpWNxqd8#ByL`u*48v~) zU6wWopEGvnYcZ$Ju6qHdZ%n^Yfgrgd%a&AlZ4xt@c^F+Ze;uud3RPDlPFlI)H6YU84!z?gClVYbWD5G zPlrD0rulqRpX34z`ZpRIa_uaA<|BiHaPm5&rI+q2azic$2SDi@6q?5Rx)2vF^y(`> zg_}Xe!n`4_L)`ye@dB+i-8;_>KJ+m4G!jVyuivoUfB^2sQ{KMKdA@qd2`kZoG3Uhs zCC*Fvlg=LRr@>Z#Ang_=Dn{M+#Mo>M4ASc6;H<9b-+4TI-M#j_!He06|k< zEmIM`JXl+A;$JzwThBF-pk*VuAVnmq>L-RV!p}s-V!NvO!wE)m`wr7Nh)^LqeEQru zmJc&lv__+_)oQ=m-hKTTeroNR%(?BwLT@cqG{OVQJY~Xv+AQ07Y60YFwb`gP*9r%r zRojTF3S}Aw9TAgs<6WV7LTMB4#+UUv<3g#1Sf}&z_Q4PB-E$W)m?NA$!HnnOu^NL1kh7FJ){ z{!81y{-!o&5BdMBZ$8b~|IzM!_5c2z?Efy=*0&=khomC0d)6M|n!_v_+b4i6u3ceU zS1c6z{%z+PU8tf)?ZhoqiPr(Ez!f;Cl3PBUeQQ@SC- z8!uDe=)CF^Ig^Vd`cRD1SUVWySfdxr@2>{GnV=yEb`0`DdN~wZ`V=y#4Rf zujl`FvHuRovnaT_CRw^y25H3J8angI8q^RM2EUVxm0rPoW}_|Xff*$%MXYs> z6&_~=w_b#CDsoL+QRC?atplfyIKtWhwJyvF#xbpkHtxgVKpZ>Ie zecV=iC+g_<@W;K~_O4plKB4E8iu!5q^!vkir;17(Zy%igtPWqR?Sr4y4|@l@l|uV( zN6>Jq!(+Ag_Go{vy<1Ux2Rr-kcJ~h6s8>`EH-!6p1g6yJ>7hc0x~aW3)q7od+dhVu z!|C>`z5Tt@pDXJ1-su6_d`*pRtE26Ige6^(cwv(y5B7v93Jc)ygsI0 z+Hczjr#0%8x>D^Q=|P=*zrDZDU2VUk(I4~pb`FnzKHhut{i*u?aDTT=FJH9_`+M83 z_S@1GO=@RDBr|L6UPF8Vig{&|T0Hyi8gjg2|}zwwp-|DE{%LJEz?-(Hn}tv}C_f@QS+3ZY-r2~pf>oHXwQ(;!+!Y==|L#3Ra& zP^3Fd#_-tYUWyUPi=l@b6&~9TZsm4}At3G}X)G9C)#7Wm>&29Cb=Q+Jb-1vAr}4xY zVZJkGOin%HIfAfD^HmCkZBfkWD-LKpen_r>h!~#2r$v0NGH%%FO`FJn1CcQN`;PwE zZSS{F+ql;R@djz;yNV!tx9Qw5!7C@!3JaK-h(SDDHrcHw)o)_mYe$eXRI6IKZBp$J z-T~2wn5u|56XLuC}hw-HKtnsXY zwJK`gH0cpmP}_=~bk+7a_#!>TbWSI|>vTukBe7!Cs=Xi4=)Z&b3 z8U-{U`qXOSdb!52rApknT7jk%e?s3(oF*W&r7))QgL6qd_RK!4s*{s_+J=d;HgsAm zXG|0rA|-$j>%gb50FMVcf{`6Ey)suF)c=Qq(zERJe2kBk!zcT#*55W8&uL8blI>_F ztQst_ZUp+=Qn3yJC6iGiF&l)V4o%AKivRhseZ2Sj=gv{PeO$PIaK))DtCO7~I{|f* z*&-upXN3_^JTZZGR+JxZ3isN{&5&8p%4@kVtRfuJSHBp97j%g?|3t4>AZ=L9z=Z-3Vz<3zlF z4f@3RK=UJtr`KNO^}U$_Wj3VTI2k!nggkNMH$>CA|7b(rp=~v;v5mI2Hx%D7*xcQD zBeuLIyvvMKE)Q(<|qmQ^^aS~uB`Z~M7wBNWg#w~`*Mw%tPDFXo4^Y^ulUQdn%r9o!w(T5X|2MJg3&Fm zpc=KtX7b!2fvq17N3;bv-i>pB7ODv;;vspvpSjE+iE_Fhr z;cwtVSjn$l)!_LHbBPXHk95mSX0(roRnp2eyaUrgayd3pqT&uBD$fMGRqBOm1Xb8MRK4mrgy*ih)DLI#Qx-C2WD-V38LSN z*`P{#{4JF$B_TXMoD|r(OItZxlXAd-0S7iUH6QxC>EYNH)+kPY!7D*Dn_ql7I6AiM zFceHdcTI_&!$n?v7{PLhxD0OrJ9Keih+{UA#AtOS1%w?SPNS9pXHrKSjA#|a=M7z} zXMWZ~&1Ep^v*{yD#yR8GDy$>U+^-mUdm5(O(qh1nklql^$& zHDM6yWl3^n3{uuKd2f;v5Fq6w$VSYZ_|H!hvfo*y6?mJBF)G=Ohv|6V-M<@K<9L%MqPfqrw^cU2aq=wvo$~+s4z)ZiV9fia_pmt}> zS@k}#4ydJ!wn#ma>1k z^6as3%}*A8Q1*|V8abpHyO0KN$}S z%L&H9YcQ07_>9=rltXC%Ytmla;SFFp1jj=Y5%)@5R4|mH2^f!z=rxej;p2gn{p7qv z=x2?f3^suj{lc7h`0}(k5Q17&(^0@ADu%mNA0dc;K}lNhy~k(#g0n-Mnbwy(;bw3x z`9tc~a?@HLGf5y$5GRKvEtf z=a~Rx6&p57tNOYv^w8C~feq_p;g3h!DIx5^LLFib?FR81Af#egZTt%6q?O7-FCHr% zL2ea0_kS2Pi{MIV$jG{I%C5%7(3%Hgyzd3mNCT=Q(ihhaM${JVKU=sj7(lsI$oeMa zdsqnfqCNOo*+XKJswo}>S{8N0lgueCe=G$iu^XX0fJ@oH`z07~)GzJA51($B?cqlA zj-?n5^&~Ow4D{9bs?5?H>!9Yp2Qw8+2D2}#P}1nOucsvkQJ7fwsz)vx81JzW0tcf^ zR>fW`a-?RJY1HrAyS1bvh#3~$&O5@G!&+R*OF-2m$%dXs@*0*ws2HN4fLe|lH@ zkLuCSEZ@k4mO)NzB7>2MJ7(y0SB6k^#K&&AVpac$HAH;rcsdZk(eA2Ri>SvpKNI&# zY!zyPDdUDe{qXKpd*|@r_1+sSHuhM;Yu|;V;zY;R8PZt@;3LiyLC?LQJAR=SIDePg z2S1(_llF_}>f=X8)$J}<(dDQZfAvV9m2~u#_&<~hYOdp?SS%$vpMLV)(FDNA+=-le zX_kz)M90bXheG>P?bl>HT0<8ngE`~4aIL(!H0KhaNsEKpya4!9-nY~exy8yBR(2r? zV#NRcl9eV_HI!V1)npJ=J%|j;C~k!C`Lw~oWI+ZRe~iRU3^$o6mIsRkf4E}^|Id@d z10YTISJz938Y0G$1Q>t~aPh(t00CY|#{?)LVnGKUW@QP%Ltq+44>p2P)!0qB#KETA z1~f?dY7N@1=!zA;YyiOFToR~Yi5=o6jzk*dJ`rF}v=?;1#I`#^Y)^e<8mA%%i5|(o zksD5c@jwzwk+Tj39NRK+e-5iDRob?Z5QWgfG{ve6`IbhIK`>&2nTN4V6}#E3l?$-* zv4lxucN_o4Q#&rkNud?%fUaF@A@N6jP3pSZ;HYZdcX>#yf+_t%PpGC}FaX%dp2{*A z`f~+Sk~Zj=QN+DbWkd<7|C8uaqh{LOOaZBn9rCsdon*)mbfO_v-mQ!~-T zgm>KNe$Y=9x{2J{O?y?9x>zY%vfGVr*@XGiDr}Ff>Ld~@e;d?^R=vg)M1mWFgc#M1 zA29F23&@T3Z@TJs4F}acS7CTaFV}U5&&=$TAcWg@aVxdnq49}eYXlp>UY4=OXi5A~ zOhnT>-IF_tVgUwt9M>~0xcl`&p>wx)KI%etVR;2b8dRD18tGK%JzZu ze1G4qXoFwY+FPdBFrOSjJ#}tsh0;M1S{W?2xz6$)2(zsF0F$t2n|0(Uq5A0T(VOe> z_Ojw7e{Wya@4c@iI6M%~oyoBV0;G&W1jqJ>HXjL!Y(#kbDhb=ereNx|CQIN3bqpK8 zhwMPNYIS{mCA>;3?&xO^1pP^fr?%uHuC*qSf0&L+%;U+dPQW(_d*J{^*U2r3ZZWk$ zm4c&l21VqkUYTe@hSFnK{W(g;CKlN$xpZF2f9PLK-K&G(0t^ag7cC3aQQMzF1n0?~ z@aQR`XLNHsyPY&INk^@m;@7&`3D5t$Q_klWVMnZ9%EYU5@+>eZN~!zj!VSalhfiP$ zasrMW@@%on$;fYg7Uo%sOKpG&=_}J`9dKk74QZ5NPgfqzP1-DeA~}0R)GMKGzo#|a zf2R;I0;lNTmV6!V?e4H8Ebf{ZK=}lS?Y=}SOB=-ku(?u(hn2K}eG5C*COaq4bV7~_ zLBDElccoXXr~yjnP@!8rp9N?or=MKZG74|&>9S=f%?9GZJIKVE9W6pAi5ZBEY0g6r z??$d;N}S*qXd~uV-Gp_Y7_~jmN}B>If6l865rEt7Ix0KSGS<}3*|6OlAsdlqI8!sL z?asP=)@*e1Q7&~>mjY$%K!|*Eta7!^5Jq&d$jt6k&X7JAlg>BAn=iPuy8xLYggUTk zVL77AAhZ&lM;@JlLJpZPF`+nP04qYo$~)rFZMq%W^=LvA3HnAc?`tv|DBD>uf2?Zh zj$_-yl27j1M53M;Ft89H4GB6VM7$WGIHxWTqqqpgP8fJZCQa}agngOFZ-G{TI$J6^ z*E+&2%^H@p6URz6(xH`TtD=Cj62T(Wn##_@0SC@TZU%GaaGg@fDC7-h%bk=>*^m;y z9E-~{ONwXyzfBnW{5#Dxu-21_85No680o8{I=j0BT3051+s=yti z7BjJbVH_>@G8G-Y4&jD5{Pqx;#7Cj%@inPMOmW6!@dS~=jA~JWIi}8yL@%b6`vaZ7 zf|k(pK*b)P>q?k zxyw-xW?80O<=|u3p9PvqtEW^8U1J_#i&n*8v@fA4h$czDOt&Rz7~yn#k(R^=ZmC^n zJ%MMJhOZ^2G8H!%AmvTGe|Qpw<7e6iP3?T$v z$7~QugmwHA+QmR-Xd*_MY%EO6zpyo-NioFtxEx#GBwaId9VhkD*%Xk{#$LA?^+R&O zB^DMFPzRk+(h*R`FyBb8Of6Z)#5HtTVDv)aCtqZ&(8g}Z$Oni!w+;nX7}>%VF%~i5 zUe8SQbz76uw#_nrf8h@(yX%-W3}$92M=?x@H!zH1V%?Cf2q=CW#)J&(I#EMC$+%e? z_E&9e$g2JxS7|w`NXtj8_h?d$Z~Qx*y3+<{U*p5inbe{y{yoLw^pOW#;m4}yV{ zDS=AFfr>a=<<9ZDT~ks(q+rsml`KrOR57tmt*9qY2n7%pC)XLifcNcWV3RGDX zM6&dOgf|a_$x1o4^##BRRj~Y9F8|6d+@{=t>lm)T<-mVk7mEfyHb7$ch~K0(cz$aa z)G9B8s6v7Bu>OV%RFO;b4pBj3@<{|P4rkP70`9+Ve*~Y>lQV5&XhF=tRm8F;^i}z9!C~NoZ?Uv>BtiH@-**lE0xR(cT+9gmx8Z;Y?BXvloezfgyCwJ?F{JaeL+@y3jkxP@Q;665LfPP) zvIMgEW*KdSn)E7i+G1N*=4!)&~la1t?Q%r0v6zL zRXDrs#Eb=SVumLZDKWc~2{t5ExHFyo9Co?EfF-M$dIIg_6K#~V)R~8l2y8FnI_xze zcyTN#`0B8yidL4?xuvQZatyyn^%GBD8$N+YO+uqZYQSOlcFNP>uZ|rxyj* z{s|Jcx%DjRJON?cEgSF`2eOK`Ho+@cu8BPCL9xdAaI8DRwj0+>SpVGy{V7MpM3w?c z*B6ILcr4iX&Sh}6)+s!$ZaRs|k_0N5@v$YPSbu3k**??`8je7U++azGmTZm(_4u`$ z;0c*Eg}Uo81h_cnn?&8`z+?S}QzvKs_9pysh+UGDrEEVXUFQpM!vzjq-w$D6V%o{U zIzSjL)$BAHjfze9*|L9?wI!Zz*R)hK?~pF!g81GW99#Ppa{9}}&if01tI63`<(zF* z?SCskxQG4;Mpd1!?b3uFa6B5dCe+Dg%1UxkODpe^-E-uAL3u-Vz6Nx%Jp=$TNlfjT zT%+yloK%?!0bR7Z1NGoQL3gkwb_jg`8w7qxTLUT>9vq)T$;@_l}#$s-Doz{ z8)756+5yF!R;@7`A|;@8J-L?bc8S9RLVq{X0S;<@O@)Y~JDFP=n>T0##VWLbxTAJ6 z%<^J$#wB>N^>wh@+h*;#(s?p@UKX|R41%jss4uiM^^_dWJTFN?Ja7fp)LBSi->{5) zBu0i)ENQFMa)wGkDj(?Ks z8dno}PL*}oOtr!fN**$!&GMHtcKb@{R}WMXmNc)Td`OKe%dn~y`zE^(DgHHCZu3G) z%Rzk8KD7X<@66nM$%#%=;!BA6wZ4n#Yu%Tl?65e;ahS&aT?*Clo0aC;QzWRWHCF^J zWmw@{PFSs6VWbyfc(YPTv%-$uLVwleN^zeB%)sh4ducKa`a<3!k8geK2N45tLr0r3 znbivaYwAZclh6$HBx^}5u21$v*l=>B zjC54X^P7wG=wP#;N2Y_t$HGS#CJ?s*xA03f^O$q<(+=(oS^ZFblx98_pba9t!9lNd z9D4Ae3FC!>Wcq&;OSBr8N9_foMO9pR)9LCKw|95jyG0=N;@iXBz1Mr@NrK|?1IWL< zo9-6d$H#}qMIE4pR-{`K1Y&x~@VokMTB#>b-W}~6zC{3bM4Eroe)6QH_C~^U484CO z*ER& zY(T(nYL~QHDy2cW48vlXv&nJ90+X^HjE$(nZDHq@ZA79{ct;z4XI&sOIsz?hWF%mn z42}lNGMD5*waT*u%clL+WK3RKKc5kQ`6d&dQyg9^HEX8L<}%=lhUE<|G362@S@#|x z)cPnK7k__*o2W87GR#5x+ydI0iVH82rfstoF=1eRi%dR=5tdFV4*wXw43mt0f8@lA zqC**lh7?ivU6Xq<8BkXzz@1^askapzr4>59L-s(iH>*HRvYPtnRYYv$Y~5_^!44km zXl}za+uGKA<7daLPx5;5%(Q(Nm{J{>j+1?AglT`qrh7K*!nPBBfOt4xN8Tbl@keCM zf~C(W97?Pm$r;My=Yyl3zMLj z%)J@R<7hZHHt~~rgP=F z9(;eIVzMPzJc5wSJLA6(Ch;etrV(3i4}uHrYNH2fb%SP?=X?qOBSB13l7S6FpaVFd zb()D?oTN)lx=3Qp?KbRyf5R*m1#oFlop4Sc=Eg@gAL$b!IRM=vIHg?Q&FaF$Uz2gXOfRL6BUU26gOZbJxV*Txv!c;!70-Xq zikYs9$UTTHVs*bT^;zAfx{Xz)T5X=G$x(^U52^e~tj`{BQUDjri4Jw@XgAN;Uev@8 zx`Y{pKdCqBR?Cnb&r)UiXJ}i?FrA^Z&Uumojf$*Aj>hG8meC~$0@gxtDyx;jmq7#x z+6QxeF~#3hFP@F4DlLc7+?-|qm}P%u>#abWd5EN!SPe&hmDTVGt~Hr7y(<`Vm;_LB ztMsEmha6p&t*kJtvCT1$$B`Wgwd)0YDDrDpu=E#5qM;2ba*zthoIHue8<|9qOA)M zDm=3iQD7pNf{b!5t4D&e^+k3afC1UJ&06y*2}tOzdfQ}|&vGqAZ&*!`W=gHHz>CEU zH3j1dOPVY9$kZ2JNt#P@=L&x)7~OJcibG+ha&G$`flT~#ppsq5mYk<_Oa<+-nRauu zi~7rB6pXs4Xbbq??5AAyBeGtMrxWL%sw#|cNdk_QU4GuxidK{vYDGEYQ_lD#X}8@Y z8Ad-LE_PhpYH3AQTtBX8<%t`J39M`?7q!@2n?M*Id_M-^47*&JcXNM9=fHMSV5i1t z8ek0ujkPGFNjW;LJzvmr=i{;2%A{_6YSPZ&^Nn<@Ijlv5*A$ zk;x>pE@f86F@&k8Om=_q%(D|KZAV+6n=_F)2-+YDfJOgzv}&MYKP)&0Q4d9(avH+5 zcX&=SDP4Q|alW{#Zg>8kFF3DopVy~9TuH0k7rdWuAgfHzJevx62VD#8z^%9m-7wQaN zkVQNd)r)V{*80=u96w}&8^p6cL8P{du}ILsfD*GN013Sy@ns)|v|~oUFDm z$zL$}So6kXi^GZwOX;qP@xZ(Fi#Ec03cgDEw{x1`iIQ^BY1USPq8u?#aROrcP5z-y zRlzY%)o>52^T9ct&f9;vleFxcoU-`=o8$DTbEFTCbDZ+ek&Yw!sIw%)aj0i{d}e!^ z&i)(;XPU!+q&I5pD6#Z|vrS`1OPM^+?}2V*Zn5VUEMCT9*bHhj!)5R!O0b|PYnw&( z+7v@|t@P)iRBY(bR_0dF_9Sb4w0*ks{cm)q_^7?)Ik|gjcc6c?(nqk+s7e$Xxo>AH z73ErjdL!aDJxyjmaCC_x-+&bF@9!&WMROyDdD5G*`azcTzMJSdwB&;b_!BVhS0tQB zBss40h}g~1bmRtZt_whNWM(-KRqYZ|k zHJ8l!X5w8Y8TWtqwg{Xb2Sgj|gV2{pXPOnhGJ@jZ7P>Me_OWJceyrW9eYnG67Y(e% zCR!z1UKz)c+?A{(QJ^KaiJ^aS4NQHcFL1JK+HMyi17;3=uwF%ukxbWyxZ;1pRZUo{P&al}$?+C8_vF^` zh@@bwHJg}7pfpLRfrbc@e)XlEGZJ3rSypfpGj1TxTqI|0IC8DLVG0MI8?22+d~gur zKq3<-9YKQLQnuczbfU2*RIcn_X;K?VQUxw&T3*y*$isUT5(D6k%GSk{5gp50xU!5S zNr~LhN%wyeax5JB9B_0HL(Xf@g#pf?s|^sP-w{;t?BMxr;|9lqRid^bAA zrU}@8(rc1u3^OBJl_8dk%;y~%nrm7IKUgC4v1;M(j`>z7y!CGU*yi&|bZAax(O=te zr!3K_66;+86Al$IZI1RNYiF5nk!*x5iHt}vdh36W0zj{QpWJL3iwW*Zj$Se5I}}|u zhPuhz4jkLa8EmB}(Y5_qtWC@SFi&tmpgVeEXfsBQ#6fff@B9eyg|)tv6Pm`886qB} zqEzxJ#Ya0czDU;UXYR1edJDcn)=G*M{kLcY_<>HF{E^Q$$~Oxrrl`xT$UBEulo!Sm zS3iG2hopPVvNQ``3+pH!v!+m~i5OmTrCU>N>=Dggbd#~ZXlwH-?+02*3l;_dH?QzY zfnOSEb)flo$NMSkdjJnMgwO$f(%T}JZp;oiH*0#YB`AO|?adFt#C0E(ih6$V$%>;} zPWW`al zn=3eDz{=l0W3UycHW4YRYR69lOor#B^DIfCCwo%@%McR=!4!(+PS9L&J6g|ziO`YS zLE#v@es;RpZNBEz{^8x0xmht<_0NBrwrNBgbvO_ zK7Y3Hzvs8Fo7C=_z`Y)9sgdddhOZLqgvGxykV zjcwaW)7Z9ciRu!e<`*xD+3I z_}he|)FF@pJGn9L#7)G||M?uh-8h?5xnxVE2sRT`7IV|nfGbibnh!sR5;PLgpBj&M zC=8krE?G#1;n&_6%{KF&$Cdpf&4aac=w4hvH z`q7#YLo>l*$}eFy`VbAjiB;wxDckSP%pQ%DtlyG0kYAPO5moM1GL$32RpSG1f=I-J zMCvLjsGn=0O~W2d(q#==~lzCdMv*8Ry(Y>o)gdO+*(R2 z)BUe%ae|)!-`HR-@5a27L=;KRCM`3bf@tPml@YT!^0#q^ZmX_lZMll2d^2IIti*&k z`B0azAu7e>zwHKYNzV|xg9B@7M3SLs2ogq;5-&8oL^3>NE)+Pwo&9ysFC3114*Y>r z%EexyR-bb8Zvdm+^RUPCM4hC_c13w1ja2){+;Yx6F7d#RHet7wi|o8Lu_Ps;{`V72 zDBmsGD#+||L zwKuH2_28HNqQw7?u%BnyObW7bW=@VAEv+3qbT94ET#GbgIy*k=Vg$1({{REP&9Xq3Fr_$ z`+FI&gDho9l-`%hBjd_Av#Y>LSoYkkb%{f|iZcX-4Q0HZqcOS;%e&oM$2k*9%Who( zMd|q+zKWWNHxos!t|C(WT02A7*Fl$*P!CZ$NPP~2Y!c@F*Gg{q9w8xls-Z9!si2<- z`ly5MkEVC1}$1hT3O$|pVE`Ui+pRJMxnvQdZ0G_yDa0D+C?Nx->DJFK5c6s$D z&ko~4^o)B|7yjw&qlPTK9CRA@e%5h}Q2<8$``kB1nswtEcJszF2hB?rb{cuQ&&bf) zj-Z_Z%4siN>shhoks(t^!X(NP_Er(WmNLGy43K3tSKJzYK#x{@LybfwbJl0WcrsV>DTh7;SnsA42k&1|%j0~pJq&`?F)IR$IVdTgs87T>6f@0kJ>P*Z2ACnDergI>(** zazE&@^%4hXS1jOVBA`Ovs$_LK6~RS^?M);&t?Sl;{lHdvy8={`#Km95+s_3GQumY` zus5lk{qw5(W2WzZtF{7z8{gppl@kSiZIxy4rq zBM5A?F$Z3!LiQe1bq8*)M?ka4WTMCRtD}$~1}46h3A>=BTsz-GkRSEeHOn{PR)Vnp z=UMy)bgm@ZVhDv`Q1)Ay{LYrqeXQ{vlICw@dViEF?|yh!z~|bu2mBHT0lJHQs)FJ1 zcxOV4>e!{?)h(f|B$GfrcEbh!U%7{LTrwX7CsaKvSPV?|mS$+UJYr^Vz=+jh@M*N~ zmG(I$rdDBhz|WuDHu(AnBU?;?%nRFZ|2Z5GvH^661%yXD=In?P^q*nA>#pUJ*HKuD zrw>78^E5cXzl3#L_d55Qz-JE!;cy$}52c8PIV+c(Y3;wBW7|kPCo7og!#?GO;Wz$+ z>gS@{FOokfendMsgO`1l{jweMTK(!$94=}upfU0AR2flQ$NMtCNr9rTXYf+rAcCUk zDc3N>jXW+P*eHROZ?zCJqUj-@dRWWyACvqx%2QVd5bn1q{Vgu!soZ}WqtSHWz3U-< zE)dF&6R197kJuu}8hd!f{qN5b5fndh0-fz;t9PlG=y<^zp_IsnaDA>5s>Wgan@lTq z9DJ@c9RO@=(qxq!*fc1&gsfpnHRi0(q7~{x!^W zv3bH=E$AefnV6Napcq%i)72CAu*EgC7t@uW!W6iZ9HKnxNHm>`t$Z&5ZeD*hunS#E5p z27i6*V!mUiHQEQ=LE;wv^;T&NHi~tlL3Ik;C}eIe6+~_8eXVN)9qp;RAPx@BJikK# z^p$cT7&LUW)%J(>6^`8RIKd;4XFpvSdHlG}drNO)_oF^Spth|mbSkH@D?UBa&4K$% zQOcNJApf~?t8LKJPS&^x_OA&Nk59~(&`)OrkP!0_USCcBP6i}_GFn8+hmhC-?|V@K zKKY>p_I^_)LCL4meDh)#I6a#=VeC18Dp*m2g_fE+H(Ryh7x~)(_o)x^K=!(n$y)$+hPF>3m$aDR4CK+&;MF=lDMvqEi}2 zb8Gx`YyD}rQ>FD&?Q&4e_`Dw=g|{SV785E>9GXqSB}8AM$(Fb7V0r1p4peH^-p$V>ei%RR_w#YR4ZdNx;2JM2tqU=2T3|}S;=IHfX{CvS`|1Q|05*q1>Ls@uTLE& zouRkvK-JD0TbsOg*Ef-D_~k5uP3salm!I9l>*lLv6Y3&fJXZF^*Z%?{s{(%aR}6%L z?mwf-9kwgfc^a};d&%DCO~*?Hf2}mE-U#V(;7g6Bn)6Nd}wg?Zu+Qkc5iHGP&JJ_ zl4{=g{zHfR;^+ePPJDn2{B7#9SpT$gk$Af>;8<1Or_6NC7sL`ld!B|U9Ss3{-cdRJ zUHM|;Z%Nn5{Dhtls(V-3$b60Aj`UvRFA0t8H2+2}4XzsLKb|cI%`sO&Sv!t~wXLr` zm~Jt29sH91cN#1ovGJLSmvb4g;GqZ7Ez=npX2<5{VX;oBwDUmPJO*0=f;wYLAyZgp zrqgM3i04CPA{F9&Ov#T3n(EzQ8Vkz+ngp1cB?!|@+8l`GCG-nx>q$rl8~C1Pn}SBt zBCxo|)q(tNz52v=LuqMwMcInr?mazPoQKit?3GXAt4W#B?VFGeikfv!1Ak)lVVz7L z-_mLQigK5N33>qHXVfWQhO;Kruc8*b$s&DC=^ms?YW%Vk5lv6X&c};d&3dDe67np` zIgXng8thT)WC{+q6(Rt`KY`t_UF@7G?vV+&1k#TkT@hSL<``GS=xK zH?X&JB^e>IYpr#H8dLyWmCeyb?Kb~{M0<9B0}X{GhJ|}mmEK$EzV;dtJ6T(^F(>!B z^0i8{GGX}OFA13h%1_+sP^`ybI5AzNQ)H{f39TI0q4=v}6*|(NCCR>MD!nMJ*VGaQ z{k{Dh@tpu4x_lPb!py{uSp|acqUa65hVZxfw!>uws#6cEusqJn(dg5-|Ned)WkpMB z--|b?#oS=M}x(#A1FHbkmCU4tW3d< z&l0Nbzj%6-BKbu2n!!K-;q=2JgcjVl*HaET690d=A6_J4u7#aeKQ*_U;OG$-f#;L-WJ_qHHwAe9&nYm}SWJ)X~CS373EfH)qmHVo;PCZ9A zN?2f>4-vkKbas1>?s1^MR-^yV_Zp1V?GTiq#WckAVHQ8Fo<8AAc(M|&*$s0ZN{moW zege_B!&_{2Fg)DjFKlAd%@q;~qcxsEn~8z)L|$$F@_q3mC$XxSoby1aHW{r@WX|F5 zIWp*7TMI#gX?xKmK%U#S*YF=LJX1}(LKockKZPSGB*NUmNxAS z`i^H&e$?dy9CHi>)JmxY1J}E(7kyv@pmGb$&5h)69u7? z_i&%_^bn=x%L#Z9&cTs050ovD1frWrbwnD`&GivP3EB*-#jyvb`dTmW^UcJ}9mYVF zTiItnauq`UlQK>pvp&=ot`y#m%Nbo8YJC7_lr6#fGIG&Tos{?J=q2a%Q)?V=T_-=c z2o5|XIe6hBrJZI%ofaPzI4n8*jgmq&cNZOeT|@t&jl8$u>&PRT5d$|AW9QP@u#0+p zqr4PJnqeYa{waDbbHH=MZ7I`g1rNZ#!Ec+@yXCMMZfagE;TQsPd<6=Ueae`PnewT<_E`_vHmK z8lEd^C|$r+&(JyAUSuv$kFnKAyAg_JZyB8aPSYldH${K1W!)7>j?Od|j|lMpSpCq< zB3FpUd3?BI@jy}ByMU)T_+7DGF`-p!wzgPFW8RTS7nM`%IctQ~7Z9Bz{RpD%{y66% zqj>&WU&GKJW9hEHPq_8*elmEuEBc`BoLI;a)9d3Z52|<$CGYRZPHfFt@0 z_ltT_V4fZ3>%`hguTm|mP7lx>Bk7)eJ*h=83j~XZHjU(o>24{f6=1 zH*v4Jfc+|(tHes~CQeRxdPP_SRr)-}q{%y19hs(@;a0MA;a^&8YdVGTOK%K=kh6D% zYSjzA?p+#Gx3Jb3sRaP+QWpoKyGZE`DATb8w3m?bzF-QURGO>aJW(Pa zW^470!i;YMrRMNipl+g`BCq1K>X&O*FFC*&BX|;|F9?!p{{RwiUT3PlyH9%*i~y<1 zQQ?{?%k0|0yKSmfI7!U~2~Li%Me?W;(A=V^;${;Zq1JMN0Gckc>xVzZHN48cD}?OP zh;dr?z#f`}?=gO2j4JKSq0|>+o2|364Nt8MDyxl!1|6yEHYMGQFl|orL}2!E*;pDB z)-W0P@ZYnr3*h8gIZ~Vdo`yIo%*j!pw2_1(G3RJ^hBwq zu0U6@4t#@%eQJ(E0@E+dve=m#iiIBTN+~~D^+*!tL3%9>>W0NL*muLFCAT)~4NQmS zK_mV}9Vp?m;YSTc>q}O~`0$`{Q-1scte>|$5Nvg}slXg@&S(2yxsve!-wce#_C314 zrV{0nBnFjJi2;)4Rq#@6EU9zf{${lmv!~ra$GJ= z1rgQ^VghoKx?z%B2U@FbnBzz12j_D*CwXkgovKYsqZ4Z<|2k<+ZB7`n(y^a#uw0{) z#qqbsNq{?kfl>upL_ORaqZlNbxi2trD}U8hHVt2kPDfnY!79$4=PwR)bFxVa+8E#S z!!A398M`cW0MN{#7jPFVq}yc z5EoIDP#mP7{bx|<-qtwN>Ruwsvn6mgbHpU}$}PcTHK}X8Gr$i`zU+c+m^nu1@7d(! z$o=u;WJkOMoY*2!a+lhl3WAZ1Z@G80xN6+lO43+Nj6pWhJCScwbiC(lYJA;47GV5Fkm%4(B%j#ri4{BW*nUz1`7lr()N z*6d`$T3A!$-Br{bI@lq-C03A74Xh4C?AA4YgTB`RGL8TtRHL9+@b7k=WdsZc+#scUs3$aT)E zv0soGoB5mm<9>j!CsXQ*F?)9&Ad&Hc-q!AoQa@*xFy!Gl;^euqq?^_+&0el*7iMVI zjG;`+mHDMIC*c+F^Y3UJBa23BJDib{V6cN%_5}&X$Q5mzyKaf3Z42jB3nruD`-qg>v{YMU03klknCPLsIwhnf z+-&vy!ym2W=o$5_^2Yg_VxTUm?{`7gZc($lB9sIN$$!*cd`6=f-d-%|PCb^{%AA^u zYPP?uj~;ZfjaC_t)j{R#wgor-`gA=8pYdnUe!~oJIJ7du%5u2~(DJ=97f#2R9>$^R zl;>5;S%sGm6W1EqXyo_pHk%ps;c2Gh#0wH@MeA`LAFD{onC9WF$H0~U>8ZeXyO?x; zOYx9=y=RF!^V5yZP1hX{wK=P0C7mzRLR!`)3M>~|Tl!-uyXI^p1q1rxbkvcFk$7zD z&x0iRMZ)IrRoZ$ERKJV64im@bO;`olf}VT0+w-~Sr=BtEP4KuJrZc9^fZ6O^F)5I| z$l7&OFo^oMGFAJZ6pMlH-;2x&y80b>Tp*HRF#uc{iRg;JgdMcTN-6IVb_#xV_<3{H z=<+}B*>m0Q{DhkhR5;5EC)+R>klaTniOu2`@VWGiLwH)p`O_l!*q_#>?-80@zfR$@ zPNOp@9ec-K6&xktSd-8Uz60LyU#oDt+sHfS#fKg867D>(9&!HB+%bQT2M53XXtQ&V zjX$@kd{2o)e3lU&Az(5^M;MfqbPny(;~CsUJc^_8=zL08G;VViIik%YZpB#L=o#}X z>UH7rUWl5|(FN@e)~*P%Hsg2_7Wi6zo2-`6M{D=_Py-{KOeF|-P5^e4{ya7}m(EI8iBKa&)3pA|J})2Fc=(9UWsUa2-Xy&~fmAUL9` zNwHvuWuA`aJQEH~5BD$rFzcluo%oe#>sxI>VVl5DZu^=|uLkJZ=gf z?)Re)Pis?=k9gKV83MA@2}PvQsD8A_uquS_DTDR3hURn`nUPoKI&wA-}fu^ zZ25BR;nsBKMzXhms>n_&Rj>90zg}x<^6&L1?~SG(Vet+XeO;#jW<_{3$sZRl>9FMP zQH>3#x=P{+ybTa^ipE0s5}GM@^3d&r#+CdTp4INB63=vrJfAg71ovpf^$*$RS7#Sc zBEA>hWtZk>JYfC&g>Q#ld%IqtLsb8Y(D{MBH;e-YU-{%sp^B+jw!@I9GurE*y0Q&U zo86UM%xAvEg@!va={S0s@o19dO<}bNx4CQ0wdQKg+uFi}t=L znNE}TvqrvFHg&72Owpw@xcb$+H*JN&9ExmRP1JvHg|Y!zdLWQQH;Kx?doa{jY*;HJ zyT*pipBNoUxLQcPRIcV>TB` zXi-=117>&e{)j*CuAFOjbQcmy4ngpak~~h5K&q#rFB7T896+fq%xXYT)4?(&;|nr^zb&8$JN6| z0t|lBNe*)tp0GA{%dv|+x(EB_rHakyGXw>GZZN6&;m2akxkQdPt#Zlr-XL=TrJ^HYrd*?}A4aPrCl0nK67S+0 zNW$^aK?`)tvUpU~l%Bro3!OMoGB2y1fR-_%|D_4x&?tdm_|T>h5d&Q3jXxL6$Lf>P z!3z^NUW{&LLT0~7%a3hkx)_>Qk15%4R`Ny+9?eib(+_`@fP11FBmI9% zz{+^-%=-te#(x8j94(FKJ)bTpbPG6cE$aWyXYDSFBdOa0f6q@C&%DEb2IA6B0#cWn zY5uWLVmCTzTBw7h#}-oLx1RDOEwYlKuPc~#n88qyUs%R0D@#!U-=eB4(P? zAKA-GsWgM7-)@^`Srej=?GZOa)mIKyYRcyONYNAf)6_o`@oB$aA5ZfXYydyMo8u+M zR6Hk0>M{1z>h$A;6yk|rJkkvypy6G_@iFLOGJxhfiXPT@PZ%_L{N7$?XH{4e&4i?- z{cp^VF;%hLv`fVpi_?kmkDP`ehnJ6!LnI!X^oL}HPn1s+yU2_}P1wO!Cx;(TX6|XN zITJ^ri$<_wPAy?s?j3ht;XbzHNh~uhhpZTbrvAY=8KOep3CCF*nBP1Md~-pI$}Df= zjW|AK6S&^*pGrOYJ3Riw|$-D*vV- z#=-g48|gCAhFMc7vpEVK7FEU-AvL`2Hx7HN{xbd~4Qk0UXxF$%ZpM&Og$~>B0*~4?M2z+D-6l zSvoq?sfc+dnM{yce2aQm{eR4<|4<%f-0tbVK&~JB>&}S43@#x+kSW`azBm}%2y!V6 zA~Y6z%ys&X4m41Joxq8bn;}+nTfgX;PSxN-4vhF@N3w0nZ`<7aoc&J0gUlEDo$^Ir zeR>AYo!{?@Q{tn+NN(uFD!u&wwBh=NJ2*@ICIfV}ZCE@ZcFxb-cV>R{ip!7|YDoQV zbD$+h_RH2a=`K#?qYuVx&Jmb#KAzkU0C{TT=hP^urG5mMK3$QGUZ!1f8LFYr=fmWXhDQGq?DEJ%JX%6Vjt@k(BA7mnZn^YL2Up9!&aFqi!m6 zrHPK%>(FMvAIN-`YmD=6$lUb}^j9!mGaXd1BN5p80%Qahr2GSf_Q78}%O-dG0&Y!t zY?K2&XfUp`^C)oN2(bz z1fA%|ImrGE%`=Q;i5iq+Om%;HN9(7s`BocnSxwr2OHn)h6Z2G@CQqZX4R6@)p9uu> zcSzjPMHJRDr~!l=A63fZ_hWu$aZf1ks>DsB-ig)H<`7WzFa;5EPZ6YnmSl%aQ$)LS zL#^~`-)#ypWHqD&cA@5Piv;2RtC(E7U-QcODtI2?8V>RbhRvBTr~xf|Wvo4#a;4up z?g#h)k%d|0k>BY=e6wY-uWViGzF6tC0#f#OklFMWTss+rL z$>95F_Zhjt43vBzV4%K}j%-@(hHsIcvtjZSBIY zx8UHYHTj!(yD}-|7nwbW>kbO-zB}JPHB_`0Sob(u@GdNz)s5NMIr@++@_7Hxu34VR z{!5H;-n~_E?w-;d0i0#ecr8@APu366m`jjHv~1acm0U+-ry=sL~{m&ufUm9_x4|s5~@$EH;Zj?e_gg@hxnTi`9r#zaE&8 ze;cWGFz4m`0-55jdSXUus$?9|eIJy*T_f2c^shl=KIX#ip77&s%fpkXX7NTq>zA!s zR#tf^sieyI@Mu+~&Ik|(hw4cEJ7?9p&451ag%rAFxK|DEz}B!*tMvAzT1IAZ{Aa3! zcZ0g(RK+0iSg)$~^BCqT5SKysPGkBEVeelOr?FpsTqJ;X@VGrta78vtuM`=-gZ%zh z8{^Vzi}&a7p|}0EX?5ZN*suoJdgnku`A3{>hE_Vc;S=JM9}~loTvQV zMhG{>H3YGHYsN z)?7)Vv{52{9W}loo5h;5mfKP3S9ljs+}|pZ5M^XtH!SQCpzknl&JDm(a~;A7jGRsJ znA_=e?C(C+(|2_9S~kL#RE;Pi6ldkAOHu6n{jZu7buSnfvO|0?LxYhZC%oTR!UQlg;8ip0gRWGRId%Ng4JJAGtx=yaBc(HR~{dO&W z#e$E4i98%=Z81D_UgA;n^%AHOYUX~8RfRC2r5+m7Hw^o*XInM?gs*8qXt`ZgxQVCz zYbj`C?7yXk{G@I_c(aWcNhkO07E)fne<6ygxXb;d?m@^$1|+~!&Z#~tFRyj`YB_yr z&u>g>-vqz5j2YsdeAnO9e@N~AyDj(-@{Z50+PcY_{!IL#4m+lQVe+Y(sJA2nxz?qe z%Hf8puuw=n0<4ADeT#>k9sM?-dVOg&^Eu!qJu)&BoZ%)55xoIrrn^R>cqQeOKL$tGicK4ytQDI5i>n=g<_W9s3ea-n?I5g(9IT&36?JNrdph$lN|kH zkVBOSLD&v9?3->qe;-?klo2c4(s`%cvM?E~nl&9wjZ-|fcb0Shz;|_JCe`ae2BYqBy3ypmAB ze|rX#YDPx?&xM6Bh5io(^tjzcmF{pBZ{Ws8lXD889_2*jrHWl&dqOr6a?-1 zTxfVsKP|WDSQt0I7SB zUd@M}NXBexN`CrOFAX=bjoqlowKbzlWYw6IJxXczA`UjFTqsDamu_p2Vm`V`k(ofY z29M8J*3je?sMU-v z3;D;->>=vu^O>J0JKfIYkma#}^Z(O*m43It({4OC&{R0`FO`r58%~YU@1Yi1Ht$+B zeEVB&Y=B+8IkHMNb;B1n>En+z8<&Z7K?#`FpGD}6n^G-}#Qo4TLm1=}Ok?cprLbMR zt3RsKWUt+}uvV2$qLfj(&vJ5#Y$*OiUE|*)=C-O)9e(ky4c_hpHn;LR@O`)b0o7n1 z@%IN@c2jV(JOYxiM8#BJjuo}AmRG;ne}Bp|_0Eb3&2YFoE26x{s*ZIPYPtrdXt)7m zS!=~9
  • JWR-i{pT+kQcG5K$iJhtsH|5>|v?$l${62GQ>vskYlknYwe${fjqcx7P zb(|K$&Uu{=K98q}w1U4_?u2hGi(g)GG*c8VxEf_qqRCyyU()4G8JZKYP`k>jz{xjv zJZn7lZD%k_dnaSe;=DN` z!YiCX^fv6B;eWA;6Bu+D!A{BBL@PPzC}i^$iJdSTizHpGy~!!G3Dr24#mt^^g^>1y z-G7E}jfr3T4)*TUr3ug`+R`?67Sw8VOrB0FhFU1TS*ael=Y8DTZMhiz+0%g8<)v*d zLOC|kudVwXg`+Qp_nd*zopC(~0YlKdEl0yMppX^QT>A`wq@RIsW_Rck>Sr&7UW_Er zs}Aq2p^}48v0kczRf(2Mz6x5eSNpqG;WG32tyCvPce zg5|X9{(?8Tu+Ah8k1@AMc*@yVL#b^ddgpw)_f!EJ6g2-lP_8L6D%Zjqu&NVBzib!Q zST@x(|3@JAm=8NC=F68p;A!XMtj6g!y+92q2vX_%R!u2=^krX8LvsZ)pAk;~84UjB z<@m+FZjVFcFxj7;jBli&ag7rbm%MZOPayw3lJ-~Mg{tIC-vw)Z+QXsewNPgYgXe0- zfRaWS;9jm<@h5HuqT-w+j(8fQAWjG9s3Vt@t(jo* zsw}7Z{hAgaKfIu5PL3A~N|#PwkWDu1se*e}g>Y--ET?d16te;;T)&)#D461)t5cm3 zu9mhO?GAIiMxxzB3?_6TWP=yg#QlXq9@lJ1k8(yWtE2K1DwX()L-qTj1zxYXj6fQ} z(RcEjW2yLZZhjZ0G*TN@=P($YO%wFY-@a#7agQ^?L*QXT@TPl(@`H)fYQ5Psh& zR#9UByzdkI(MZjGQ2@I;fq_-3A)58yXM_O1TsoEd2OgfesYhce!3x9Fl>PmaBM&#z z51whjqWb522Yz=PJexEUNNP7=r>sWM&`pUq zwfSA!SzSh1I$?myAyO4l>Y+&>hBHQa*1!sEf1=V?Mi`I?2L?zoBjS#V9JB9o2m~NQ zH})iGylaA92fX`MO1?QG%sK2~)OnT}>y!+;m-TmxmZI;x;c+p6AblS^wGG!kL2Dh} zpvHy{?oJT!=d1VAB^8 z!tgQ**)NU=?M=uS_{AtRu_>V&{wmNhiXZ>5Z<*dCj?M-bm5-p;xyTTjHUNfhv=?bH z7aC}|O)>V+SzT%UA$NJS*_&Yaw)t&N#$dSAlPHg35T7JN-Jq=r(+?*ptT0}wUWkkW z?3ZTskgV7`j!*$ib6ZZ1S}#o_@bOQ%s-yP_R4#tTj`y>D_O`d2o`9O0Co@3D8X+LC z63UZK@v~lK2UuURkI`EAl}ILE@4+S5$WOCb(vm^`nXZ7OdBT4DiX}PWIwy|kAlO~Q z1{jJk9Z2pr4+JdE9OVs|dI{4uB=B$;&QpoJ$ko1_{p!v{;fMdo_=z03*nvk(7e`05 z@ijyi2;{!!9Bf;uT*QOap#g|=T7(u&Ef8DWeD06dQ`43@Y64}2iF110K{g75Tfw^|5uEDv(6Cl_^=;_&e2rRt=w5{p(3F6eScmt810AhGt zvfkXr{~ARK7*4Y2r$oUJfK1svhkAcYNED|7VZ&v3C^R3_YGRyQx$bv%6AZTY`tVoF zg_{EbN5A+oQbL9C6z2me`XTqU(`ZpMrj*qnh8mp?D}~!BIPUZWnoF5HEaHm?xY%!5 zTp~qR@Uc0GpEs~3c%p}B5Cvn@I1|5iBM*HWp|*d80PsDlJf*{8_lITw zz2e6m(|5?j4?<6sT(&WcF*$Aa$siN|QNLm;AbOJLWHM2}8@wR2}ffDFDg! zMx5Ga#7MYB2Qh*93G}wq_ne)97FWMO;~yc^N_1APbHc`1I9z((dnz0!WdDgVUll)8 zss4i{pzX-~Nx!WYVuEA-x*UoYp;zP|dpdwa0M}B4=VRF>-BE1ngN6EmG?d>>R9z*p z4Mu`dCzB(!Mi=8v>fSp|rbj-$qa1)8?BVD4a5oS>Wlzgu1|a*Gy!<>ysmA$EY|^KI z7KE5N^oE8X`tYgL2VawA^;b_?Ss5vnDVW>8N>Z%MsSIW;8~a3YCP=S*r3dHbj^+*P z10|oD@svT+<^J6`8D$H91F1~zo{ZR!DW$}|fHsU!Gqyi2YM#*L-p6pBINN;F4&!{- zkD`XPeot~h2jJxs#of!T5G_{r8VNyzx6d4uu{1Hwg5tg+T)yS>ALkGu1 z*S<2Yq0ui^2FEXM&E&&alt#=ku-$Vgjt-lsMzgo0fNL=&GoKZTz$0I+!^I8;9r`XI zEQLU`ba;sNPJaz8$B`x5sk?U0AT~`5duaNj(2CEd2@o=X;k9S}0-NBro(?O5XHd!| zE_vD15s{;!S<+w={UZFbs6#cMiE88!&qP2$B`F=a@rXxxs10|}{XE&8uU}fy`eE!M zO9d4PaS7R)hsU3LtBr;av4Lp)t*W?8U}`NOxGSGecDNmhLmXy%6Ezlnq_c7#T`KfA zD{Z;(FW^*sM2j=AR~GwEP;_`vh}wh6JTg0#P|ks_=uiF4=SCb;MLRuX88;T%yE?jq zi7y8Ulw(~S6}*3>nP94o7Y6N!KmrW72;tofX@b!( zuCfq;9*?DpCUpEIMZ+VYIX^4BM0h-WF*XsU6y86>4U4+ZN_v>OirN&R{>hFt zK3~mUe!+oPi%*a{X2q^L!bh;VDiiTVkTZ2E0hVba$|jAVp`Nw+E$7wF5-Et#0ZR|$ z3J`um*d2YxkB^u4HyIO8Ae4tM!GnuOmG~n_6l6~g{2?r+*5Q^=xc5lOSPpm~@Hd!{9b~T2siG}w zFtSdJhV?~YPqeJ~R70B1`@46oUyI@`Ku1m15D%a5Oq%_sfH`#))ZgS4|19Qu3PW|+ zK+;e>fbxedE}|r<9pZZ$2@V7nCjD>BK!^YQH}nUKfYdi-k{gHteAKC8wgYhzx_GZ* zCEhkvrs~irabdFTQDPwj955{)*M0b>l8f$A61>odFX0z#sJ+Nb5!A+rJ4T%;aPOw2 zCBq4~fHfu(@u&d#1!9`l9krN`)U!AzpY zRhGs4Ph9n^oSkYLbvLOv2x+{iT~i0~_ye6Fu}sB~Z}1jp4y>M#P(Eiaem7~(fx0;h zCXGMoNIajd95GWnFa+N~jo56wQGx1;+4A)G(>NVN2szoAs$Uj1=ierZU+S-ACy zLtVQ4d3Y7RdOKPurCkQTyA(*2t7!f+k1t>PSqI|{DTc8K#sb=r9P&F@>stQ*FM_wABOPp2Z|8S^A#A&;YV*reaGG#P*puK1wvW= zx<*2625(SKZqA!|3%2(kRF%zz&_}){Wa2H>pzs;PrQBRt`Dm|`y$DBH1|v(ZG3#YQ zNXC1RTVCs0xIw9!23Xr@lUo1e8GUe;@2S{=T`K$J5*q%fR@M=k!2#Kr2ptF!LEy_j z&^vkENM209D*PD1KnwbQvA+rOtO$KSNd`Vmp05KjuDpL~8ZgY{Jbp+M#z-1lWc=~*6HLGH%prS6%suKiJ3)-}3;=JTdbOl<&2>|K zKKqZ{!3`m(6-YPOKh|>h=@~RWbz#bfp6OC=T3L{`z!QHmf5UH_+7VN znr2a8zH@3UbXcs{=W=WB;yJAG!y+)gR(m(=e69wNlC|t&Gr#2)iC$;)&m&m&qiEqL z|MM%N0vT-jW0D3`$HM%(-ym_;I^Z?W9Dap-;Cl?Q1-!|g6t1hZCwKX zIR%ZNd6aE~nJ*j9U*@34zE9vb!HbW+!C9y-f5pUy4aFW6-w;~2?+|>Bq93u)eJilv zR(DOgiB+Z4xPIQIjQ-XLjur8H40r?hsu}|gC%*|3ilDy|Z4pJJAF@NcKZ9l;D@Rrk zR2-}nphVUT!dVGSeBBOSZ!Pj&6|H&PyVr#%nFOV{rh%SMOW<-RL1CU>jnrq@NXdEx z)MxP}4CNMO#0IAyIP1AbG`9Iy{rHw=9|k(^|2n;;Ws7@IWje^Zl0WMW2zzBja=l6e z^LYE%c6bIUXjq(sMAAXn(e5VCJ2&uPg|k46k|OsHzex)vGlnmVk0ursFoXFIY7ASi zhF(reCM3{n`6#IS_2Co|bo?_11kWS^Kr{YbR{{UcO1J9S*G=bj91rbM8N?C2qf&)v zxxzZB6E5Op#P@Z}q$39`WCZq>8#rqNj6*|@y2DZuwbhLA5~ zD1yMwpoq2qVF6g*+_bmrUU@fZ^=x}~dVb6jN$Rj8C+Stnlc~v0QV)YK5QOLm&ZgQV z%b~MTXd1TB{&H6u`uOenpt)wEdJb5eTEg?%j7bZzLzyU$rIV)^Mib3#i&c(~6D?5N zlXr6tk0Q&gaRrT<22FPj4o=fhoAec9xO(cAd#FV#+&vXLPFuQ;e4=E7B)$R9dci+8 zAZwB$PE{_=KQY(Q_J+kcZHn&~?e|Bo-?o>mmxA!xMKc6qb_0+O_tmMdk$|d`ji`Pf zx1R>~Vu(BF2k`Ng{?Q$I2|6(`v9X!%)||{+yECeaqsaI_|lwqoWi} zft_;Qg6rlVA9|<$9T83{M~)4sW{!9gya(qJo?;!EO3MCm#BUl9}(k}2Yu?h1xI#0lBx-laRhT7#&U5q_Du!DDr9SEqem`>zYoFXa zq^MXGPgb`t?4LN_z0tS;5I7hZ2m}cx0+Ue&b^)G%Mwu8Iio^Q94o0(aFf;!j2cyjr zxN9(l5Jm5~hjk6(m)+hiOb@dAE6+F?BLL#&o%!=oIC>j#r&pI?bG*hv`R~y&C!~K}aG8H|*Hs@6lfnH|Ay|4*nSV0x~b5la-t`S9nPBx{b&}F99$h z&eBHnktaQ + type forward + port "#{ENV['HEALTHMODEL_REPLICASET_SERVICE_SERVICE_PORT']}" + bind 0.0.0.0 + chunk_size_limit 4m + + + #Kubernetes pod inventory + + type kubepodinventory + tag oms.containerinsights.KubePodInventory + run_interval 60 + log_level debug + + + #Kubernetes Persistent Volume inventory + + type kubepvinventory + tag oms.containerinsights.KubePVInventory + run_interval 60 + log_level debug + + + #Kubernetes events + + type kubeevents + tag oms.containerinsights.KubeEvents + run_interval 60 + log_level debug + + + #Kubernetes Nodes + + type kubenodeinventory + tag oms.containerinsights.KubeNodeInventory + run_interval 60 + log_level debug + + + #Kubernetes health + + type kubehealth + tag kubehealth.ReplicaSet + run_interval 60 + log_level debug + + + #cadvisor perf- Windows nodes + + type wincadvisorperf + tag oms.api.wincadvisorperf + run_interval 60 + log_level debug + + + #Kubernetes object state - deployments + + type kubestatedeployments + tag oms.containerinsights.KubeStateDeployments + run_interval 60 + log_level debug + + + #Kubernetes object state - HPA + + type kubestatehpa + tag oms.containerinsights.KubeStateHpa + run_interval 60 + log_level debug + + + + type filter_inventory2mdm + log_level info + + + #custom_metrics_mdm filter plugin for perf data from windows nodes + + type filter_cadvisor2mdm + metrics_to_collect cpuUsageNanoCores,memoryWorkingSetBytes,pvUsedBytes + log_level info + + + #health model aggregation filter + + type filter_health_model_builder + + + + type out_oms + log_level debug + num_threads 2 + buffer_chunk_limit 4m + buffer_type file + buffer_path %STATE_DIR_WS%/out_oms_kubepods*.buffer + buffer_queue_limit 20 + buffer_queue_full_action drop_oldest_chunk + flush_interval 20s + retry_limit 10 + retry_wait 5s + max_retry_wait 5m + + + + type out_oms + log_level debug + num_threads 5 + buffer_chunk_limit 4m + buffer_type file + buffer_path %STATE_DIR_WS%/state/out_oms_kubepv*.buffer + buffer_queue_limit 20 + buffer_queue_full_action drop_oldest_chunk + flush_interval 20s + retry_limit 10 + retry_wait 5s + max_retry_wait 5m + + + + type out_oms + log_level debug + num_threads 2 + buffer_chunk_limit 4m + buffer_type file + buffer_path %STATE_DIR_WS%/out_oms_kubeevents*.buffer + buffer_queue_limit 20 + buffer_queue_full_action drop_oldest_chunk + flush_interval 20s + retry_limit 10 + retry_wait 5s + max_retry_wait 5m + + + + type out_oms + log_level debug + num_threads 2 + buffer_chunk_limit 4m + buffer_type file + buffer_path %STATE_DIR_WS%/out_oms_kubeservices*.buffer + buffer_queue_limit 20 + buffer_queue_full_action drop_oldest_chunk + flush_interval 20s + retry_limit 10 + retry_wait 5s + max_retry_wait 5m + + + + type out_oms + log_level debug + num_threads 2 + buffer_chunk_limit 4m + buffer_type file + buffer_path %STATE_DIR_WS%/state/out_oms_kubenodes*.buffer + buffer_queue_limit 20 + buffer_queue_full_action drop_oldest_chunk + flush_interval 20s + retry_limit 10 + retry_wait 5s + max_retry_wait 5m + + + + type out_oms + log_level debug + num_threads 3 + buffer_chunk_limit 4m + buffer_type file + buffer_path %STATE_DIR_WS%/out_oms_containernodeinventory*.buffer + buffer_queue_limit 20 + flush_interval 20s + retry_limit 10 + retry_wait 5s + max_retry_wait 5m + + + + type out_oms + log_level debug + num_threads 2 + buffer_chunk_limit 4m + buffer_type file + buffer_path %STATE_DIR_WS%/out_oms_kubeperf*.buffer + buffer_queue_limit 20 + buffer_queue_full_action drop_oldest_chunk + flush_interval 20s + retry_limit 10 + retry_wait 5s + max_retry_wait 5m + + + + type out_mdm + log_level debug + num_threads 5 + buffer_chunk_limit 4m + buffer_type file + buffer_path %STATE_DIR_WS%/out_mdm_*.buffer + buffer_queue_limit 20 + buffer_queue_full_action drop_oldest_chunk + flush_interval 20s + retry_limit 10 + retry_wait 5s + max_retry_wait 5m + retry_mdm_post_wait_minutes 30 + + + + type out_oms + log_level debug + num_threads 5 + buffer_chunk_limit 4m + buffer_type file + buffer_path %STATE_DIR_WS%/out_oms_api_wincadvisorperf*.buffer + buffer_queue_limit 20 + buffer_queue_full_action drop_oldest_chunk + flush_interval 20s + retry_limit 10 + retry_wait 5s + max_retry_wait 5m + + + + type out_mdm + log_level debug + num_threads 5 + buffer_chunk_limit 4m + buffer_type file + buffer_path %STATE_DIR_WS%/out_mdm_cdvisorperf*.buffer + buffer_queue_limit 20 + buffer_queue_full_action drop_oldest_chunk + flush_interval 20s + retry_limit 10 + retry_wait 5s + max_retry_wait 5m + retry_mdm_post_wait_minutes 30 + + + + type out_oms + log_level debug + num_threads 5 + buffer_chunk_limit 4m + buffer_type file + buffer_path %STATE_DIR_WS%/out_oms_kubehealth*.buffer + buffer_queue_limit 20 + buffer_queue_full_action drop_oldest_chunk + flush_interval 20s + retry_limit 10 + retry_wait 5s + max_retry_wait 5m + + + + type out_oms + log_level debug + num_threads 5 + buffer_chunk_limit 4m + buffer_type file + buffer_path %STATE_DIR_WS%/out_oms_insightsmetrics*.buffer + buffer_queue_limit 20 + buffer_queue_full_action drop_oldest_chunk + flush_interval 20s + retry_limit 10 + retry_wait 5s + max_retry_wait 5m + + +metadata: + name: omsagent-rs-config + namespace: kube-system +--- +apiVersion: v1 +kind: Secret +metadata: + name: omsagent-secret + namespace: kube-system +type: Opaque +data: + #BASE64 ENCODED (Both WSID & KEY) INSIDE DOUBLE QUOTE ("") + WSID: "NDAzMWRlYjctNzliZS00NTgxLThhMzAtNDdiY2FhMzU3YmE2Cg==" + KEY: "WUZQZG9FR2ttYlQzVkJwWm85VWkyUlJLR2ErVkVETE9sQWJTMTdzOEVyVXpHejZPWER2TVc0bVhoYzlPb0lVZEc2OTdjSitFbDBmVlRyT0x3SWpFVVE9PQo=" +--- +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: omsagent + namespace: kube-system + labels: + component: oms-agent + tier: node +spec: + updateStrategy: + type: RollingUpdate + selector: + matchLabels: + component: oms-agent + tier: node + template: + metadata: + labels: + component: oms-agent + tier: node + annotations: + agentVersion: "1.10.0.1" + dockerProviderVersion: "16.0.0-0" + schema-versions: "v1" + spec: + serviceAccountName: omsagent + dnsConfig: + options: + - name: ndots + value: "3" + containers: + - name: omsagent + image: "mcr.microsoft.com/azuremonitor/containerinsights/cidev:khushi-test-70" + imagePullPolicy: Always + resources: + limits: + cpu: 500m + memory: 600Mi + requests: + cpu: 75m + memory: 225Mi + env: + # azure devops pipeline uses AKS_RESOURCE_ID and AKS_REGION hence ensure to uncomment these + - name: AKS_RESOURCE_ID + value: "/subscriptions/b4b0cbab-64dc-42bc-9d95-ea7c8d0cf188/resourcegroups/khushi-watch-test/providers/Microsoft.ContainerService/managedClusters/khushi-watch-test" + - name: AKS_REGION + value: "West US 2" + # this used for e2e test and setting this just emits some additional log statements which used for the e2e tests + - name: ISTEST + value: "true" + - name: USEMMAP + value: "true" + #Uncomment below two lines for ACS clusters and set the cluster names manually. Also comment out the above two lines for ACS clusters + #- name: ACS_RESOURCE_NAME + # value: "my_acs_cluster_name" + - name: CONTROLLER_TYPE + value: "DaemonSet" + - name: NODE_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP + # Update this with the user assigned msi client id for omsagent + - name: USER_ASSIGNED_IDENTITY_CLIENT_ID + value: "" + - name: AZMON_CONTAINERLOGS_ONEAGENT_REGIONS + value: "koreacentral,norwayeast,eastus2" + securityContext: + privileged: true + ports: + - containerPort: 25225 + protocol: TCP + - containerPort: 25224 + protocol: UDP + volumeMounts: + - mountPath: /hostfs + name: host-root + readOnly: true + - mountPath: /var/run/host + name: docker-sock + - mountPath: /var/log + name: host-log + - mountPath: /var/lib/docker/containers + name: containerlog-path + readOnly: true + - mountPath: /mnt/docker + name: containerlog-path-2 + readOnly: true + - mountPath: /mnt/containers + name: containerlog-path-3 + readOnly: true + - mountPath: /etc/kubernetes/host + name: azure-json-path + - mountPath: /etc/omsagent-secret + name: omsagent-secret + readOnly: true + - mountPath: /etc/config/settings + name: settings-vol-config + readOnly: true + - mountPath: /etc/config/settings/adx + name: omsagent-adx-secret + readOnly: true + livenessProbe: + exec: + command: + - /bin/bash + - -c + - /opt/livenessprobe.sh + initialDelaySeconds: 60 + periodSeconds: 60 + timeoutSeconds: 15 +#Only in sidecar scraping mode + - name: omsagent-prometheus + image: "mcr.microsoft.com/azuremonitor/containerinsights/cidev:khushi-test-70" + imagePullPolicy: Always + resources: + limits: + cpu: 500m + memory: 1Gi + requests: + cpu: 75m + memory: 225Mi + env: + # azure devops pipeline uses AKS_RESOURCE_ID and AKS_REGION hence ensure to uncomment these + - name: AKS_RESOURCE_ID + value: "/subscriptions/b4b0cbab-64dc-42bc-9d95-ea7c8d0cf188/resourcegroups/khushi-watch-test/providers/Microsoft.ContainerService/managedClusters/khushi-watch-test" + - name: AKS_REGION + value: "West US 2" + #Uncomment below two lines for ACS clusters and set the cluster names manually. Also comment out the above two lines for ACS clusters + #- name: ACS_RESOURCE_NAME + # value: "my_acs_cluster_name" + - name: CONTAINER_TYPE + value: "PrometheusSidecar" + - name: CONTROLLER_TYPE + value: "DaemonSet" + - name: NODE_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP + # Update this with the user assigned msi client id for omsagent + - name: USER_ASSIGNED_IDENTITY_CLIENT_ID + value: "" + securityContext: + privileged: true + volumeMounts: + - mountPath: /etc/kubernetes/host + name: azure-json-path + - mountPath: /etc/omsagent-secret + name: omsagent-secret + readOnly: true + - mountPath: /etc/config/settings + name: settings-vol-config + readOnly: true + - mountPath: /etc/config/osm-settings + name: osm-settings-vol-config + readOnly: true + livenessProbe: + exec: + command: + - /bin/bash + - -c + - /opt/livenessprobe.sh + initialDelaySeconds: 60 + periodSeconds: 60 + timeoutSeconds: 15 + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - labelSelector: + matchExpressions: + # kubernetes.io/os label doesnt exist in k8s versions < 1.14 so make sure to choose label based on k8s version in aks yaml + - key: kubernetes.io/os + operator: In + values: + - linux + - key: type + operator: NotIn + values: + - virtual-kubelet + # Tolerate a NoSchedule taint on master that ACS Engine sets. + tolerations: + - operator: "Exists" + effect: "NoSchedule" + - operator: "Exists" + effect: "NoExecute" + - operator: "Exists" + effect: "PreferNoSchedule" + volumes: + - name: host-root + hostPath: + path: / + - name: docker-sock + hostPath: + path: /var/run + - name: container-hostname + hostPath: + path: /etc/hostname + - name: host-log + hostPath: + path: /var/log + - name: containerlog-path + hostPath: + path: /var/lib/docker/containers + - name: containerlog-path-2 + hostPath: + path: /mnt/docker + - name: containerlog-path-3 + hostPath: + path: /mnt/containers + - name: azure-json-path + hostPath: + path: /etc/kubernetes + - name: omsagent-secret + secret: + secretName: omsagent-secret + - name: settings-vol-config + configMap: + name: container-azm-ms-agentconfig + optional: true + - name: omsagent-adx-secret + secret: + secretName: omsagent-adx-secret + optional: true + - name: osm-settings-vol-config + configMap: + name: container-azm-ms-osmconfig + optional: true +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: omsagent-rs + namespace: kube-system + labels: + component: oms-agent + tier: node +spec: + replicas: 1 + selector: + matchLabels: + rsName: "omsagent-rs" + strategy: + type: RollingUpdate + template: + metadata: + labels: + rsName: "omsagent-rs" + annotations: + agentVersion: "1.10.0.1" + dockerProviderVersion: "16.0.0-0" + schema-versions: "v1" + spec: + serviceAccountName: omsagent + containers: + - name: omsagent + image: "mcr.microsoft.com/azuremonitor/containerinsights/cidev:khushi-test-70" + imagePullPolicy: Always + resources: + limits: + cpu: 1 + memory: 1Gi + requests: + cpu: 150m + memory: 250Mi + env: + - name: AKS_RESOURCE_ID + value: "/subscriptions/b4b0cbab-64dc-42bc-9d95-ea7c8d0cf188/resourcegroups/khushi-watch-test/providers/Microsoft.ContainerService/managedClusters/khushi-watch-test" + - name: AKS_REGION + value: "West US 2" + # this used for e2e test and setting this just emits some additional log statements which used for the e2e tests + - name: ISTEST + value: "true" + - name: USEMMAP + value: "true" + # Uncomment below two lines for ACS clusters and set the cluster names manually. Also comment out the above two lines for ACS clusters + #- name: ACS_RESOURCE_NAME + # value: "my_acs_cluster_name" + - name: CONTROLLER_TYPE + value: "ReplicaSet" + - name: NODE_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP + # Update this with the user assigned msi client id for omsagent + - name: USER_ASSIGNED_IDENTITY_CLIENT_ID + value: "" + # Add the below environment variable to true only in sidecar enabled regions, else set it to false + - name: SIDECAR_SCRAPING_ENABLED + value: "true" + securityContext: + privileged: true + ports: + - containerPort: 25225 + protocol: TCP + - containerPort: 25224 + protocol: UDP + - containerPort: 25227 + protocol: TCP + name: in-rs-tcp + volumeMounts: + - mountPath: /var/run/host + name: docker-sock + - mountPath: /var/log + name: host-log + - mountPath: /etc/kubernetes/host + name: azure-json-path + - mountPath: /etc/omsagent-secret + name: omsagent-secret + readOnly: true + - mountPath: /etc/config + name: omsagent-rs-config + - mountPath: /etc/config/settings + name: settings-vol-config + readOnly: true + - mountPath: /etc/config/settings/adx + name: omsagent-adx-secret + - mountPath: /etc/config/osm-settings + name: osm-settings-vol-config + readOnly: true + # livenessProbe: + # exec: + # command: + # - /bin/bash + # - -c + # - /opt/livenessprobe.sh + # initialDelaySeconds: 60 + # periodSeconds: 60 + # timeoutSeconds: 15 + affinity: + nodeAffinity: + # affinity to schedule on to ephemeral os node if its available + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 1 + preference: + matchExpressions: + - key: storageprofile + operator: NotIn + values: + - managed + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - labelSelector: + matchExpressions: + - key: beta.kubernetes.io/os + operator: In + values: + - linux + - key: watch-test + operator: In + values: + - watch + - key: type + operator: NotIn + values: + - virtual-kubelet + # The following label selector is removed for AKS, this is only required for non AKS + - key: kubernetes.io/role + operator: NotIn + values: + - master + # The following tolerations are removed for AKS, this is only required for non AKS + tolerations: + - operator: "Exists" + effect: "NoSchedule" + - operator: "Exists" + effect: "NoExecute" + - operator: "Exists" + effect: "PreferNoSchedule" + volumes: + - name: docker-sock + hostPath: + path: /var/run + - name: container-hostname + hostPath: + path: /etc/hostname + - name: host-log + hostPath: + path: /var/log + - name: azure-json-path + hostPath: + path: /etc/kubernetes + - name: omsagent-secret + secret: + secretName: omsagent-secret + - name: omsagent-rs-config + configMap: + name: omsagent-rs-config + - name: settings-vol-config + configMap: + name: container-azm-ms-agentconfig + optional: true + - name: omsagent-adx-secret + secret: + secretName: omsagent-adx-secret + optional: true + - name: osm-settings-vol-config + configMap: + name: container-azm-ms-osmconfig + optional: true +--- + apiVersion: apps/v1 + kind: DaemonSet + metadata: + name: omsagent-win + namespace: kube-system + labels: + component: oms-agent-win + tier: node-win + spec: + updateStrategy: + type: RollingUpdate + selector: + matchLabels: + component: oms-agent-win + tier: node-win + template: + metadata: + labels: + component: oms-agent-win + tier: node-win + annotations: + agentVersion: "1.10.0.1" + dockerProviderVersion: "16.0.0-0" + schema-versions: "v1" + spec: + serviceAccountName: omsagent + dnsConfig: + options: + - name: ndots + value: "3" + containers: + - name: omsagent-win + image: "mcr.microsoft.com/azuremonitor/containerinsights/ciprod:win-ciprod06112021" + imagePullPolicy: IfNotPresent + resources: + limits: + cpu: 200m + memory: 600Mi + env: + # azure devops pipeline uses AKS_RESOURCE_ID and AKS_REGION hence ensure to uncomment these + - name: AKS_RESOURCE_ID + value: "/subscriptions/b4b0cbab-64dc-42bc-9d95-ea7c8d0cf188/resourcegroups/khushi-watch-test/providers/Microsoft.ContainerService/managedClusters/khushi-watch-test" + - name: AKS_REGION + value: "West US 2" + #- name: ACS_RESOURCE_NAME + # value: "my_acs_cluster_name" + - name: CONTROLLER_TYPE + value: "DaemonSet" + - name: HOSTNAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: PODNAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: NODE_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP + - name: SIDECAR_SCRAPING_ENABLED + value: "true" + # Update this with the user assigned msi client id for omsagent + - name: USER_ASSIGNED_IDENTITY_CLIENT_ID + value: "" + # Add this only for clouds that require cert bootstrapping + - name: REQUIRES_CERT_BOOTSTRAP + value: "true" + volumeMounts: + - mountPath: C:\ProgramData\docker\containers + name: docker-windows-containers + readOnly: true + - mountPath: C:\var #Read + Write access on this for position file + name: docker-windows-kuberenetes-container-logs + - mountPath: C:\etc\config\settings + name: settings-vol-config + readOnly: true + - mountPath: C:\etc\omsagent-secret + name: omsagent-secret + readOnly: true + - mountPath: C:\etc\config\adx + name: omsagent-adx-secret + readOnly: true + # Need to mount this only for airgapped clouds - Commenting this since it wont exist in non airgapped clouds + # - mountPath: C:\ca + # name: ca-certs + # readOnly: true + - mountPath: C:\etc\kubernetes\host + name: azure-json-path + readOnly: true + livenessProbe: + exec: + command: + - cmd + - /c + - C:\opt\omsagentwindows\scripts\cmd\livenessProbe.cmd + periodSeconds: 60 + initialDelaySeconds: 180 + timeoutSeconds: 15 + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: beta.kubernetes.io/os + operator: In + values: + - windows + tolerations: + - key: "CriticalAddonsOnly" + operator: "Exists" + - operator: "Exists" + effect: NoExecute + - operator: "Exists" + effect: NoSchedule + - operator: "Exists" + effect: PreferNoSchedule + volumes: + - name: docker-windows-kuberenetes-container-logs + hostPath: + path: C:\var + - name: azure-json-path + hostPath: + path: C:\k + # Need to mount this only for airgapped clouds - Commenting this since it wont exist in non airgapped clouds + #- name: ca-certs + # hostPath: + # path: C:\ca + - name: docker-windows-containers + hostPath: + path: C:\ProgramData\docker\containers + type: DirectoryOrCreate + - name: settings-vol-config + configMap: + name: container-azm-ms-agentconfig + optional: true + - name: omsagent-secret + secret: + secretName: omsagent-secret + - name: omsagent-adx-secret + secret: + secretName: omsagent-adx-secret + optional: true +--- +kind: Service +apiVersion: v1 +metadata: + name: healthmodel-replicaset-service + namespace: kube-system +spec: + selector: + rsName: "omsagent-rs" + ports: + - protocol: TCP + port: 25227 + targetPort: in-rs-tcp +--- +# this is for versions >=1.19, for versions <1.19 we continue to use v1beta1 +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: healthstates.azmon.container.insights + namespace: kube-system +spec: + group: azmon.container.insights + versions: + - name: v1 + served: true + storage: true + schema: + openAPIV3Schema: + type: object + properties: + state: + type: string + scope: Namespaced + names: + plural: healthstates + kind: HealthState From 3f6869ef5656dbd3c529205c2b267df243cd424e Mon Sep 17 00:00:00 2001 From: Khushi Chaudhari Date: Tue, 24 Aug 2021 17:29:21 -0700 Subject: [PATCH 72/76] temp changes 2 --- source/plugins/ruby/in_kube_podinventory.rb | 31 +++++++++++++++------ 1 file changed, 22 insertions(+), 9 deletions(-) diff --git a/source/plugins/ruby/in_kube_podinventory.rb b/source/plugins/ruby/in_kube_podinventory.rb index 44036a1d1..40c0dfbd2 100644 --- a/source/plugins/ruby/in_kube_podinventory.rb +++ b/source/plugins/ruby/in_kube_podinventory.rb @@ -104,6 +104,7 @@ def start @watchthread = Thread.new(&method(:watch)) @runthread = Thread.new(&method(:run_periodic)) @@podTelemetryTimeTracker = DateTime.now.to_time.to_i + @@WatcherTimeTracker = DateTime.now.to_time.to_i end end @@ -139,7 +140,7 @@ def populate_hash(podInventory) end end - def getNoticeRecord(notice) + def getNoticeRecord(notice, serviceRecords) # Helper function that extracts necessary fields from notice JSON record = {} item = notice["object"] @@ -152,6 +153,7 @@ def getNoticeRecord(notice) podNameSpace = item["metadata"]["namespace"] #TODO: change uid later, handle case of horizontal scaling of pods but no controller (explained in getPodUid in KubenetesApiClient) podUid = item["metadata"]["uid"] + # podUid = KubenetesApiClient.getPodUid(podNameSpace, item["metadata"]) nodeName = "" # For unscheduled (non-started) pods nodeName does NOT exist @@ -200,7 +202,7 @@ def getNoticeRecord(notice) record["ClusterId"] = KubernetesApiClient.getClusterId record["ClusterName"] = KubernetesApiClient.getClusterName #TODO: make a call to getServiceNameFromLabels -- need to pass in serviceRecords for this - record["ServiceName"] = "" + record["ServiceName"] = getServiceNameFromLabels(item["metadata"]["namespace"], item["metadata"]["labels"], serviceRecords) if !item["metadata"]["ownerReferences"].nil? record["ControllerKind"] = item["metadata"]["ownerReferences"][0]["kind"] @@ -238,13 +240,15 @@ def getNoticeRecord(notice) def watch loop do enumerate + serviceRecords = @serviceRecords begin - @Watcher = @KubernetesWatchClient.watch_pods(resource_version: @collection_version, timeoutSeconds: 380, as: :parsed) + @Watcher = @KubernetesWatchClient.watch_pods(resource_version: @collection_version, timeoutSeconds: 300, allowWatchBookmarks: true, as: :parsed) @Watcher.each do |notice| + $log.info("in_kube_podinventory::watch : inside watch pods! collection version: #{@collection_version}.") if !notice.nil? && !notice.empty? item = notice["object"] # Construct record with necessary fields (same fields as getPodInventoryRecords) - record = getNoticeRecord(notice) + record = getNoticeRecord(notice, serviceRecords) @mutex.synchronize { @noticeHash[item["metadata"]["uid"]] = record } @@ -562,6 +566,16 @@ def parse_and_emit_merge_updates(podInventoryRecords) emittedPodCount = 0 begin #begin block start + timeDifference = (DateTime.now.to_time.to_i - @@WatcherTimeTracker).abs + timeDifferenceInMinutes = timeDifference / 60 + if (timeDifferenceInMinutes >= 5) + $log.info("parse_and_emit_merge_updates::resetting watcher to handle api server timeout :#{Time.now.utc.iso8601}") + @@WatcherTimeTracker = DateTime.now.to_time.to_i + if !@Watcher.nil? + @Watcher.finish + end + end + # Getting windows nodes from kubeapi winNodes = KubernetesApiClient.getWindowsNodesArray podInventoryRecords.each do |uid, record| @@ -614,11 +628,8 @@ def merge_updates if @podInventoryHash[uid].nil? @podInventoryHash[uid] = record else - # TODO: will need to modify other fields later - $log.info("in_kube_podinventory::merge_updates : modify case where it is only a modification. old status: #{@podInventoryHash[uid]["PodStatus"]}. new status: #{record["PodStatus"]}") - val = @podInventoryHash[uid] - val["PodStatus"] = record["PodStatus"] - @podInventoryHash[uid] = val + $log.info("in_kube_podinventory::merge_updates : pure modify case") + @podInventoryHash[uid] = record end when "DELETED" if @podInventoryHash.key?(uid) @@ -626,6 +637,8 @@ def merge_updates else $log.info("in_kube_podinventory::merge_updates: key did not exist in hash so unable to delete.") end + when "BOOKMARK" + $log.info("in_kube_podinventory::merge_updates: received a BOOKMARK event.") else $log.info("in_kube_podinventory::merge_updates: something went wrong and didn't enter any cases for switch, notice type was #{record["NoticeType"]}") end From 22e3281bdabe6a7366a5056ab97ca857805ec636 Mon Sep 17 00:00:00 2001 From: Khushi Chaudhari Date: Wed, 25 Aug 2021 17:10:36 -0700 Subject: [PATCH 73/76] code clean up using getPodInventoryRecords --- source/plugins/ruby/in_kube_podinventory.rb | 220 +++++++++++--------- 1 file changed, 116 insertions(+), 104 deletions(-) diff --git a/source/plugins/ruby/in_kube_podinventory.rb b/source/plugins/ruby/in_kube_podinventory.rb index 40c0dfbd2..43bb5a929 100644 --- a/source/plugins/ruby/in_kube_podinventory.rb +++ b/source/plugins/ruby/in_kube_podinventory.rb @@ -120,7 +120,7 @@ def shutdown end end - def populate_hash(podInventory) + def populate_podinventory_cache(podInventory) batchTime = Time.now.utc.iso8601 serviceRecords = @serviceRecords begin @@ -135,120 +135,127 @@ def populate_hash(podInventory) end end rescue => exception - $log.info("in_kube_podinventory::populate_hash : populating podInventoryHash failed.") + $log.info("in_kube_podinventory::populate_podinventory_cache : populating podInventoryHash failed.") $log.debug_backtrace(exception.backtrace) end end - def getNoticeRecord(notice, serviceRecords) - # Helper function that extracts necessary fields from notice JSON - record = {} - item = notice["object"] - #TODO: check assumption that batch time can be current time (CollectionTime) - batchTime = Time.now.utc.iso8601 + # def getNoticeRecord(notice, serviceRecords) + # # Helper function that extracts necessary fields from notice JSON + # record = {} + # item = notice["object"] + # #TODO: check assumption that batch time can be current time (CollectionTime) + # batchTime = Time.now.utc.iso8601 - begin - record["CollectionTime"] = batchTime - record["Name"] = item["metadata"]["name"] - podNameSpace = item["metadata"]["namespace"] - #TODO: change uid later, handle case of horizontal scaling of pods but no controller (explained in getPodUid in KubenetesApiClient) - podUid = item["metadata"]["uid"] - # podUid = KubenetesApiClient.getPodUid(podNameSpace, item["metadata"]) - - nodeName = "" - # For unscheduled (non-started) pods nodeName does NOT exist - if !item["spec"]["nodeName"].nil? - nodeName = item["spec"]["nodeName"] - end - - record["PodUid"] = podUid - record["PodLabel"] = [item["metadata"]["labels"]] - record["Namespace"] = podNameSpace - record["PodCreationTimeStamp"] = item["metadata"]["creationTimestamp"] - - if !item["status"]["startTime"].nil? - record["PodStartTime"] = item["status"]["startTime"] - else - record["PodStartTime"] = "" - end - - #podStatus - # NodeLost scenario -- pod(s) in the lost node is still being reported as running - podReadyCondition = true - if !item["status"]["reason"].nil? && item["status"]["reason"] == "NodeLost" && !item["status"]["conditions"].nil? - item["status"]["conditions"].each do |condition| - if condition["type"] == "Ready" && condition["status"] == "False" - podReadyCondition = false - break - end - end - end - if podReadyCondition == false - record["PodStatus"] = "Unknown" - elsif !item["metadata"]["deletionTimestamp"].nil? && !item["metadata"]["deletionTimestamp"].empty? - record["PodStatus"] = Constants::POD_STATUS_TERMINATING - else - record["PodStatus"] = item["status"]["phase"] - end - - # For unscheduled (non-started) pods podIP does NOT exist - if !item["status"]["podIP"].nil? - record["PodIp"] = item["status"]["podIP"] - else - record["PodIp"] = "" - end - - record["Computer"] = nodeName - record["ClusterId"] = KubernetesApiClient.getClusterId - record["ClusterName"] = KubernetesApiClient.getClusterName - #TODO: make a call to getServiceNameFromLabels -- need to pass in serviceRecords for this - record["ServiceName"] = getServiceNameFromLabels(item["metadata"]["namespace"], item["metadata"]["labels"], serviceRecords) - - if !item["metadata"]["ownerReferences"].nil? - record["ControllerKind"] = item["metadata"]["ownerReferences"][0]["kind"] - record["ControllerName"] = item["metadata"]["ownerReferences"][0]["name"] - # @controllerSet.add(record["ControllerKind"] + record["ControllerName"]) - # Adding controller kind to telemetry ro information about customer workload - # if (@controllerData[record["ControllerKind"]].nil?) - # @controllerData[record["ControllerKind"]] = 1 - # else - # controllerValue = @controllerData[record["ControllerKind"]] - # @controllerData[record["ControllerKind"]] += 1 - # end - end - - podRestartCount = 0 - record["PodRestartCount"] = 0 - - #TODO: popular real values for container fields - record["ContainerID"] = "" - record["ContainerName"] = "" - record["ContainerRestartCount"] = 0 - record["ContainerRestartReason"] = "" - record["ContainerStatus"] = "" - record["ContainerCreationTimeStamp"] = Time.now.utc.iso8601 - record["ContainerLastStatus"] = Hash.new - - record["NoticeType"] = notice["type"] + # begin + # record["CollectionTime"] = batchTime + # record["Name"] = item["metadata"]["name"] + # podNameSpace = item["metadata"]["namespace"] + # #TODO: change uid later, handle case of horizontal scaling of pods but no controller (explained in getPodUid in KubenetesApiClient) + # podUid = item["metadata"]["uid"] + # # podUid = KubenetesApiClient.getPodUid(podNameSpace, item["metadata"]) + + # nodeName = "" + # # For unscheduled (non-started) pods nodeName does NOT exist + # if !item["spec"]["nodeName"].nil? + # nodeName = item["spec"]["nodeName"] + # end + + # record["PodUid"] = podUid + # record["PodLabel"] = [item["metadata"]["labels"]] + # record["Namespace"] = podNameSpace + # record["PodCreationTimeStamp"] = item["metadata"]["creationTimestamp"] + + # if !item["status"]["startTime"].nil? + # record["PodStartTime"] = item["status"]["startTime"] + # else + # record["PodStartTime"] = "" + # end + + # #podStatus + # # NodeLost scenario -- pod(s) in the lost node is still being reported as running + # podReadyCondition = true + # if !item["status"]["reason"].nil? && item["status"]["reason"] == "NodeLost" && !item["status"]["conditions"].nil? + # item["status"]["conditions"].each do |condition| + # if condition["type"] == "Ready" && condition["status"] == "False" + # podReadyCondition = false + # break + # end + # end + # end + # if podReadyCondition == false + # record["PodStatus"] = "Unknown" + # elsif !item["metadata"]["deletionTimestamp"].nil? && !item["metadata"]["deletionTimestamp"].empty? + # record["PodStatus"] = Constants::POD_STATUS_TERMINATING + # else + # record["PodStatus"] = item["status"]["phase"] + # end + # # For unscheduled (non-started) pods podIP does NOT exist + # if !item["status"]["podIP"].nil? + # record["PodIp"] = item["status"]["podIP"] + # else + # record["PodIp"] = "" + # end + + # record["Computer"] = nodeName + # record["ClusterId"] = KubernetesApiClient.getClusterId + # record["ClusterName"] = KubernetesApiClient.getClusterName + # #TODO: make a call to getServiceNameFromLabels -- need to pass in serviceRecords for this + # record["ServiceName"] = getServiceNameFromLabels(item["metadata"]["namespace"], item["metadata"]["labels"], serviceRecords) + + # if !item["metadata"]["ownerReferences"].nil? + # record["ControllerKind"] = item["metadata"]["ownerReferences"][0]["kind"] + # record["ControllerName"] = item["metadata"]["ownerReferences"][0]["name"] + # # @controllerSet.add(record["ControllerKind"] + record["ControllerName"]) + # # Adding controller kind to telemetry ro information about customer workload + # # if (@controllerData[record["ControllerKind"]].nil?) + # # @controllerData[record["ControllerKind"]] = 1 + # # else + # # controllerValue = @controllerData[record["ControllerKind"]] + # # @controllerData[record["ControllerKind"]] += 1 + # # end + # end + + # podRestartCount = 0 + # record["PodRestartCount"] = 0 + + # #TODO: popular real values for container fields + # record["ContainerID"] = "" + # record["ContainerName"] = "" + # record["ContainerRestartCount"] = 0 + # record["ContainerRestartReason"] = "" + # record["ContainerStatus"] = "" + # record["ContainerCreationTimeStamp"] = Time.now.utc.iso8601 + # record["ContainerLastStatus"] = Hash.new + + # record["NoticeType"] = notice["type"] - rescue => exception - puts "getNoticeRecord failed: #{exception.backtrace}" - end - return record - end + # rescue => exception + # puts "getNoticeRecord failed: #{exception.backtrace}" + # end + # return record + # end def watch loop do enumerate - serviceRecords = @serviceRecords + # serviceRecords = @serviceRecords begin @Watcher = @KubernetesWatchClient.watch_pods(resource_version: @collection_version, timeoutSeconds: 300, allowWatchBookmarks: true, as: :parsed) @Watcher.each do |notice| $log.info("in_kube_podinventory::watch : inside watch pods! collection version: #{@collection_version}.") - if !notice.nil? && !notice.empty? + # if type error then break -- look at sample code + if notice["type"] == "ERROR" + $log.info("in_kube_podinventory::watch : notice type was error. restarting watch.") + break + end + if !notice.nil? && !notice.empty? item = notice["object"] + batchTime = Time.now.utc.iso8601 # Construct record with necessary fields (same fields as getPodInventoryRecords) - record = getNoticeRecord(notice, serviceRecords) + # record = getNoticeRecord(notice, serviceRecords) + record = getPodInventoryRecords(item, @serviceRecords, batchTime) + # $log.info("in_kube_podinventory::watch : record looks like: #{record}") @mutex.synchronize { @noticeHash[item["metadata"]["uid"]] = record } @@ -304,14 +311,16 @@ def enumerate(podList = nil) continuationToken = nil $log.info("in_kube_podinventory::enumerate : Getting pods from Kube API @ #{Time.now.utc.iso8601}") continuationToken, podInventory = KubernetesApiClient.getResourcesAndContinuationToken("pods?limit=#{@PODS_CHUNK_SIZE}") - @collection_version = podInventory["metadata"]["resourceVersion"] + if !podInventory["metadata"].nil? && !podInventory["metadata"].empty? && !podInventory["metadata"]["resourceVersion"].nil? && !podInventory["metadata"]["resourceVersion"].empty? + @collection_version = podInventory["metadata"]["resourceVersion"] + end $log.info("in_kube_podinventory::enumerate : Received collection version: #{@collection_version}") $log.info("in_kube_podinventory::enumerate : Done getting pods from Kube API @ #{Time.now.utc.iso8601}") podsAPIChunkEndTime = (Time.now.to_f * 1000).to_i @podsAPIE2ELatencyMs = (podsAPIChunkEndTime - podsAPIChunkStartTime) if (!podInventory.nil? && !podInventory.empty? && podInventory.key?("items") && !podInventory["items"].nil? && !podInventory["items"].empty?) $log.info("in_kube_podinventory::enumerate : number of pod items :#{podInventory["items"].length} from Kube API @ #{Time.now.utc.iso8601}") - populate_hash(podInventory) + populate_podinventory_cache(podInventory) parse_and_emit_records(podInventory, @serviceRecords, continuationToken, batchTime) else $log.warn "in_kube_podinventory::enumerate:Received empty podInventory" @@ -323,13 +332,15 @@ def enumerate(podList = nil) podsAPIChunkStartTime = (Time.now.to_f * 1000).to_i continuationToken, podInventory = KubernetesApiClient.getResourcesAndContinuationToken("pods?limit=#{@PODS_CHUNK_SIZE}&continue=#{continuationToken}") # need to add collection resource version here - @collection_version = podInventory["metadata"]["resourceVersion"] + if !podInventory["metadata"].nil? && !podInventory["metadata"].empty? && !podInventory["metadata"]["resourceVersion"].nil? && !podInventory["metadata"]["resourceVersion"].empty? + @collection_version = podInventory["metadata"]["resourceVersion"] + end $log.info("in_kube_podinventory::enumerate : continuation token was not null. received collection version: #{@collection_version}") podsAPIChunkEndTime = (Time.now.to_f * 1000).to_i @podsAPIE2ELatencyMs = @podsAPIE2ELatencyMs + (podsAPIChunkEndTime - podsAPIChunkStartTime) if (!podInventory.nil? && !podInventory.empty? && podInventory.key?("items") && !podInventory["items"].nil? && !podInventory["items"].empty?) $log.info("in_kube_podinventory::enumerate : number of pod items :#{podInventory["items"].length} from Kube API @ #{Time.now.utc.iso8601}") - populate_hash(podInventory) + populate_podinventory_cache(podInventory) parse_and_emit_records(podInventory, @serviceRecords, continuationToken, batchTime) else $log.warn "in_kube_podinventory::enumerate:Received empty podInventory" @@ -339,7 +350,8 @@ def enumerate(podList = nil) @podInventoryE2EProcessingLatencyMs = ((Time.now.to_f * 1000).to_i - podInventoryStartTime) # Setting these to nil so that we dont hold memory until GC kicks in podInventory = nil - @serviceRecords = nil + # TODO: commenting next line for watch + # @serviceRecords = nil # Adding telemetry to send pod telemetry every 5 minutes timeDifference = (DateTime.now.to_time.to_i - @@podTelemetryTimeTracker).abs From 3b01db79eac52e3b018f6516c3d9a5c4a0055001 Mon Sep 17 00:00:00 2001 From: Khushi Chaudhari Date: Thu, 26 Aug 2021 10:36:50 -0700 Subject: [PATCH 74/76] removing bookmarks and addressing noticetype bug --- source/plugins/ruby/in_kube_podinventory.rb | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/source/plugins/ruby/in_kube_podinventory.rb b/source/plugins/ruby/in_kube_podinventory.rb index 43bb5a929..b70c32d92 100644 --- a/source/plugins/ruby/in_kube_podinventory.rb +++ b/source/plugins/ruby/in_kube_podinventory.rb @@ -239,23 +239,24 @@ def populate_podinventory_cache(podInventory) def watch loop do enumerate - # serviceRecords = @serviceRecords + serviceRecords = @serviceRecords begin - @Watcher = @KubernetesWatchClient.watch_pods(resource_version: @collection_version, timeoutSeconds: 300, allowWatchBookmarks: true, as: :parsed) + @Watcher = @KubernetesWatchClient.watch_pods(resource_version: @collection_version, timeoutSeconds: 300, as: :parsed) @Watcher.each do |notice| $log.info("in_kube_podinventory::watch : inside watch pods! collection version: #{@collection_version}.") - # if type error then break -- look at sample code if notice["type"] == "ERROR" $log.info("in_kube_podinventory::watch : notice type was error. restarting watch.") break end if !notice.nil? && !notice.empty? + $log.info("in_kube_podinventory::watch : notice was not null and not empty.") item = notice["object"] batchTime = Time.now.utc.iso8601 # Construct record with necessary fields (same fields as getPodInventoryRecords) - # record = getNoticeRecord(notice, serviceRecords) - record = getPodInventoryRecords(item, @serviceRecords, batchTime) - # $log.info("in_kube_podinventory::watch : record looks like: #{record}") + records = getPodInventoryRecords(item, serviceRecords, batchTime) + record = records.first() + record["NoticeType"] = notice["type"] + $log.info("in_kube_podinventory::watch : record looks like: #{record}") @mutex.synchronize { @noticeHash[item["metadata"]["uid"]] = record } @@ -688,7 +689,7 @@ def run_periodic merge_updates $log.info("in_kube_podinventory::run_periodic.merge_updates.end #{Time.now.utc.iso8601}") rescue => errorStr - $log.warn "in_kube_podinventory::run_periodic: enumerate Failed to retrieve pod inventory: #{errorStr}" + $log.warn "in_kube_podinventory::run_periodic: merge_updates Failed to retrieve pod inventory: #{errorStr}" ApplicationInsightsUtility.sendExceptionTelemetry(errorStr) end end From f430ccf10b900d8e2212a7aae0ed7049f38299af Mon Sep 17 00:00:00 2001 From: Khushi Chaudhari Date: Thu, 26 Aug 2021 13:10:48 -0700 Subject: [PATCH 75/76] moved telemetry in enumerate to merge updates --- source/plugins/ruby/in_kube_podinventory.rb | 222 ++++++-------------- 1 file changed, 69 insertions(+), 153 deletions(-) diff --git a/source/plugins/ruby/in_kube_podinventory.rb b/source/plugins/ruby/in_kube_podinventory.rb index b70c32d92..55672cb57 100644 --- a/source/plugins/ruby/in_kube_podinventory.rb +++ b/source/plugins/ruby/in_kube_podinventory.rb @@ -140,102 +140,6 @@ def populate_podinventory_cache(podInventory) end end - # def getNoticeRecord(notice, serviceRecords) - # # Helper function that extracts necessary fields from notice JSON - # record = {} - # item = notice["object"] - # #TODO: check assumption that batch time can be current time (CollectionTime) - # batchTime = Time.now.utc.iso8601 - - # begin - # record["CollectionTime"] = batchTime - # record["Name"] = item["metadata"]["name"] - # podNameSpace = item["metadata"]["namespace"] - # #TODO: change uid later, handle case of horizontal scaling of pods but no controller (explained in getPodUid in KubenetesApiClient) - # podUid = item["metadata"]["uid"] - # # podUid = KubenetesApiClient.getPodUid(podNameSpace, item["metadata"]) - - # nodeName = "" - # # For unscheduled (non-started) pods nodeName does NOT exist - # if !item["spec"]["nodeName"].nil? - # nodeName = item["spec"]["nodeName"] - # end - - # record["PodUid"] = podUid - # record["PodLabel"] = [item["metadata"]["labels"]] - # record["Namespace"] = podNameSpace - # record["PodCreationTimeStamp"] = item["metadata"]["creationTimestamp"] - - # if !item["status"]["startTime"].nil? - # record["PodStartTime"] = item["status"]["startTime"] - # else - # record["PodStartTime"] = "" - # end - - # #podStatus - # # NodeLost scenario -- pod(s) in the lost node is still being reported as running - # podReadyCondition = true - # if !item["status"]["reason"].nil? && item["status"]["reason"] == "NodeLost" && !item["status"]["conditions"].nil? - # item["status"]["conditions"].each do |condition| - # if condition["type"] == "Ready" && condition["status"] == "False" - # podReadyCondition = false - # break - # end - # end - # end - # if podReadyCondition == false - # record["PodStatus"] = "Unknown" - # elsif !item["metadata"]["deletionTimestamp"].nil? && !item["metadata"]["deletionTimestamp"].empty? - # record["PodStatus"] = Constants::POD_STATUS_TERMINATING - # else - # record["PodStatus"] = item["status"]["phase"] - # end - # # For unscheduled (non-started) pods podIP does NOT exist - # if !item["status"]["podIP"].nil? - # record["PodIp"] = item["status"]["podIP"] - # else - # record["PodIp"] = "" - # end - - # record["Computer"] = nodeName - # record["ClusterId"] = KubernetesApiClient.getClusterId - # record["ClusterName"] = KubernetesApiClient.getClusterName - # #TODO: make a call to getServiceNameFromLabels -- need to pass in serviceRecords for this - # record["ServiceName"] = getServiceNameFromLabels(item["metadata"]["namespace"], item["metadata"]["labels"], serviceRecords) - - # if !item["metadata"]["ownerReferences"].nil? - # record["ControllerKind"] = item["metadata"]["ownerReferences"][0]["kind"] - # record["ControllerName"] = item["metadata"]["ownerReferences"][0]["name"] - # # @controllerSet.add(record["ControllerKind"] + record["ControllerName"]) - # # Adding controller kind to telemetry ro information about customer workload - # # if (@controllerData[record["ControllerKind"]].nil?) - # # @controllerData[record["ControllerKind"]] = 1 - # # else - # # controllerValue = @controllerData[record["ControllerKind"]] - # # @controllerData[record["ControllerKind"]] += 1 - # # end - # end - - # podRestartCount = 0 - # record["PodRestartCount"] = 0 - - # #TODO: popular real values for container fields - # record["ContainerID"] = "" - # record["ContainerName"] = "" - # record["ContainerRestartCount"] = 0 - # record["ContainerRestartReason"] = "" - # record["ContainerStatus"] = "" - # record["ContainerCreationTimeStamp"] = Time.now.utc.iso8601 - # record["ContainerLastStatus"] = Hash.new - - # record["NoticeType"] = notice["type"] - - # rescue => exception - # puts "getNoticeRecord failed: #{exception.backtrace}" - # end - # return record - # end - def watch loop do enumerate @@ -353,34 +257,6 @@ def enumerate(podList = nil) podInventory = nil # TODO: commenting next line for watch # @serviceRecords = nil - - # Adding telemetry to send pod telemetry every 5 minutes - timeDifference = (DateTime.now.to_time.to_i - @@podTelemetryTimeTracker).abs - timeDifferenceInMinutes = timeDifference / 60 - if (timeDifferenceInMinutes >= 5) - telemetryFlush = true - end - - # Flush AppInsights telemetry once all the processing is done - if telemetryFlush == true - telemetryProperties = {} - telemetryProperties["Computer"] = @@hostName - telemetryProperties["PODS_CHUNK_SIZE"] = @PODS_CHUNK_SIZE - telemetryProperties["PODS_EMIT_STREAM_BATCH_SIZE"] = @PODS_EMIT_STREAM_BATCH_SIZE - telemetryProperties["WatchRestartCount"] = @watchRestartCount - ApplicationInsightsUtility.sendCustomEvent("KubePodInventoryHeartBeatEvent", telemetryProperties) - ApplicationInsightsUtility.sendMetricTelemetry("PodCount", @podCount, {}) - ApplicationInsightsUtility.sendMetricTelemetry("ServiceCount", @serviceCount, {}) - telemetryProperties["ControllerData"] = @controllerData.to_json - ApplicationInsightsUtility.sendMetricTelemetry("ControllerCount", @controllerSet.length, telemetryProperties) - if @winContainerCount > 0 - telemetryProperties["ClusterWideWindowsContainersCount"] = @winContainerCount - ApplicationInsightsUtility.sendCustomEvent("WindowsContainerInventoryEvent", telemetryProperties) - end - ApplicationInsightsUtility.sendMetricTelemetry("PodInventoryE2EProcessingLatencyMs", @podInventoryE2EProcessingLatencyMs, telemetryProperties) - ApplicationInsightsUtility.sendMetricTelemetry("PodsAPIE2ELatencyMs", @podsAPIE2ELatencyMs, telemetryProperties) - @@podTelemetryTimeTracker = DateTime.now.to_time.to_i - end rescue => errorStr $log.warn "in_kube_podinventory::enumerate:Failed in enumerate: #{errorStr}" $log.debug_backtrace(errorStr.backtrace) @@ -628,42 +504,82 @@ def parse_and_emit_merge_updates(podInventoryRecords) end def merge_updates - $log.info("in_kube_podinventory::merge_updates: number of updates in notice hash #{@noticeHash.size} @#{Time.now.utc.iso8601}") - uidList = [] - - @mutex.synchronize { - @noticeHash.each do |uid, record| - uidList.append(uid) - case record["NoticeType"] - when "ADDED" - @podInventoryHash[uid] = record - when "MODIFIED" - if @podInventoryHash[uid].nil? - @podInventoryHash[uid] = record - else - $log.info("in_kube_podinventory::merge_updates : pure modify case") + begin + $log.info("in_kube_podinventory::merge_updates: number of updates in notice hash #{@noticeHash.size} @#{Time.now.utc.iso8601}") + telemetryFlush = false + currentTime = Time.now + batchTime = currentTime.utc.iso8601 + uidList = [] + + @mutex.synchronize { + @noticeHash.each do |uid, record| + uidList.append(uid) + case record["NoticeType"] + when "ADDED" @podInventoryHash[uid] = record - end - when "DELETED" - if @podInventoryHash.key?(uid) - @podInventoryHash.delete(uid) + when "MODIFIED" + if @podInventoryHash[uid].nil? + @podInventoryHash[uid] = record + else + # TODO: remove below log statement later + $log.info("in_kube_podinventory::merge_updates : pure modify case") + @podInventoryHash[uid] = record + end + when "DELETED" + if @podInventoryHash.key?(uid) + @podInventoryHash.delete(uid) + else + $log.info("in_kube_podinventory::merge_updates: key did not exist in hash so unable to delete.") + end + when "BOOKMARK" + $log.info("in_kube_podinventory::merge_updates: received a BOOKMARK event.") else - $log.info("in_kube_podinventory::merge_updates: key did not exist in hash so unable to delete.") + $log.info("in_kube_podinventory::merge_updates: something went wrong and didn't enter any cases for switch, notice type was #{record["NoticeType"]}") end - when "BOOKMARK" - $log.info("in_kube_podinventory::merge_updates: received a BOOKMARK event.") - else - $log.info("in_kube_podinventory::merge_updates: something went wrong and didn't enter any cases for switch, notice type was #{record["NoticeType"]}") end + + # remove all looked at uids from the noticeHash + uidList.each do |uid| + @noticeHash.delete(uid) + end + # TODO: copy noticeHash to tempHash and use tempHash to loop through so we dont lock on it for a long time + } + parse_and_emit_merge_updates(@podInventoryHash) + + # Adding telemetry to send pod telemetry every 5 minutes + timeDifference = (DateTime.now.to_time.to_i - @@podTelemetryTimeTracker).abs + timeDifferenceInMinutes = timeDifference / 60 + if (timeDifferenceInMinutes >= 5) + telemetryFlush = true end - # remove all looked at uids from the noticeHash - uidList.each do |uid| - @noticeHash.delete(uid) + # Flush AppInsights telemetry once all the processing is done + if telemetryFlush == true + telemetryProperties = {} + telemetryProperties["Computer"] = @@hostName + telemetryProperties["PODS_CHUNK_SIZE"] = @PODS_CHUNK_SIZE + telemetryProperties["PODS_EMIT_STREAM_BATCH_SIZE"] = @PODS_EMIT_STREAM_BATCH_SIZE + telemetryProperties["WatchRestartCount"] = @watchRestartCount + # reset watchRestartCount + @watchRestartCount = 0 + ApplicationInsightsUtility.sendCustomEvent("KubePodInventoryHeartBeatEvent", telemetryProperties) + ApplicationInsightsUtility.sendMetricTelemetry("PodCount", @podCount, {}) + ApplicationInsightsUtility.sendMetricTelemetry("ServiceCount", @serviceCount, {}) + telemetryProperties["ControllerData"] = @controllerData.to_json + ApplicationInsightsUtility.sendMetricTelemetry("ControllerCount", @controllerSet.length, telemetryProperties) + if @winContainerCount > 0 + telemetryProperties["ClusterWideWindowsContainersCount"] = @winContainerCount + ApplicationInsightsUtility.sendCustomEvent("WindowsContainerInventoryEvent", telemetryProperties) + end + ApplicationInsightsUtility.sendMetricTelemetry("PodInventoryE2EProcessingLatencyMs", @podInventoryE2EProcessingLatencyMs, telemetryProperties) + ApplicationInsightsUtility.sendMetricTelemetry("PodsAPIE2ELatencyMs", @podsAPIE2ELatencyMs, telemetryProperties) + @@podTelemetryTimeTracker = DateTime.now.to_time.to_i end - # TODO: copy noticeHash to tempHash and use tempHash to loop through so we dont lock on it for a long time - } - parse_and_emit_merge_updates(@podInventoryHash) + rescue => errorStr + $log.warn "in_kube_podinventory::merge_updates:Failed in merge_updates: #{errorStr}" + $log.debug_backtrace(errorStr.backtrace) + ApplicationInsightsUtility.sendExceptionTelemetry(errorStr) + end end def run_periodic From 6aa77ec97f9505c97c53f58ba480ed7f828ac696 Mon Sep 17 00:00:00 2001 From: Khushi Chaudhari Date: Fri, 27 Aug 2021 14:41:03 -0700 Subject: [PATCH 76/76] addressed comments, added telemetry, podInvHash reset, serviceRecords added to merge updates --- kubernetes/omsagent-watch.yaml | 6 +-- source/plugins/ruby/in_kube_podinventory.rb | 42 ++++++++++++++++++--- 2 files changed, 39 insertions(+), 9 deletions(-) diff --git a/kubernetes/omsagent-watch.yaml b/kubernetes/omsagent-watch.yaml index d2669cfbf..b4d2cd144 100644 --- a/kubernetes/omsagent-watch.yaml +++ b/kubernetes/omsagent-watch.yaml @@ -368,7 +368,7 @@ spec: value: "3" containers: - name: omsagent - image: "mcr.microsoft.com/azuremonitor/containerinsights/cidev:khushi-test-70" + image: "mcr.microsoft.com/azuremonitor/containerinsights/cidev:khushi-test-73" imagePullPolicy: Always resources: limits: @@ -448,7 +448,7 @@ spec: timeoutSeconds: 15 #Only in sidecar scraping mode - name: omsagent-prometheus - image: "mcr.microsoft.com/azuremonitor/containerinsights/cidev:khushi-test-70" + image: "mcr.microsoft.com/azuremonitor/containerinsights/cidev:khushi-test-73" imagePullPolicy: Always resources: limits: @@ -591,7 +591,7 @@ spec: serviceAccountName: omsagent containers: - name: omsagent - image: "mcr.microsoft.com/azuremonitor/containerinsights/cidev:khushi-test-70" + image: "mcr.microsoft.com/azuremonitor/containerinsights/cidev:khushi-test-73" imagePullPolicy: Always resources: limits: diff --git a/source/plugins/ruby/in_kube_podinventory.rb b/source/plugins/ruby/in_kube_podinventory.rb index 55672cb57..dccb4be79 100644 --- a/source/plugins/ruby/in_kube_podinventory.rb +++ b/source/plugins/ruby/in_kube_podinventory.rb @@ -148,19 +148,19 @@ def watch @Watcher = @KubernetesWatchClient.watch_pods(resource_version: @collection_version, timeoutSeconds: 300, as: :parsed) @Watcher.each do |notice| $log.info("in_kube_podinventory::watch : inside watch pods! collection version: #{@collection_version}.") - if notice["type"] == "ERROR" - $log.info("in_kube_podinventory::watch : notice type was error. restarting watch.") - break - end if !notice.nil? && !notice.empty? $log.info("in_kube_podinventory::watch : notice was not null and not empty.") + if notice["type"] == "ERROR" + $log.info("in_kube_podinventory::watch : notice type was error. restarting watch.") + break + end item = notice["object"] batchTime = Time.now.utc.iso8601 # Construct record with necessary fields (same fields as getPodInventoryRecords) records = getPodInventoryRecords(item, serviceRecords, batchTime) record = records.first() record["NoticeType"] = notice["type"] - $log.info("in_kube_podinventory::watch : record looks like: #{record}") + # $log.info("in_kube_podinventory::watch : record looks like: #{record}") @mutex.synchronize { @noticeHash[item["metadata"]["uid"]] = record } @@ -189,7 +189,8 @@ def enumerate(podList = nil) batchTime = currentTime.utc.iso8601 @serviceRecords = [] @podInventoryE2EProcessingLatencyMs = 0 - podInventoryStartTime = (Time.now.to_f * 1000).to_i + podInventoryStartTime = (Time.now.to_f * 1000).to_i + @podInventoryHash = {} # Get services first so that we dont need to make a call for very chunk $log.info("in_kube_podinventory::enumerate : Getting services from Kube API @ #{Time.now.utc.iso8601}") @@ -482,6 +483,8 @@ def parse_and_emit_merge_updates(podInventoryRecords) end router.emit_stream(@tag, eventStream) if eventStream emittedPodCount += eventStream.count + # Updating value for AppInsights telemetry + @podCount += emittedPodCount eventStream = Fluent::MultiEventStream.new end @@ -492,6 +495,8 @@ def parse_and_emit_merge_updates(podInventoryRecords) $log.info("kubePodInventoryEmitStreamSuccess @ #{Time.now.utc.iso8601}") end emittedPodCount += eventStream.count + # Updating value for AppInsights telemetry + @podCount += emittedPodCount eventStream = nil end @@ -511,7 +516,26 @@ def merge_updates batchTime = currentTime.utc.iso8601 uidList = [] + # Get services first so that we dont need to make a call for very chunk + $log.info("in_kube_podinventory::enumerate : Getting services from Kube API @ #{Time.now.utc.iso8601}") + serviceInfo = KubernetesApiClient.getKubeResourceInfo("services") + # serviceList = JSON.parse(KubernetesApiClient.getKubeResourceInfo("services").body) + $log.info("in_kube_podinventory::enumerate : Done getting services from Kube API @ #{Time.now.utc.iso8601}") + + if !serviceInfo.nil? + $log.info("in_kube_podinventory::enumerate:Start:Parsing services data using yajl @ #{Time.now.utc.iso8601}") + serviceList = Yajl::Parser.parse(StringIO.new(serviceInfo.body)) + $log.info("in_kube_podinventory::enumerate:End:Parsing services data using yajl @ #{Time.now.utc.iso8601}") + serviceInfo = nil + # service inventory records much smaller and fixed size compared to serviceList + @serviceRecords = KubernetesApiClient.getKubeServicesInventoryRecords(serviceList, batchTime) + # updating for telemetry + @serviceCount += @serviceRecords.length + serviceList = nil + end + @mutex.synchronize { + noticeHashLockStartTime = (Time.now.to_f * 1000).to_i @noticeHash.each do |uid, record| uidList.append(uid) case record["NoticeType"] @@ -543,6 +567,10 @@ def merge_updates @noticeHash.delete(uid) end # TODO: copy noticeHash to tempHash and use tempHash to loop through so we dont lock on it for a long time + + noticeHashLockTotalTime = ((Time.now.to_f * 1000).to_i - noticeHashLockStartTime) + $log.info("in_kube_podinventory::merge_updates : notice hash lock total time = #{noticeHashLockTotalTime}") + $log.info("in_kube_podinventory::merge_updates : number of pods in @podInventoryHash = #{@podInventoryHash.size}, size = ") } parse_and_emit_merge_updates(@podInventoryHash) @@ -564,6 +592,8 @@ def merge_updates @watchRestartCount = 0 ApplicationInsightsUtility.sendCustomEvent("KubePodInventoryHeartBeatEvent", telemetryProperties) ApplicationInsightsUtility.sendMetricTelemetry("PodCount", @podCount, {}) + # TODO: fix serviceCount => cant get actual value unless we watch for it or call it every minute + # get service count in merge_updates so it happens every minute ApplicationInsightsUtility.sendMetricTelemetry("ServiceCount", @serviceCount, {}) telemetryProperties["ControllerData"] = @controllerData.to_json ApplicationInsightsUtility.sendMetricTelemetry("ControllerCount", @controllerSet.length, telemetryProperties)