add sidekiq_notice_only_once configuration to reduce noise from retriable sidekiq failures

lib/new_relic/agent/configuration/default_source.rb

@@ -1411,6 +1411,13 @@ def self.enforce_fallback(allowed_values: nil, fallback: nil)
           :allowed_from_server => false,
           :description => 'If `true`, disables [Sidekiq instrumentation](/docs/agents/ruby-agent/background-jobs/sidekiq-instrumentation).'
         },
+        :sidekiq_notice_only_once => {
+          :default => false,
+          :public => true,
+          :type => Boolean,
+          :allowed_from_server => false,
+          :description => 'If `true`, enables reporting only on last sidekiq retry [Sidekiq instrumentation](/docs/agents/ruby-agent/background-jobs/sidekiq-instrumentation).'
+        },
         :disable_roda_auto_middleware => {
           :default => false,
           :public => true,

lib/new_relic/agent/instrumentation/sidekiq.rb

@@ -40,13 +40,19 @@
         end
       end
 
-      if config.respond_to?(:error_handlers)
+      if config.respond_to?(:error_handlers) && !NewRelic::Agent.config[:sidekiq_notice_only_once]
         # Sidekiq 3.0.0 - 7.1.4 expect error_handlers to have 2 arguments
         # Sidekiq 7.1.5+ expect error_handlers to have 3 arguments
         config.error_handlers << proc do |error, _ctx, *_|
           NewRelic::Agent.notice_error(error)
         end
       end
+
+      if config.respond_to?(:death_handlers) && NewRelic::Agent.config[:sidekiq_notice_only_once]
+        config.death_handlers << proc do |_, error|
+          NewRelic::Agent.notice_error(error)
+        end
+      end
     end
   end
 end

newrelic.yml

@@ -331,6 +331,10 @@ common: &default_settings
   # If true, disables Sidekiq instrumentation.
   # disable_sidekiq: false
 
+  # If true, when a sidekiq job fails only notify it after the last failing retry before going to the dead queue
+  # i.e. skip failures that will be retried
+  # sidekiq_notice_only_once: false
+
   # If true, disables agent middleware for Sinatra. This middleware is responsible
   # for advanced feature support such as cross application tracing, page load
   # timing, and error collection.