Merge pull request #86 from DataDog/anmarchenko/rspec_instrumentation

[CIVIS-7950] Add rspec instrumentation for test suite level visibility

integration/app/spec/basic_spec.rb

@@ -1,4 +1,8 @@
+require_relative "shared_examples"
+
 RSpec.describe "Let's do some math" do
+  include_examples "Testing shared examples"
+
   it "add" do
     expect(1 + 1).to eq(2)
   end

integration/app/spec/shared_examples.rb

@@ -0,0 +1,7 @@
+RSpec.shared_examples "Testing shared examples" do
+  context "shared examples" do
+    it "adds 1 and 1" do
+      expect(1 + 1).to eq(2)
+    end
+  end
+end

integration/app/spec/spec_helper.rb

@@ -5,9 +5,6 @@
   c.service = "datadog-ci-integration-app"
   c.env = "local"
 
-  c.tracing.enabled = true
-  c.tracing.test_mode.enabled = true
-
   c.ci.enabled = true
   c.ci.instrument :rspec
 end

lib/datadog/ci/contrib/rspec/configuration/settings.rb

@@ -19,7 +19,7 @@ class Settings < Datadog::CI::Contrib::Settings
 
             option :service_name do |o|
               o.type :string
-              o.default { Datadog.configuration.service_without_fallback || Ext::SERVICE_NAME }
+              o.default { Datadog.configuration.service_without_fallback || Ext::DEFAULT_SERVICE_NAME }
             end
 
             # @deprecated Will be removed in 1.0

lib/datadog/ci/contrib/rspec/example.rb

@@ -26,11 +26,11 @@ def run(example_group_instance, reporter)
 
               CI.trace_test(
                 test_name,
-                metadata[:example_group][:file_path],
+                metadata[:example_group][:rerun_file_path],
                 tags: {
                   CI::Ext::Test::TAG_FRAMEWORK => Ext::FRAMEWORK,
                   CI::Ext::Test::TAG_FRAMEWORK_VERSION => CI::Contrib::RSpec::Integration.version.to_s,
-                  CI::Ext::Test::TAG_TYPE => Ext::TEST_TYPE
+                  CI::Ext::Test::TAG_TYPE => CI::Ext::Test::TEST_TYPE
                 },
                 service: configuration[:service_name]
               ) do |test_span|

lib/datadog/ci/contrib/rspec/example_group.rb

@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+require_relative "../../ext/test"
+require_relative "ext"
+
+module Datadog
+  module CI
+    module Contrib
+      module RSpec
+        # Instrument RSpec::Core::Example
+        module ExampleGroup
+          def self.included(base)
+            base.singleton_class.prepend(ClassMethods)
+          end
+
+          # Instance methods for configuration
+          module ClassMethods
+            def run(reporter = ::RSpec::Core::NullReporter)
+              return super unless configuration[:enabled]
+              return super unless top_level?
+
+              test_suite = Datadog::CI.start_test_suite(file_path)
+
+              result = super
+
+              if result
+                test_suite.passed!
+              else
+                test_suite.failed!
+              end
+              test_suite.finish
+
+              result
+            end
+
+            private
+
+            def configuration
+              Datadog.configuration.ci[:rspec]
+            end
+          end
+        end
+      end
+    end
+  end
+end

lib/datadog/ci/contrib/rspec/ext.rb

@@ -7,13 +7,14 @@ module RSpec
         # RSpec integration constants
         # TODO: mark as `@public_api` when GA, to protect from resource and tag name changes.
         module Ext
-          APP = "rspec"
+          FRAMEWORK = "rspec"
+          DEFAULT_SERVICE_NAME = "rspec"
+
           ENV_ENABLED = "DD_TRACE_RSPEC_ENABLED"
+
+          # TODO: remove in 1.0
           ENV_OPERATION_NAME = "DD_TRACE_RSPEC_OPERATION_NAME"
-          FRAMEWORK = "rspec"
           OPERATION_NAME = "rspec.example"
-          SERVICE_NAME = "rspec"
-          TEST_TYPE = "test"
         end
       end
     end

lib/datadog/ci/contrib/rspec/patcher.rb

@@ -1,7 +1,10 @@
 # frozen_string_literal: true
 
 require "datadog/tracing/contrib/patcher"
+
 require_relative "example"
+require_relative "example_group"
+require_relative "runner"
 
 module Datadog
   module CI
@@ -19,6 +22,8 @@ def target_version
 
           def patch
             ::RSpec::Core::Example.include(Example)
+            ::RSpec::Core::Runner.include(Runner)
+            ::RSpec::Core::ExampleGroup.include(ExampleGroup)
           end
         end
       end

lib/datadog/ci/contrib/rspec/runner.rb

@@ -0,0 +1,57 @@
+# frozen_string_literal: true
+
+require_relative "../../ext/test"
+require_relative "ext"
+
+module Datadog
+  module CI
+    module Contrib
+      module RSpec
+        # Instrument RSpec::Core::Runner
+        module Runner
+          def self.included(base)
+            base.prepend(InstanceMethods)
+          end
+
+          module InstanceMethods
+            def run_specs(example_groups)
+              return super unless configuration[:enabled]
+
+              test_session = CI.start_test_session(
+                tags: {
+                  CI::Ext::Test::TAG_FRAMEWORK => Ext::FRAMEWORK,
+                  CI::Ext::Test::TAG_FRAMEWORK_VERSION => CI::Contrib::RSpec::Integration.version.to_s,
+                  CI::Ext::Test::TAG_TYPE => CI::Ext::Test::TEST_TYPE
+                },
+                service: configuration[:service_name]
+              )
+
+              test_module = CI.start_test_module(test_session.name)
+
+              result = super
+
+              if result != 0
+                # TODO: repeating this twice feels clunky, we need to remove test_module API before GA
+                test_module.failed!
+                test_session.failed!
+              else
+                test_module.passed!
+                test_session.passed!
+              end
+              test_module.finish
+              test_session.finish
+
+              result
+            end
+
+            private
+
+            def configuration
+              Datadog.configuration.ci[:rspec]
+            end
+          end
+        end
+      end
+    end
+  end
+end

lib/datadog/ci/ext/test.rb

@@ -20,6 +20,8 @@ module Test
         TAG_TYPE = "test.type"
         TAG_COMMAND = "test.command"
 
+        TEST_TYPE = "test"
+
         # those tags are special and they are used to correlate tests with the test sessions, suites, and modules
         TAG_TEST_SESSION_ID = "_test.session_id"
         TAG_TEST_MODULE_ID = "_test.module_id"

lib/datadog/ci/test.rb

@@ -20,6 +20,28 @@ def finish
 
         CI.deactivate_test(self)
       end
+
+      # Span id of the running test suite this test belongs to.
+      # @return [String] the span id of the test suite.
+      def test_suite_id
+        get_tag(Ext::Test::TAG_TEST_SUITE_ID)
+      end
+
+      def test_suite_name
+        get_tag(Ext::Test::TAG_SUITE)
+      end
+
+      # Span id of the running test module this test belongs to.
+      # @return [String] the span id of the test module.
+      def test_module_id
+        get_tag(Ext::Test::TAG_TEST_MODULE_ID)
+      end
+
+      # Span id of the running test module this test belongs to.
+      # @return [String] the span id of the test session.
+      def test_session_id
+        get_tag(Ext::Test::TAG_TEST_SESSION_ID)
+      end
     end
   end
 end

lib/datadog/ci/test_session.rb

@@ -20,6 +20,14 @@ def finish
         CI.deactivate_test_session
       end
 
+      # Return the test session's name which is equal to test command used
+      # @return [String] the command for this test session.
+      def name
+        get_tag(Ext::Test::TAG_COMMAND)
+      end
+
+      # Return the test session tags that could be inherited by sub-spans
+      # @return [Hash] the tags to be inherited by sub-spans.
       def inheritable_tags
         return @inheritable_tags if defined?(@inheritable_tags)
 

lib/datadog/ci/test_visibility/recorder.rb

@@ -210,6 +210,7 @@ def build_test_suite(tracer_span, tags)
         def build_test(tracer_span, tags)
           test = Test.new(tracer_span)
           set_initial_tags(test, tags)
+          validate_test_suite_level_visibility_correctness(test)
           test
         end
 
@@ -287,6 +288,31 @@ def start_datadog_tracer_span(span_name, span_options, &block)
         def null_span
           @null_span ||= NullSpan.new
         end
+
+        def validate_test_suite_level_visibility_correctness(test)
+          return unless test_suite_level_visibility_enabled
+
+          if test.test_suite_id.nil?
+            Datadog.logger.debug do
+              "Test [#{test.name}] does not have a test suite associated with it. " \
+              "Expected test suite [#{test.test_suite_name}] to be running."
+            end
+          end
+
+          if test.test_module_id.nil?
+            Datadog.logger.debug do
+              "Test [#{test.name}] does not have a test module associated with it. " \
+              "Make sure that there is a test module running within a session."
+            end
+          end
+
+          if test.test_session_id.nil?
+            Datadog.logger.debug do
+              "Test [#{test.name}] does not have a test session associated with it. " \
+              "Make sure that there is a test session running."
+            end
+          end
+        end
       end
     end
   end

sig/datadog/ci.rbs

@@ -18,6 +18,8 @@ module Datadog
 
     def self.active_test: () -> Datadog::CI::Test?
 
+    def self.active_test_suite: (String test_suite_name) -> Datadog::CI::TestSuite?
+
     def self.active_span: (String span_type) -> Datadog::CI::Span?
 
     def self.deactivate_test: (Datadog::CI::Test test) -> void

sig/datadog/ci/contrib/rspec/example_group.rbs

@@ -0,0 +1,21 @@
+module Datadog
+  module CI
+    module Contrib
+      module RSpec
+        module ExampleGroup
+          def self.included: (untyped base) -> untyped
+
+          module ClassMethods
+            include ::RSpec::Core::ExampleGroup::ClassMethods
+
+            def run: (?untyped reporter) -> untyped
+
+            private
+
+            def configuration: () -> untyped
+          end
+        end
+      end
+    end
+  end
+end

sig/datadog/ci/contrib/rspec/ext.rbs

@@ -3,19 +3,13 @@ module Datadog
     module Contrib
       module RSpec
         module Ext
-          APP: String
-
           ENV_ENABLED: String
 
-          ENV_OPERATION_NAME: String
-
           FRAMEWORK: String
+          DEFAULT_SERVICE_NAME: String
 
           OPERATION_NAME: String
-
-          SERVICE_NAME: String
-
-          TEST_TYPE: String
+          ENV_OPERATION_NAME: String
         end
       end
     end

sig/datadog/ci/contrib/rspec/runner.rbs

@@ -0,0 +1,21 @@
+module Datadog
+  module CI
+    module Contrib
+      module RSpec
+        module Runner
+          def self.included: (untyped base) -> untyped
+
+          module InstanceMethods
+            include ::RSpec::Core::Runner
+
+            def run_specs: (untyped example_groups) -> untyped
+
+            private
+
+            def configuration: () -> untyped
+          end
+        end
+      end
+    end
+  end
+end

sig/datadog/ci/ext/test.rbs

@@ -32,6 +32,8 @@ module Datadog
 
         TAG_TEST_SUITE_ID: String
 
+        TEST_TYPE: String
+
         SPECIAL_TAGS: Array[String]
 
         INHERITABLE_TAGS: Array[String]

sig/datadog/ci/test.rbs

@@ -2,6 +2,10 @@ module Datadog
   module CI
     class Test < Span
       def finish: () -> void
+      def test_suite_id: () -> String?
+      def test_suite_name: () -> String?
+      def test_module_id: () -> String?
+      def test_session_id: () -> String?
     end
   end
 end

sig/datadog/ci/test_visibility/recorder.rbs

@@ -79,6 +79,8 @@ module Datadog
         def start_datadog_tracer_span: (String span_name, Hash[untyped, untyped] span_options) ?{ (untyped) -> untyped } -> untyped
 
         def set_inherited_globals: (Hash[untyped, untyped] tags) -> void
+
+        def validate_test_suite_level_visibility_correctness: (Datadog::CI::Test test) -> void
       end
     end
   end