From 3f6aeef5ea7e4e5ac1859eae9a7814b72b306594 Mon Sep 17 00:00:00 2001 From: Andrey Date: Tue, 17 Dec 2024 12:29:52 +0100 Subject: [PATCH 1/5] add flaky_test_retries_enabled tag to git_requests.settings_response telemetry metric --- lib/datadog/ci/ext/telemetry.rb | 1 + lib/datadog/ci/remote/library_settings_client.rb | 3 ++- sig/datadog/ci/ext/telemetry.rbs | 2 ++ spec/datadog/ci/remote/library_settings_client_spec.rb | 5 ++++- 4 files changed, 9 insertions(+), 2 deletions(-) diff --git a/lib/datadog/ci/ext/telemetry.rb b/lib/datadog/ci/ext/telemetry.rb index 60419006..d5094f3f 100644 --- a/lib/datadog/ci/ext/telemetry.rb +++ b/lib/datadog/ci/ext/telemetry.rb @@ -81,6 +81,7 @@ module Telemetry TAG_COVERAGE_ENABLED = "coverage_enabled" TAG_ITR_SKIP_ENABLED = "itrskip_enabled" TAG_EARLY_FLAKE_DETECTION_ENABLED = "early_flake_detection_enabled" + TAG_FLAKY_TEST_RETRIES_ENABLED = "flaky_test_retries_enabled" TAG_EARLY_FLAKE_DETECTION_ABORT_REASON = "early_flake_detection_abort_reason" TAG_PROVIDER = "provider" TAG_AUTO_INJECTED = "auto_injected" diff --git a/lib/datadog/ci/remote/library_settings_client.rb b/lib/datadog/ci/remote/library_settings_client.rb index 9b50c0e6..52f15d7d 100644 --- a/lib/datadog/ci/remote/library_settings_client.rb +++ b/lib/datadog/ci/remote/library_settings_client.rb @@ -59,7 +59,8 @@ def fetch(test_session) { Ext::Telemetry::TAG_COVERAGE_ENABLED => library_settings.code_coverage_enabled?.to_s, Ext::Telemetry::TAG_ITR_SKIP_ENABLED => library_settings.tests_skipping_enabled?.to_s, - Ext::Telemetry::TAG_EARLY_FLAKE_DETECTION_ENABLED => library_settings.early_flake_detection_enabled?.to_s + Ext::Telemetry::TAG_EARLY_FLAKE_DETECTION_ENABLED => library_settings.early_flake_detection_enabled?.to_s, + Ext::Telemetry::TAG_FLAKY_TEST_RETRIES_ENABLED => library_settings.flaky_test_retries_enabled?.to_s } ) diff --git a/sig/datadog/ci/ext/telemetry.rbs b/sig/datadog/ci/ext/telemetry.rbs index 94b7c1df..461e7181 100644 --- a/sig/datadog/ci/ext/telemetry.rbs +++ b/sig/datadog/ci/ext/telemetry.rbs @@ -130,6 +130,8 @@ module Datadog TAG_EARLY_FLAKE_DETECTION_ENABLED: "early_flake_detection_enabled" + TAG_FLAKY_TEST_RETRIES_ENABLED: "flaky_test_retries_enabled" + TAG_EARLY_FLAKE_DETECTION_ABORT_REASON: "early_flake_detection_abort_reason" TAG_PROVIDER: "provider" diff --git a/spec/datadog/ci/remote/library_settings_client_spec.rb b/spec/datadog/ci/remote/library_settings_client_spec.rb index 4e83c6a8..0227899e 100644 --- a/spec/datadog/ci/remote/library_settings_client_spec.rb +++ b/spec/datadog/ci/remote/library_settings_client_spec.rb @@ -128,7 +128,10 @@ metric = telemetry_metric(:inc, "git_requests.settings_response") expect(metric.tags).to eq( - "coverage_enabled" => "true", "itrskip_enabled" => "false", "early_flake_detection_enabled" => "true" + "coverage_enabled" => "true", + "itrskip_enabled" => "false", + "early_flake_detection_enabled" => "true", + "flaky_test_retries_enabled" => "true" ) end From bb46767c86fdfffa97d45818ba23f73b5905322d Mon Sep 17 00:00:00 2001 From: Andrey Date: Tue, 17 Dec 2024 12:32:25 +0100 Subject: [PATCH 2/5] add constants for test_session metric tags that we'll use in future --- lib/datadog/ci/ext/telemetry.rb | 3 +++ sig/datadog/ci/ext/telemetry.rbs | 4 ++++ 2 files changed, 7 insertions(+) diff --git a/lib/datadog/ci/ext/telemetry.rb b/lib/datadog/ci/ext/telemetry.rb index d5094f3f..56e56118 100644 --- a/lib/datadog/ci/ext/telemetry.rb +++ b/lib/datadog/ci/ext/telemetry.rb @@ -83,8 +83,11 @@ module Telemetry TAG_EARLY_FLAKE_DETECTION_ENABLED = "early_flake_detection_enabled" TAG_FLAKY_TEST_RETRIES_ENABLED = "flaky_test_retries_enabled" TAG_EARLY_FLAKE_DETECTION_ABORT_REASON = "early_flake_detection_abort_reason" + # tags for test_session metric TAG_PROVIDER = "provider" TAG_AUTO_INJECTED = "auto_injected" + TAG_AGENTLESS_LOG_SUBMISSION_ENABLED = "agentless_log_submission_enabled" + TAG_FAIL_FAST_TEST_ORDER_ENABLED = "fail_fast_test_order_enabled" module EventType TEST = "test" diff --git a/sig/datadog/ci/ext/telemetry.rbs b/sig/datadog/ci/ext/telemetry.rbs index 461e7181..f77cb5cf 100644 --- a/sig/datadog/ci/ext/telemetry.rbs +++ b/sig/datadog/ci/ext/telemetry.rbs @@ -138,6 +138,10 @@ module Datadog TAG_AUTO_INJECTED: "auto_injected" + TAG_AGENTLESS_LOG_SUBMISSION_ENABLED: "agentless_log_submission_enabled" + + TAG_FAIL_FAST_TEST_ORDER_ENABLED: "fail_fast_test_order_enabled" + module EventType TEST: "test" SUITE: "suite" From 259b70a2a80279bc19aeb3eb4ab11da73e425deb Mon Sep 17 00:00:00 2001 From: Andrey Date: Tue, 17 Dec 2024 13:00:07 +0100 Subject: [PATCH 3/5] remove SCA failing due to a bug --- .github/workflows/datadog-sca.yml | 25 ------------------------- 1 file changed, 25 deletions(-) delete mode 100644 .github/workflows/datadog-sca.yml diff --git a/.github/workflows/datadog-sca.yml b/.github/workflows/datadog-sca.yml deleted file mode 100644 index eadf85b0..00000000 --- a/.github/workflows/datadog-sca.yml +++ /dev/null @@ -1,25 +0,0 @@ -on: [push] - -name: Datadog Software Composition Analysis - -jobs: - software-composition-analysis: - runs-on: ubuntu-22.04 - name: Datadog SBOM Generation and Upload - steps: - - name: Checkout - uses: actions/checkout@v4 - - name: Set up Ruby - uses: ruby/setup-ruby@v1 - with: - bundler-cache: true - ruby-version: '3.3' - - name: Check imported libraries are secure and compliant - id: datadog-software-composition-analysis - uses: DataDog/datadog-sca-github-action@main - with: - dd_api_key: ${{ secrets.DD_API_KEY }} - dd_app_key: ${{ secrets.DD_APP_KEY }} - dd_service: datadog-ci-rb - dd_env: ci - dd_site: datadoghq.com From c7eca65639f22cae87128c7d68ad12c40250f551 Mon Sep 17 00:00:00 2001 From: Andrey Date: Tue, 17 Dec 2024 13:08:07 +0100 Subject: [PATCH 4/5] add itr_enabled tag to git_requests.settings_response telemetry metric; improve LibrarySettingsClient spec --- lib/datadog/ci/ext/telemetry.rb | 4 +- .../ci/remote/library_settings_client.rb | 3 +- sig/datadog/ci/ext/telemetry.rbs | 2 + .../ci/remote/library_settings_client_spec.rb | 178 +++++++++--------- 4 files changed, 96 insertions(+), 91 deletions(-) diff --git a/lib/datadog/ci/ext/telemetry.rb b/lib/datadog/ci/ext/telemetry.rb index 56e56118..21f5c9d3 100644 --- a/lib/datadog/ci/ext/telemetry.rb +++ b/lib/datadog/ci/ext/telemetry.rb @@ -69,6 +69,7 @@ module Telemetry TAG_BROWSER_DRIVER = "browser_driver" TAG_IS_RUM = "is_rum" TAG_IS_RETRY = "is_retry" + TAG_EARLY_FLAKE_DETECTION_ABORT_REASON = "early_flake_detection_abort_reason" TAG_IS_NEW = "is_new" TAG_LIBRARY = "library" TAG_ENDPOINT = "endpoint" @@ -78,11 +79,12 @@ module Telemetry TAG_REQUEST_COMPRESSED = "rq_compressed" TAG_RESPONSE_COMPRESSED = "rs_compressed" TAG_COMMAND = "command" + # tags for git_requests.settings_response metric TAG_COVERAGE_ENABLED = "coverage_enabled" + TAG_ITR_ENABLED = "itr_enabled" TAG_ITR_SKIP_ENABLED = "itrskip_enabled" TAG_EARLY_FLAKE_DETECTION_ENABLED = "early_flake_detection_enabled" TAG_FLAKY_TEST_RETRIES_ENABLED = "flaky_test_retries_enabled" - TAG_EARLY_FLAKE_DETECTION_ABORT_REASON = "early_flake_detection_abort_reason" # tags for test_session metric TAG_PROVIDER = "provider" TAG_AUTO_INJECTED = "auto_injected" diff --git a/lib/datadog/ci/remote/library_settings_client.rb b/lib/datadog/ci/remote/library_settings_client.rb index 52f15d7d..e01e0b49 100644 --- a/lib/datadog/ci/remote/library_settings_client.rb +++ b/lib/datadog/ci/remote/library_settings_client.rb @@ -60,7 +60,8 @@ def fetch(test_session) Ext::Telemetry::TAG_COVERAGE_ENABLED => library_settings.code_coverage_enabled?.to_s, Ext::Telemetry::TAG_ITR_SKIP_ENABLED => library_settings.tests_skipping_enabled?.to_s, Ext::Telemetry::TAG_EARLY_FLAKE_DETECTION_ENABLED => library_settings.early_flake_detection_enabled?.to_s, - Ext::Telemetry::TAG_FLAKY_TEST_RETRIES_ENABLED => library_settings.flaky_test_retries_enabled?.to_s + Ext::Telemetry::TAG_FLAKY_TEST_RETRIES_ENABLED => library_settings.flaky_test_retries_enabled?.to_s, + Ext::Telemetry::TAG_ITR_ENABLED => library_settings.itr_enabled?.to_s } ) diff --git a/sig/datadog/ci/ext/telemetry.rbs b/sig/datadog/ci/ext/telemetry.rbs index f77cb5cf..2738586c 100644 --- a/sig/datadog/ci/ext/telemetry.rbs +++ b/sig/datadog/ci/ext/telemetry.rbs @@ -126,6 +126,8 @@ module Datadog TAG_COVERAGE_ENABLED: "coverage_enabled" + TAG_ITR_ENABLED: "itr_enabled" + TAG_ITR_SKIP_ENABLED: "itrskip_enabled" TAG_EARLY_FLAKE_DETECTION_ENABLED: "early_flake_detection_enabled" diff --git a/spec/datadog/ci/remote/library_settings_client_spec.rb b/spec/datadog/ci/remote/library_settings_client_spec.rb index 0227899e..9dd409cd 100644 --- a/spec/datadog/ci/remote/library_settings_client_spec.rb +++ b/spec/datadog/ci/remote/library_settings_client_spec.rb @@ -5,14 +5,13 @@ RSpec.describe Datadog::CI::Remote::LibrarySettingsClient do include_context "Telemetry spy" - let(:api) { spy("api") } let(:dd_env) { "ci" } let(:config_tags) { {} } subject(:client) { described_class.new(api: api, dd_env: dd_env, config_tags: config_tags) } describe "#fetch" do - subject { client.fetch(test_session) } + subject(:response) { client.fetch(test_session) } let(:service) { "service" } let(:tracer_span) do @@ -31,82 +30,82 @@ end let(:test_session) { Datadog::CI::TestSession.new(tracer_span) } - let(:path) { Datadog::CI::Ext::Transport::DD_API_SETTINGS_PATH } + context "when api is present" do + let(:api) { spy("api") } + before do + allow(api).to receive(:api_request).and_return(http_response) + end - it "requests the settings" do - subject + let(:attributes) do + { + "code_coverage" => "1", + "tests_skipping" => "false", + "itr_enabled" => "True", + "require_git" => require_git, + "flaky_test_retries_enabled" => "true", + "early_flake_detection" => { + "enabled" => "true", + "slow_test_retries" => { + "5s" => 10, + "10s" => 5, + "30s" => 3, + "5m" => 2 + }, + "faulty_session_threshold" => 30 + } + } + end - expect(api).to have_received(:api_request) do |args| - expect(args[:path]).to eq(path) + let(:http_response) do + double( + "http_response", + ok?: true, + payload: { + "data" => { + "id" => "123", + "type" => Datadog::CI::Ext::Transport::DD_API_SETTINGS_TYPE, + "attributes" => attributes + } + }.to_json, + request_compressed: false, + duration_ms: 1.2 + ) + end - data = JSON.parse(args[:payload])["data"] + let(:require_git) { false } - expect(data["id"]).to eq(Datadog::Core::Environment::Identity.id) - expect(data["type"]).to eq(Datadog::CI::Ext::Transport::DD_API_SETTINGS_TYPE) + let(:path) { Datadog::CI::Ext::Transport::DD_API_SETTINGS_PATH } - attributes = data["attributes"] - expect(attributes["service"]).to eq(service) - expect(attributes["env"]).to eq(dd_env) - expect(attributes["test_level"]).to eq("test") - expect(attributes["repository_url"]).to eq("repository_url") - expect(attributes["branch"]).to eq("branch") - expect(attributes["sha"]).to eq("commit_sha") + it "requests the settings" do + subject - configurations = attributes["configurations"] - expect(configurations["os.platform"]).to eq("platform") - expect(configurations["os.architecture"]).to eq("arch") - expect(configurations["os.version"]).to eq("version") - expect(configurations["runtime.name"]).to eq("runtime_name") - expect(configurations["runtime.version"]).to eq("runtime_version") - end - end + expect(api).to have_received(:api_request) do |args| + expect(args[:path]).to eq(path) - context "parsing response" do - subject(:response) { client.fetch(test_session) } + data = JSON.parse(args[:payload])["data"] - context "when api is present" do - before do - allow(api).to receive(:api_request).and_return(http_response) - end + expect(data["id"]).to eq(Datadog::Core::Environment::Identity.id) + expect(data["type"]).to eq(Datadog::CI::Ext::Transport::DD_API_SETTINGS_TYPE) - context "when response is OK" do - let(:attributes) do - { - "code_coverage" => "1", - "tests_skipping" => "false", - "itr_enabled" => "True", - "require_git" => require_git, - "flaky_test_retries_enabled" => "true", - "early_flake_detection" => { - "enabled" => "true", - "slow_test_retries" => { - "5s" => 10, - "10s" => 5, - "30s" => 3, - "5m" => 2 - }, - "faulty_session_threshold" => 30 - } - } - end + attributes = data["attributes"] + expect(attributes["service"]).to eq(service) + expect(attributes["env"]).to eq(dd_env) + expect(attributes["test_level"]).to eq("test") + expect(attributes["repository_url"]).to eq("repository_url") + expect(attributes["branch"]).to eq("branch") + expect(attributes["sha"]).to eq("commit_sha") - let(:http_response) do - double( - "http_response", - ok?: true, - payload: { - "data" => { - "id" => "123", - "type" => Datadog::CI::Ext::Transport::DD_API_SETTINGS_TYPE, - "attributes" => attributes - } - }.to_json, - request_compressed: false, - duration_ms: 1.2 - ) - end - let(:require_git) { false } + configurations = attributes["configurations"] + expect(configurations["os.platform"]).to eq("platform") + expect(configurations["os.architecture"]).to eq("arch") + expect(configurations["os.version"]).to eq("version") + expect(configurations["runtime.name"]).to eq("runtime_name") + expect(configurations["runtime.version"]).to eq("runtime_version") + end + end + context "parsing response" do + context "when response is OK" do it "parses the response" do expect(response.ok?).to be true expect(response.payload).to eq(attributes) @@ -129,6 +128,7 @@ metric = telemetry_metric(:inc, "git_requests.settings_response") expect(metric.tags).to eq( "coverage_enabled" => "true", + "itr_enabled" => "true", "itrskip_enabled" => "false", "early_flake_detection_enabled" => "true", "flaky_test_retries_enabled" => "true" @@ -146,6 +146,22 @@ expect(response.require_git?).to be true end end + + context "with custom configuration" do + let(:config_tags) { {"key" => "value"} } + + it "requests the settings" do + subject + + expect(api).to have_received(:api_request) do |args| + data = JSON.parse(args[:payload])["data"] + + attributes = data["attributes"] + configurations = attributes["configurations"] + expect(configurations["custom"]).to eq("key" => "value") + end + end + end end context "when response is not OK" do @@ -221,31 +237,15 @@ end end end - - context "when there is no api" do - let(:api) { nil } - - it "returns an empty response" do - expect(response.ok?).to be false - expect(response.payload).to eq("itr_enabled" => false) - expect(response.require_git?).to be false - end - end end - context "with custom configuration" do - let(:config_tags) { {"key" => "value"} } - - it "requests the settings" do - subject - - expect(api).to have_received(:api_request) do |args| - data = JSON.parse(args[:payload])["data"] + context "when there is no api" do + let(:api) { nil } - attributes = data["attributes"] - configurations = attributes["configurations"] - expect(configurations["custom"]).to eq("key" => "value") - end + it "returns an empty response" do + expect(response.ok?).to be false + expect(response.payload).to eq("itr_enabled" => false) + expect(response.require_git?).to be false end end end From 96f8c5a36705921886b6eefa4d607f3d50a305fe Mon Sep 17 00:00:00 2001 From: Andrey Date: Tue, 17 Dec 2024 13:11:10 +0100 Subject: [PATCH 5/5] add require_git tag to git_requests.settings_response telemetry metric --- lib/datadog/ci/ext/telemetry.rb | 1 + lib/datadog/ci/remote/library_settings_client.rb | 3 ++- sig/datadog/ci/ext/telemetry.rbs | 2 ++ spec/datadog/ci/remote/library_settings_client_spec.rb | 3 ++- 4 files changed, 7 insertions(+), 2 deletions(-) diff --git a/lib/datadog/ci/ext/telemetry.rb b/lib/datadog/ci/ext/telemetry.rb index 21f5c9d3..1e38de1c 100644 --- a/lib/datadog/ci/ext/telemetry.rb +++ b/lib/datadog/ci/ext/telemetry.rb @@ -83,6 +83,7 @@ module Telemetry TAG_COVERAGE_ENABLED = "coverage_enabled" TAG_ITR_ENABLED = "itr_enabled" TAG_ITR_SKIP_ENABLED = "itrskip_enabled" + TAG_REQUIRE_GIT = "require_git" TAG_EARLY_FLAKE_DETECTION_ENABLED = "early_flake_detection_enabled" TAG_FLAKY_TEST_RETRIES_ENABLED = "flaky_test_retries_enabled" # tags for test_session metric diff --git a/lib/datadog/ci/remote/library_settings_client.rb b/lib/datadog/ci/remote/library_settings_client.rb index e01e0b49..12dfa476 100644 --- a/lib/datadog/ci/remote/library_settings_client.rb +++ b/lib/datadog/ci/remote/library_settings_client.rb @@ -61,7 +61,8 @@ def fetch(test_session) Ext::Telemetry::TAG_ITR_SKIP_ENABLED => library_settings.tests_skipping_enabled?.to_s, Ext::Telemetry::TAG_EARLY_FLAKE_DETECTION_ENABLED => library_settings.early_flake_detection_enabled?.to_s, Ext::Telemetry::TAG_FLAKY_TEST_RETRIES_ENABLED => library_settings.flaky_test_retries_enabled?.to_s, - Ext::Telemetry::TAG_ITR_ENABLED => library_settings.itr_enabled?.to_s + Ext::Telemetry::TAG_ITR_ENABLED => library_settings.itr_enabled?.to_s, + Ext::Telemetry::TAG_REQUIRE_GIT => library_settings.require_git?.to_s } ) diff --git a/sig/datadog/ci/ext/telemetry.rbs b/sig/datadog/ci/ext/telemetry.rbs index 2738586c..9ef2fd3c 100644 --- a/sig/datadog/ci/ext/telemetry.rbs +++ b/sig/datadog/ci/ext/telemetry.rbs @@ -130,6 +130,8 @@ module Datadog TAG_ITR_SKIP_ENABLED: "itrskip_enabled" + TAG_REQUIRE_GIT: "require_git" + TAG_EARLY_FLAKE_DETECTION_ENABLED: "early_flake_detection_enabled" TAG_FLAKY_TEST_RETRIES_ENABLED: "flaky_test_retries_enabled" diff --git a/spec/datadog/ci/remote/library_settings_client_spec.rb b/spec/datadog/ci/remote/library_settings_client_spec.rb index 9dd409cd..5d3fa946 100644 --- a/spec/datadog/ci/remote/library_settings_client_spec.rb +++ b/spec/datadog/ci/remote/library_settings_client_spec.rb @@ -131,7 +131,8 @@ "itr_enabled" => "true", "itrskip_enabled" => "false", "early_flake_detection_enabled" => "true", - "flaky_test_retries_enabled" => "true" + "flaky_test_retries_enabled" => "true", + "require_git" => "false" ) end