Skip to content

Commit 86dc030

Browse files
gnufedevitor-de-araujo
gnufede
and
vitor-de-araujo
authored
chore(ci_visibility): known tests enabled (#13006)
CI Visibility: Reads "known tests enabled" setting from settings and acts according to it (superseding EFD). Also sends the value through telemetry. ## Checklist - [x] PR author has checked that all the criteria below are met - The PR description includes an overview of the change - The PR description articulates the motivation for the change - The change includes tests OR the PR description describes a testing strategy - The PR description notes risks associated with the change, if any - Newly-added code is easy to change - The change follows the [library release note guidelines](https://ddtrace.readthedocs.io/en/stable/releasenotes.html) - The change includes or references documentation updates if necessary - Backport labels are set (if [applicable](https://ddtrace.readthedocs.io/en/latest/contributing.html#backporting)) ## Reviewer Checklist - [x] Reviewer has checked that all the criteria below are met - Title is accurate - All changes are related to the pull request's stated goal - Avoids breaking [API](https://ddtrace.readthedocs.io/en/stable/versioning.html#interfaces) changes - Testing strategy adequately addresses listed risks - Newly-added code is easy to change - Release note makes sense to a user of the library - If necessary, author has acknowledged and discussed the performance implications of this PR as reported in the benchmarks PR comment - Backport labels are set in a manner that is consistent with the [release branch maintenance policy](https://ddtrace.readthedocs.io/en/latest/contributing.html#backporting) --------- Co-authored-by: Vítor De Araújo <[email protected]>
1 parent 25684ea commit 86dc030

14 files changed

+230
-28
lines changed

‎ddtrace/internal/ci_visibility/_api_client.py

+4
Original file line numberDiff line numberDiff line change
@@ -118,6 +118,7 @@ class TestVisibilityAPISettings:
118118
require_git: bool = False
119119
itr_enabled: bool = False
120120
flaky_test_retries_enabled: bool = False
121+
known_tests_enabled: bool = False
121122
early_flake_detection: EarlyFlakeDetectionSettings = dataclasses.field(default_factory=EarlyFlakeDetectionSettings)
122123
test_management: TestManagementSettings = dataclasses.field(default_factory=TestManagementSettings)
123124

@@ -386,6 +387,7 @@ def fetch_settings(self) -> TestVisibilityAPISettings:
386387
flaky_test_retries_enabled = attributes["flaky_test_retries_enabled"] or asbool(
387388
os.getenv("_DD_TEST_FORCE_ENABLE_ATR")
388389
)
390+
known_tests_enabled = attributes["known_tests_enabled"]
389391

390392
if attributes["early_flake_detection"]["enabled"]:
391393
early_flake_detection = EarlyFlakeDetectionSettings(
@@ -426,6 +428,7 @@ def fetch_settings(self) -> TestVisibilityAPISettings:
426428
require_git=require_git,
427429
itr_enabled=itr_enabled,
428430
flaky_test_retries_enabled=flaky_test_retries_enabled,
431+
known_tests_enabled=known_tests_enabled,
429432
early_flake_detection=early_flake_detection,
430433
test_management=test_management,
431434
)
@@ -436,6 +439,7 @@ def fetch_settings(self) -> TestVisibilityAPISettings:
436439
require_git=api_settings.require_git,
437440
itr_enabled=api_settings.itr_enabled,
438441
flaky_test_retries_enabled=api_settings.flaky_test_retries_enabled,
442+
known_tests_enabled=api_settings.known_tests_enabled,
439443
early_flake_detection_enabled=api_settings.early_flake_detection.enabled,
440444
test_management_enabled=api_settings.test_management.enabled,
441445
)

‎ddtrace/internal/ci_visibility/api/_base.py

+8
Original file line numberDiff line numberDiff line change
@@ -70,6 +70,7 @@ class TestVisibilitySessionSettings:
7070
itr_test_skipping_level: Optional[ITR_SKIPPING_LEVEL] = None
7171
itr_correlation_id: str = ""
7272
coverage_enabled: bool = False
73+
known_tests_enabled: bool = False
7374
efd_settings: EarlyFlakeDetectionSettings = dataclasses.field(default_factory=EarlyFlakeDetectionSettings)
7475
atr_settings: AutoTestRetriesSettings = dataclasses.field(default_factory=AutoTestRetriesSettings)
7576
test_management_settings: TestManagementSettings = dataclasses.field(default_factory=TestManagementSettings)
@@ -208,6 +209,9 @@ def _finish_span(self, override_finish_time: Optional[float] = None) -> None:
208209
if self._session_settings.efd_settings is not None and self._session_settings.efd_settings.enabled:
209210
self._set_efd_tags()
210211

212+
if self._session_settings.known_tests_enabled:
213+
self._set_known_tests_tags()
214+
211215
if self._session_settings.atr_settings is not None and self._session_settings.atr_settings.enabled:
212216
self._set_atr_tags()
213217

@@ -278,6 +282,10 @@ def _set_efd_tags(self) -> None:
278282
"""EFD tags are only set at the test or session level"""
279283
pass
280284

285+
def _set_known_tests_tags(self) -> None:
286+
"""Known test tags are only set at the test level"""
287+
pass
288+
281289
def _set_atr_tags(self) -> None:
282290
"""ATR tags are only set at the test level"""
283291
pass

‎ddtrace/internal/ci_visibility/api/_test.py

+18-7
Original file line numberDiff line numberDiff line change
@@ -89,6 +89,7 @@ def __init__(
8989
self._is_disabled = is_disabled
9090
self._is_attempt_to_fix = is_attempt_to_fix
9191

92+
self._is_known_tests_enabled = session_settings.known_tests_enabled
9293
self._efd_is_retry = is_efd_retry
9394
self._efd_retries: List[TestVisibilityTest] = []
9495
self._efd_abort_reason: Optional[str] = None
@@ -125,6 +126,23 @@ def _set_item_tags(self) -> None:
125126
if self._overwritten_suite_name is not None:
126127
self.set_tag(test.SUITE, self._overwritten_suite_name)
127128

129+
def _set_known_tests_tags(self) -> None:
130+
# NOTE: The `is_new` tag is currently being set in the context of:
131+
# - Known tests enabled
132+
# - EFD
133+
if not self.is_new():
134+
return
135+
136+
session = self.get_session()
137+
if session is not None and self._session_settings.efd_settings.enabled:
138+
# If a session is considered faulty, we do not want to tag the
139+
# test as new.
140+
if not session.efd_is_faulty_session():
141+
self.set_tag(TEST_IS_NEW, self._is_new)
142+
143+
elif self._is_known_tests_enabled:
144+
self.set_tag(TEST_IS_NEW, self._is_new)
145+
128146
def _set_efd_tags(self) -> None:
129147
if self._efd_is_retry:
130148
self.set_tag(TEST_IS_RETRY, self._efd_is_retry)
@@ -133,13 +151,6 @@ def _set_efd_tags(self) -> None:
133151
if self._efd_abort_reason is not None:
134152
self.set_tag(TEST_EFD_ABORT_REASON, self._efd_abort_reason)
135153

136-
# NOTE: The is_new tag is currently only being set in the context of EFD (since that is the only context in
137-
# which unique tests are fetched). Additionally, if a session is considered faulty, we do not want to tag the
138-
# test as new.
139-
session = self.get_session()
140-
if self.is_new() and session is not None and not session.efd_is_faulty_session():
141-
self.set_tag(TEST_IS_NEW, self._is_new)
142-
143154
def _set_atr_tags(self) -> None:
144155
if self._atr_is_retry:
145156
self.set_tag(TEST_IS_RETRY, self._atr_is_retry)

‎ddtrace/internal/ci_visibility/recorder.py

+20-6
Original file line numberDiff line numberDiff line change
@@ -304,6 +304,10 @@ def __init__(
304304
"API-provided settings: Early Flake Detection enabled: %s",
305305
self._api_settings.early_flake_detection.enabled,
306306
)
307+
log.info(
308+
"API-provided settings: Known Tests enabled: %s",
309+
self._api_settings.known_tests_enabled,
310+
)
307311
log.info("API-provided settings: Auto Test Retries enabled: %s", self._api_settings.flaky_test_retries_enabled)
308312
log.info("Detected configurations: %s", str(self._configurations))
309313

@@ -440,12 +444,19 @@ def test_skipping_enabled(cls) -> bool:
440444
return False
441445
return cls._instance._api_settings.skipping_enabled
442446

447+
@classmethod
448+
def is_known_tests_enabled(cls) -> bool:
449+
if cls._instance is None:
450+
return False
451+
return cls._instance._api_settings.known_tests_enabled
452+
443453
@classmethod
444454
def is_efd_enabled(cls) -> bool:
445455
if cls._instance is None:
446456
return False
447457
return (
448-
cls._instance._api_settings.early_flake_detection.enabled
458+
cls._instance._api_settings.known_tests_enabled # Known Tests Enabled takes precedence over EFD
459+
and cls._instance._api_settings.early_flake_detection.enabled
449460
and ddconfig._test_visibility_early_flake_detection_enabled
450461
)
451462

@@ -578,12 +589,14 @@ def enable(cls, tracer=None, config=None, service=None) -> None:
578589
"test skipping: %s, "
579590
"Early Flake Detection: %s, "
580591
"Auto Test Retries: %s, "
581-
"Flaky Test Management: %s",
592+
"Flaky Test Management: %s, "
593+
"Known Tests: %s",
582594
cls._instance._collect_coverage_enabled,
583595
CIVisibility.test_skipping_enabled(),
584596
CIVisibility.is_efd_enabled(),
585597
CIVisibility.is_atr_enabled(),
586598
CIVisibility.is_test_management_enabled(),
599+
CIVisibility.is_known_tests_enabled(),
587600
)
588601

589602
@classmethod
@@ -619,13 +632,13 @@ def _task_fetch_tests_to_skip():
619632
log.info("ITR correlation ID: %s", self._itr_data.correlation_id)
620633

621634
def _task_fetch_known_tests():
622-
if CIVisibility.is_efd_enabled():
635+
if CIVisibility.is_known_tests_enabled():
623636
known_test_ids = self._fetch_known_tests()
624637
if known_test_ids is None:
625-
log.warning("Failed to fetch unique tests for Early Flake Detection")
638+
log.warning("Failed to fetch known tests for Early Flake Detection")
626639
else:
627640
self._known_test_ids = known_test_ids
628-
log.info("Unique tests fetched for Early Flake Detection: %s", len(self._known_test_ids))
641+
log.info("Known tests fetched for Early Flake Detection: %s", len(self._known_test_ids))
629642
else:
630643
if (
631644
self._api_settings.early_flake_detection.enabled
@@ -1048,6 +1061,7 @@ def _on_discover_session(discover_args: TestSession.DiscoverArgs) -> None:
10481061
itr_test_skipping_level=instance._itr_skipping_level,
10491062
itr_correlation_id=instance._itr_meta.get(ITR_CORRELATION_ID_TAG_NAME, ""),
10501063
coverage_enabled=CIVisibility.should_collect_coverage(),
1064+
known_tests_enabled=CIVisibility.is_known_tests_enabled(),
10511065
efd_settings=efd_api_settings,
10521066
atr_settings=atr_api_settings,
10531067
test_management_settings=test_management_api_settings,
@@ -1245,7 +1259,7 @@ def _on_discover_test(discover_args: Test.DiscoverArgs) -> None:
12451259
# New tests are currently only considered for EFD:
12461260
# - if known tests were fetched properly (enforced by is_known_test)
12471261
# - if they have no parameters
1248-
if CIVisibility.is_efd_enabled() and discover_args.test_id.parameters is None:
1262+
if CIVisibility.is_known_tests_enabled() and discover_args.test_id.parameters is None:
12491263
is_new = not CIVisibility.is_known_test(discover_args.test_id)
12501264
else:
12511265
is_new = False

‎ddtrace/internal/ci_visibility/telemetry/git.py

+5
Original file line numberDiff line numberDiff line change
@@ -61,6 +61,7 @@ def record_settings_response(
6161
require_git: Optional[bool] = False,
6262
itr_enabled: Optional[bool] = False,
6363
flaky_test_retries_enabled: Optional[bool] = False,
64+
known_tests_enabled: Optional[bool] = False,
6465
early_flake_detection_enabled: Optional[bool] = False,
6566
test_management_enabled: Optional[bool] = False,
6667
) -> None:
@@ -71,13 +72,15 @@ def record_settings_response(
7172
", require_git=%s"
7273
", itr_enabled=%s"
7374
", flaky_test_retries_enabled=%s"
75+
", known_tests_enabled=%s"
7476
", early_flake_detection_enabled=%s"
7577
", test_management_enabled=%s",
7678
coverage_enabled,
7779
skipping_enabled,
7880
require_git,
7981
itr_enabled,
8082
flaky_test_retries_enabled,
83+
known_tests_enabled,
8184
early_flake_detection_enabled,
8285
test_management_enabled,
8386
)
@@ -93,6 +96,8 @@ def record_settings_response(
9396
response_tags.append(("itr_enabled", "true"))
9497
if flaky_test_retries_enabled:
9598
response_tags.append(("flaky_test_retries_enabled", "true"))
99+
if known_tests_enabled:
100+
response_tags.append(("known_tests_enabled", "true"))
96101
if early_flake_detection_enabled:
97102
response_tags.append(("early_flake_detection_enabled", "true"))
98103
if test_management_enabled:

‎tests/ci_visibility/api/fake_runner_efd_all_pass.py

+3-1
Original file line numberDiff line numberDiff line change
@@ -281,7 +281,9 @@ def main():
281281
with mock.patch(
282282
"ddtrace.internal.ci_visibility.recorder.CIVisibility._check_enabled_features",
283283
return_value=TestVisibilityAPISettings(
284-
require_git=False, early_flake_detection=EarlyFlakeDetectionSettings(True)
284+
require_git=False,
285+
early_flake_detection=EarlyFlakeDetectionSettings(True),
286+
known_tests_enabled=True,
285287
),
286288
):
287289
ext_api.enable_test_visibility()

‎tests/ci_visibility/api/fake_runner_efd_faulty_session.py

+3-1
Original file line numberDiff line numberDiff line change
@@ -215,7 +215,9 @@ def main():
215215
with mock.patch(
216216
"ddtrace.internal.ci_visibility.recorder.CIVisibility._check_enabled_features",
217217
return_value=TestVisibilityAPISettings(
218-
require_git=False, early_flake_detection=EarlyFlakeDetectionSettings(True)
218+
require_git=False,
219+
early_flake_detection=EarlyFlakeDetectionSettings(True),
220+
known_tests_enabled=True,
219221
),
220222
):
221223
ext_api.enable_test_visibility()

‎tests/ci_visibility/api/fake_runner_efd_mix_fail.py

+3-1
Original file line numberDiff line numberDiff line change
@@ -281,7 +281,9 @@ def main():
281281
with mock.patch(
282282
"ddtrace.internal.ci_visibility.recorder.CIVisibility._check_enabled_features",
283283
return_value=TestVisibilityAPISettings(
284-
require_git=False, early_flake_detection=EarlyFlakeDetectionSettings(True)
284+
require_git=False,
285+
early_flake_detection=EarlyFlakeDetectionSettings(True),
286+
known_tests_enabled=True,
285287
),
286288
):
287289
ext_api.enable_test_visibility()

‎tests/ci_visibility/api/fake_runner_efd_mix_pass.py

+3-1
Original file line numberDiff line numberDiff line change
@@ -285,7 +285,9 @@ def main():
285285
with mock.patch(
286286
"ddtrace.internal.ci_visibility.recorder.CIVisibility._check_enabled_features",
287287
return_value=TestVisibilityAPISettings(
288-
require_git=False, early_flake_detection=EarlyFlakeDetectionSettings(True)
288+
require_git=False,
289+
early_flake_detection=EarlyFlakeDetectionSettings(True),
290+
known_tests_enabled=True,
289291
),
290292
):
291293
ext_api.enable_test_visibility()

‎tests/ci_visibility/api/test_api_fake_runners.py

+9-9
Original file line numberDiff line numberDiff line change
@@ -65,7 +65,7 @@ def test_manual_api_fake_runner_all_pass(self):
6565
replace_os_env=True,
6666
), mock.patch(
6767
"ddtrace.internal.ci_visibility._api_client._TestVisibilityAPIClientBase.fetch_settings",
68-
return_value=TestVisibilityAPISettings(False, False, False, False),
68+
return_value=TestVisibilityAPISettings(False, False, False, False, False),
6969
), mock.patch(
7070
"ddtrace.internal.ci_visibility.recorder.ddconfig", _get_default_civisibility_ddconfig()
7171
):
@@ -91,7 +91,7 @@ def test_manual_api_fake_runner_all_fail(self):
9191
):
9292
with mock.patch(
9393
"ddtrace.internal.ci_visibility._api_client._TestVisibilityAPIClientBase.fetch_settings",
94-
return_value=TestVisibilityAPISettings(False, False, False, False),
94+
return_value=TestVisibilityAPISettings(False, False, False, False, False),
9595
), mock.patch("ddtrace.internal.ci_visibility.recorder.ddconfig", _get_default_civisibility_ddconfig()):
9696
subprocess.run(["python", "fake_runner_all_fail.py"])
9797

@@ -114,7 +114,7 @@ def test_manual_api_fake_runner_all_skip(self):
114114
replace_os_env=True,
115115
), mock.patch(
116116
"ddtrace.internal.ci_visibility._api_client._TestVisibilityAPIClientBase.fetch_settings",
117-
return_value=TestVisibilityAPISettings(False, False, False, False),
117+
return_value=TestVisibilityAPISettings(False, False, False, False, False),
118118
), mock.patch(
119119
"ddtrace.internal.ci_visibility.recorder.ddconfig", _get_default_civisibility_ddconfig()
120120
):
@@ -139,7 +139,7 @@ def test_manual_api_fake_runner_all_itr_skip_test_level(self):
139139
replace_os_env=True,
140140
), mock.patch(
141141
"ddtrace.internal.ci_visibility._api_client._TestVisibilityAPIClientBase.fetch_settings",
142-
return_value=TestVisibilityAPISettings(False, False, False, False),
142+
return_value=TestVisibilityAPISettings(False, False, False, False, False),
143143
), mock.patch(
144144
"ddtrace.internal.ci_visibility.recorder.ddconfig", _get_default_civisibility_ddconfig()
145145
):
@@ -164,7 +164,7 @@ def test_manual_api_fake_runner_all_itr_skip_suite_level(self):
164164
replace_os_env=True,
165165
), mock.patch(
166166
"ddtrace.internal.ci_visibility._api_client._TestVisibilityAPIClientBase.fetch_settings",
167-
return_value=TestVisibilityAPISettings(False, False, False, False),
167+
return_value=TestVisibilityAPISettings(False, False, False, False, False),
168168
), mock.patch(
169169
"ddtrace.internal.ci_visibility.recorder.ddconfig",
170170
_get_default_civisibility_ddconfig(itr_skipping_level=ITR_SKIPPING_LEVEL.SUITE),
@@ -190,7 +190,7 @@ def test_manual_api_fake_runner_mix_pass(self):
190190
replace_os_env=True,
191191
), mock.patch(
192192
"ddtrace.internal.ci_visibility._api_client._TestVisibilityAPIClientBase.fetch_settings",
193-
return_value=TestVisibilityAPISettings(False, False, False, False),
193+
return_value=TestVisibilityAPISettings(False, False, False, False, False),
194194
), mock.patch(
195195
"ddtrace.internal.ci_visibility.recorder.ddconfig", _get_default_civisibility_ddconfig()
196196
):
@@ -215,7 +215,7 @@ def test_manual_api_fake_runner_mix_fail(self):
215215
replace_os_env=True,
216216
), mock.patch(
217217
"ddtrace.internal.ci_visibility._api_client._TestVisibilityAPIClientBase.fetch_settings",
218-
return_value=TestVisibilityAPISettings(False, False, False, False),
218+
return_value=TestVisibilityAPISettings(False, False, False, False, False),
219219
), mock.patch(
220220
"ddtrace.internal.ci_visibility.recorder.ddconfig", _get_default_civisibility_ddconfig()
221221
):
@@ -241,7 +241,7 @@ def test_manual_api_fake_runner_mix_fail_itr_test_level(self):
241241
replace_os_env=True,
242242
), mock.patch(
243243
"ddtrace.internal.ci_visibility._api_client._TestVisibilityAPIClientBase.fetch_settings",
244-
return_value=TestVisibilityAPISettings(False, False, False, False),
244+
return_value=TestVisibilityAPISettings(False, False, False, False, False),
245245
), mock.patch(
246246
"ddtrace.internal.ci_visibility.recorder.ddconfig", _get_default_civisibility_ddconfig()
247247
):
@@ -267,7 +267,7 @@ def test_manual_api_fake_runner_mix_fail_itr_suite_level(self):
267267
replace_os_env=True,
268268
), mock.patch(
269269
"ddtrace.internal.ci_visibility._api_client._TestVisibilityAPIClientBase.fetch_settings",
270-
return_value=TestVisibilityAPISettings(False, False, False, False),
270+
return_value=TestVisibilityAPISettings(False, False, False, False, False),
271271
), mock.patch(
272272
"ddtrace.internal.ci_visibility.recorder.ddconfig", _get_default_civisibility_ddconfig()
273273
):

‎tests/ci_visibility/api_client/_util.py

+2
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,7 @@ def _get_setting_api_response(
4747
flaky_test_retries_enabled=False,
4848
efd_present=False, # This controls whether a default EFD response is present (instead of only {"enabled": false}
4949
efd_detection_enabled=False,
50+
known_tests_enabled=False,
5051
efd_5s=10,
5152
efd_10s=5,
5253
efd_30s=3,
@@ -66,6 +67,7 @@ def _get_setting_api_response(
6667
"itr_enabled": itr_enabled,
6768
"require_git": require_git,
6869
"tests_skipping": tests_skipping,
70+
"known_tests_enabled": known_tests_enabled,
6971
},
7072
}
7173
}

0 commit comments

Comments
 (0)