Skip to content

Commit 7320b6f

Browse files
feat(ci_visibility): support selenium in test visibility (#11610)
Introduces support for Selenium, including integration with the RUM product for session recording/replays (requested in #10203 ). A new `hatch` environment is added which tests support for the feature in both `v1` and `v2` versions of the `pytest` plugin, with a matching addition to test suite and specs. A few other changes are added: - properly adding the `test.type` tag when using the test API (used by manual test runners and the new pytest plugin) - adding the `type` tag when the test span is originally created (so that it may be accessed by the Selenium integration) - telemetry for events created/finished for tests is being split out into its own function because the multipurpose single function was becoming unreasonably branch-y with the addition of yet another set of tags ## Checklist - [x] PR author has checked that all the criteria below are met - The PR description includes an overview of the change - The PR description articulates the motivation for the change - The change includes tests OR the PR description describes a testing strategy - The PR description notes risks associated with the change, if any - Newly-added code is easy to change - The change follows the [library release note guidelines](https://ddtrace.readthedocs.io/en/stable/releasenotes.html) - The change includes or references documentation updates if necessary - Backport labels are set (if [applicable](https://ddtrace.readthedocs.io/en/latest/contributing.html#backporting)) ## Reviewer Checklist - [x] Reviewer has checked that all the criteria below are met - Title is accurate - All changes are related to the pull request's stated goal - Avoids breaking [API](https://ddtrace.readthedocs.io/en/stable/versioning.html#interfaces) changes - Testing strategy adequately addresses listed risks - Newly-added code is easy to change - Release note makes sense to a user of the library - If necessary, author has acknowledged and discussed the performance implications of this PR as reported in the benchmarks PR comment - Backport labels are set in a manner that is consistent with the [release branch maintenance policy](https://ddtrace.readthedocs.io/en/latest/contributing.html#backporting) --------- Co-authored-by: Vítor De Araújo <[email protected]>
1 parent 474dfb1 commit 7320b6f

File tree

38 files changed

+5823
-4296
lines changed

38 files changed

+5823
-4296
lines changed

Diff for: .github/CODEOWNERS

+6-1
Original file line numberDiff line numberDiff line change
@@ -51,7 +51,7 @@ tests/opentracer @DataDog/apm-core-python
5151
tests/runtime @DataDog/apm-core-python
5252
tests/tracer @DataDog/apm-core-python
5353

54-
# CI App and related
54+
# Test Visibility and related
5555
ddtrace/contrib/asynctest @DataDog/ci-app-libraries
5656
ddtrace/contrib/coverage @DataDog/ci-app-libraries
5757
ddtrace/contrib/pytest @DataDog/ci-app-libraries
@@ -81,6 +81,11 @@ scripts/ci_visibility/* @DataDog/ci-app-libraries
8181
ddtrace/contrib/freezegun @DataDog/ci-app-libraries
8282
ddtrace/contrib/internal/freezegun @DataDog/ci-app-libraries
8383
tests/contrib/freezegun @DataDog/ci-app-libraries
84+
# Test Visibility: Selenium integration
85+
ddtrace/contrib/selenium @DataDog/ci-app-libraries
86+
ddtrace/internal/selenium @DataDog/ci-app-libraries
87+
tests/contrib/selenium @DataDog/ci-app-libraries
88+
tests/snapshots/test_selenium_* @DataDog/ci-app-libraries
8489

8590
# Debugger
8691
ddtrace/debugging/ @DataDog/debugger-python

Diff for: ddtrace/_monkey.py

+1
Original file line numberDiff line numberDiff line change
@@ -101,6 +101,7 @@
101101
"subprocess": True,
102102
"unittest": True,
103103
"coverage": False,
104+
"selenium": True,
104105
}
105106

106107

Diff for: ddtrace/contrib/internal/selenium/patch.py

+185
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,185 @@
1+
import os
2+
import time
3+
import typing as t
4+
5+
from wrapt.importer import when_imported
6+
7+
from ddtrace import config
8+
from ddtrace.internal.logger import get_logger
9+
from ddtrace.internal.wrapping.context import WrappingContext
10+
import ddtrace.tracer
11+
12+
13+
if t.TYPE_CHECKING:
14+
import selenium.webdriver.remote.webdriver
15+
16+
log = get_logger(__name__)
17+
18+
T = t.TypeVar("T")
19+
20+
_RUM_STOP_SESSION_SCRIPT = """
21+
if (window.DD_RUM && window.DD_RUM.stopSession) {
22+
window.DD_RUM.stopSession();
23+
return true;
24+
} else {
25+
return false;
26+
}
27+
"""
28+
29+
_DEFAULT_FLUSH_SLEEP_MS = 500
30+
31+
32+
def _get_flush_sleep_ms() -> int:
33+
env_flush_sleep_ms = os.getenv("DD_CIVISIBILITY_RUM_FLUSH_WAIT_MILLIS")
34+
if env_flush_sleep_ms is None:
35+
return _DEFAULT_FLUSH_SLEEP_MS
36+
37+
try:
38+
return int(env_flush_sleep_ms)
39+
except Exception: # noqa E722
40+
log.warning(
41+
"Could not convert DD_CIVISIBILITY_RUM_FLUSH_WAIT_MILLIS value %s to int, using default: %s",
42+
env_flush_sleep_ms,
43+
_DEFAULT_FLUSH_SLEEP_MS,
44+
)
45+
return _DEFAULT_FLUSH_SLEEP_MS
46+
47+
48+
config._add(
49+
"selenium",
50+
dict(flush_sleep_ms=_get_flush_sleep_ms()),
51+
)
52+
53+
54+
class SeleniumWrappingContextBase(WrappingContext):
55+
def _handle_enter(self) -> None:
56+
pass
57+
58+
def _handle_return(self) -> None:
59+
pass
60+
61+
def _get_webdriver_instance(self) -> "selenium.webdriver.remote.webdriver.WebDriver":
62+
try:
63+
return self.get_local("self")
64+
except KeyError:
65+
log.debug("Could not get Selenium WebDriver instance")
66+
return None
67+
68+
def __enter__(self) -> "SeleniumWrappingContextBase":
69+
super().__enter__()
70+
71+
try:
72+
self._handle_enter()
73+
except Exception: # noqa: E722
74+
log.debug("Error handling selenium instrumentation enter", exc_info=True)
75+
76+
return self
77+
78+
def __return__(self, value: T) -> T:
79+
"""Always return the original value no matter what our instrumentation does"""
80+
try:
81+
self._handle_return()
82+
except Exception: # noqa: E722
83+
log.debug("Error handling instrumentation return", exc_info=True)
84+
85+
return value
86+
87+
88+
class SeleniumGetWrappingContext(SeleniumWrappingContextBase):
89+
def _handle_return(self) -> None:
90+
root_span = ddtrace.tracer.current_root_span()
91+
test_trace_id = root_span.trace_id
92+
93+
if root_span is None or root_span.get_tag("type") != "test":
94+
return
95+
96+
webdriver_instance = self._get_webdriver_instance()
97+
98+
if webdriver_instance is None:
99+
return
100+
101+
# The trace IDs for Test Visibility data using the CIVisibility protocol are 64-bit
102+
# TODO[ci_visibility]: properly identify whether to use 64 or 128 bit trace_ids
103+
trace_id_64bit = test_trace_id % 2**64
104+
105+
webdriver_instance.add_cookie({"name": "datadog-ci-visibility-test-execution-id", "value": str(trace_id_64bit)})
106+
107+
root_span.set_tag("test.is_browser", "true")
108+
root_span.set_tag("test.browser.driver", "selenium")
109+
root_span.set_tag("test.browser.driver_version", get_version())
110+
111+
# Submit empty values for browser names or version if multiple are found
112+
browser_name = webdriver_instance.capabilities.get("browserName")
113+
browser_version = webdriver_instance.capabilities.get("browserVersion")
114+
115+
existing_browser_name = root_span.get_tag("test.browser.name")
116+
if existing_browser_name is None:
117+
root_span.set_tag("test.browser.name", browser_name)
118+
elif existing_browser_name not in ["", browser_name]:
119+
root_span.set_tag("test.browser.name", "")
120+
121+
existing_browser_version = root_span.get_tag("test.browser.version")
122+
if existing_browser_version is None:
123+
root_span.set_tag("test.browser.version", browser_version)
124+
elif existing_browser_version not in ["", browser_version]:
125+
root_span.set_tag("test.browser.version", "")
126+
127+
128+
class SeleniumQuitWrappingContext(SeleniumWrappingContextBase):
129+
def _handle_enter(self) -> None:
130+
root_span = ddtrace.tracer.current_root_span()
131+
132+
if root_span is None or root_span.get_tag("type") != "test":
133+
return
134+
135+
webdriver_instance = self._get_webdriver_instance()
136+
137+
if webdriver_instance is None:
138+
return
139+
140+
is_rum_active = webdriver_instance.execute_script(_RUM_STOP_SESSION_SCRIPT)
141+
time.sleep(config.selenium.flush_sleep_ms / 1000)
142+
143+
if is_rum_active:
144+
root_span.set_tag("test.is_rum_active", "true")
145+
146+
webdriver_instance.delete_cookie("datadog-ci-visibility-test-execution-id")
147+
148+
149+
def get_version() -> str:
150+
import selenium
151+
152+
try:
153+
return selenium.__version__
154+
except AttributeError:
155+
log.debug("Could not get Selenium version")
156+
return ""
157+
158+
159+
def patch() -> None:
160+
import selenium
161+
162+
if getattr(selenium, "_datadog_patch", False):
163+
return
164+
165+
@when_imported("selenium.webdriver.remote.webdriver")
166+
def _(m):
167+
SeleniumGetWrappingContext(m.WebDriver.get).wrap()
168+
SeleniumQuitWrappingContext(m.WebDriver.quit).wrap()
169+
SeleniumQuitWrappingContext(m.WebDriver.close).wrap()
170+
171+
selenium._datadog_patch = True
172+
173+
174+
def unpatch() -> None:
175+
import selenium
176+
from selenium.webdriver.remote.webdriver import WebDriver
177+
178+
if not getattr(selenium, "_datadog_patch", False):
179+
return
180+
181+
SeleniumGetWrappingContext.extract(WebDriver.get).unwrap()
182+
SeleniumQuitWrappingContext.extract(WebDriver.quit).unwrap()
183+
SeleniumQuitWrappingContext.extract(WebDriver.close).unwrap()
184+
185+
selenium._datadog_patch = False

Diff for: ddtrace/contrib/selenium/__init__.py

+37
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
"""
2+
The Selenium integration enriches Test Visibility data with extra tags and, if available,
3+
Real User Monitoring session replays.
4+
5+
Enabling
6+
~~~~~~~~
7+
8+
The Selenium integration is enabled by default in test contexts (eg: pytest, or unittest). Use
9+
:func:`patch()<ddtrace.patch>` to enable the integration::
10+
11+
from ddtrace import patch
12+
patch(selenium=True)
13+
14+
15+
When using pytest, the `--ddtrace-patch-all` flag is required in order for this integration to
16+
be enabled.
17+
18+
Configuration
19+
~~~~~~~~~~~~~
20+
21+
The Selenium integration can be configured using the following options:
22+
23+
DD_CIVISIBILITY_RUM_FLUSH_WAIT_MILLIS: The time in milliseconds to wait after flushing the RUM session.
24+
"""
25+
from ...internal.utils.importlib import require_modules
26+
27+
28+
required_modules = ["selenium"]
29+
30+
with require_modules(required_modules) as missing_modules:
31+
if not missing_modules:
32+
# Expose public methods
33+
from ..internal.selenium.patch import get_version
34+
from ..internal.selenium.patch import patch
35+
from ..internal.selenium.patch import unpatch
36+
37+
__all__ = ["get_version", "patch", "unpatch"]

Diff for: ddtrace/internal/ci_visibility/api/_base.py

+3-2
Original file line numberDiff line numberDiff line change
@@ -183,6 +183,9 @@ def _start_span(self) -> None:
183183
span_type=SpanTypes.TEST,
184184
activate=True,
185185
)
186+
# Setting initial tags is necessary for integrations that might look at the span before it is finished
187+
self._span.set_tag(EVENT_TYPE, self._event_type)
188+
self._span.set_tag(SPAN_KIND, "test")
186189
log.debug("Started span %s for item %s", self._span, self)
187190

188191
@_require_span
@@ -219,8 +222,6 @@ def _set_default_tags(self) -> None:
219222

220223
self.set_tags(
221224
{
222-
EVENT_TYPE: self._event_type,
223-
SPAN_KIND: "test",
224225
COMPONENT: self._session_settings.test_framework,
225226
test.FRAMEWORK: self._session_settings.test_framework,
226227
test.FRAMEWORK_VERSION: self._session_settings.test_framework_version,

Diff for: ddtrace/internal/ci_visibility/api/_test.py

+26-9
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@
55
from typing import Optional
66
from typing import Union
77

8+
from ddtrace.ext import SpanTypes
89
from ddtrace.ext import test
910
from ddtrace.ext.test_visibility import ITR_SKIPPING_LEVEL
1011
from ddtrace.ext.test_visibility._item_ids import TestId
@@ -21,8 +22,8 @@
2122
from ddtrace.internal.ci_visibility.constants import TEST_IS_NEW
2223
from ddtrace.internal.ci_visibility.constants import TEST_IS_RETRY
2324
from ddtrace.internal.ci_visibility.telemetry.constants import EVENT_TYPES
24-
from ddtrace.internal.ci_visibility.telemetry.events import record_event_created
25-
from ddtrace.internal.ci_visibility.telemetry.events import record_event_finished
25+
from ddtrace.internal.ci_visibility.telemetry.events import record_event_created_test
26+
from ddtrace.internal.ci_visibility.telemetry.events import record_event_finished_test
2627
from ddtrace.internal.logger import get_logger
2728
from ddtrace.internal.test_visibility._efd_mixins import EFDTestStatus
2829
from ddtrace.internal.test_visibility._internal_item_ids import InternalTestId
@@ -119,20 +120,20 @@ def _set_span_tags(self) -> None:
119120
self._span.set_exc_info(self._exc_info.exc_type, self._exc_info.exc_value, self._exc_info.exc_traceback)
120121

121122
def _telemetry_record_event_created(self):
122-
record_event_created(
123-
event_type=self._event_type_metric_name,
123+
record_event_created_test(
124124
test_framework=self._session_settings.test_framework_metric_name,
125-
is_benchmark=self._is_benchmark if self._is_benchmark is not None else None,
125+
is_benchmark=self._is_benchmark,
126126
)
127127

128128
def _telemetry_record_event_finished(self):
129-
record_event_finished(
130-
event_type=self._event_type_metric_name,
129+
record_event_finished_test(
131130
test_framework=self._session_settings.test_framework_metric_name,
132-
is_benchmark=self._is_benchmark if self._is_benchmark is not None else None,
133-
is_new=self._is_new if self._is_new is not None else None,
131+
is_benchmark=self._is_benchmark,
132+
is_new=self.is_new(),
134133
is_retry=self._efd_is_retry or self._atr_is_retry,
135134
early_flake_detection_abort_reason=self._efd_abort_reason,
135+
is_rum=self._is_rum(),
136+
browser_driver=self._get_browser_driver(),
136137
)
137138

138139
def finish_test(
@@ -143,6 +144,9 @@ def finish_test(
143144
override_finish_time: Optional[float] = None,
144145
) -> None:
145146
log.debug("Test Visibility: finishing %s, with status: %s, reason: %s", self, status, reason)
147+
148+
self.set_tag(test.TYPE, SpanTypes.TEST)
149+
146150
if status is not None:
147151
self.set_status(status)
148152
if reason is not None:
@@ -379,3 +383,16 @@ def atr_get_final_status(self) -> TestStatus:
379383
return TestStatus.PASS
380384

381385
return TestStatus.FAIL
386+
387+
#
388+
# Selenium / RUM functionality
389+
#
390+
def _is_rum(self):
391+
if self._span is None:
392+
return False
393+
return self._span.get_tag("is_rum_active") == "true"
394+
395+
def _get_browser_driver(self):
396+
if self._span is None:
397+
return None
398+
return self._span.get_tag("test.browser.driver")

0 commit comments

Comments
 (0)