Skip to content

Commit

Permalink
Ensure profiling is fine-grained in the Regression Detector (#20081)
Browse files Browse the repository at this point in the history
* Ensure profiling is fine-grained in the Regression Detector

This commit updates the environment flags on our experiments to ensure that
profiling runs are fine grained through the whole of the run. Updated with
reference to the ongoing investigation in #19990

REF SMPTNG-12

Signed-off-by: Brian L. Troutwine <[email protected]>

* CPUDURATION -> CPU_DURATION

---------

Signed-off-by: Brian L. Troutwine <[email protected]>
Co-authored-by: George Hahn <[email protected]>
  • Loading branch information
blt and GeorgeHahn authored Oct 12, 2023
1 parent 8ec5e1c commit 2a2ba63
Show file tree
Hide file tree
Showing 14 changed files with 115 additions and 67 deletions.
13 changes: 8 additions & 5 deletions test/regression/cases/file_to_blackhole/experiment.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -5,13 +5,16 @@ environment:
DD_TELEMETRY_ENABLED: true

profiling_environment:
DD_INTERNAL_PROFILING_ENABLED: true
DD_INTERNAL_PROFILING_UNIX_SOCKET: /var/run/datadog/apm.socket
DD_INTERNAL_PROFILING_BLOCK_PROFILE_RATE: 10000
DD_INTERNAL_PROFILING_CPU_DURATION: 1m
DD_INTERNAL_PROFILING_DELTA_PROFILES: true
DD_INTERNAL_PROFILING_ENABLED: true
DD_INTERNAL_PROFILING_ENABLE_GOROUTINE_STACKTRACES: true
DD_INTERNAL_PROFILING_EXTRA_TAGS: experiment:file_to_blackhole
DD_INTERNAL_PROFILING_MUTEX_PROFILE_FRACTION: 10
DD_INTERNAL_PROFILING_PERIOD: 1m
DD_INTERNAL_PROFILING_UNIX_SOCKET: /var/run/datadog/apm.socket
DD_PROFILING_EXECUTION_TRACE_ENABLED: true
DD_PROFILING_EXECUTION_TRACE_PERIOD: 1m
DD_PROFILING_WAIT_PROFILE: true
DD_INTERNAL_PROFILING_BLOCK_PROFILE_RATE: 10000
DD_INTERNAL_PROFILING_MUTEX_PROFILE_FRACTION: 10

DD_INTERNAL_PROFILING_EXTRA_TAGS: experiment:file_to_blackhole
13 changes: 11 additions & 2 deletions test/regression/cases/file_tree/experiment.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,16 @@ environment:
DD_RUNTIME_SECURITY_CONFIG_REMOTE_CONFIGURATION_ENABLED: true

profiling_environment:
DD_INTERNAL_PROFILING_ENABLED: true
DD_INTERNAL_PROFILING_UNIX_SOCKET: /var/run/datadog/apm.socket
DD_INTERNAL_PROFILING_BLOCK_PROFILE_RATE: 10000
DD_INTERNAL_PROFILING_CPU_DURATION: 1m
DD_INTERNAL_PROFILING_DELTA_PROFILES: true
DD_INTERNAL_PROFILING_ENABLED: true
DD_INTERNAL_PROFILING_ENABLE_GOROUTINE_STACKTRACES: true
DD_INTERNAL_PROFILING_MUTEX_PROFILE_FRACTION: 10
DD_INTERNAL_PROFILING_PERIOD: 1m
DD_INTERNAL_PROFILING_UNIX_SOCKET: /var/run/datadog/apm.socket
DD_PROFILING_EXECUTION_TRACE_ENABLED: true
DD_PROFILING_EXECUTION_TRACE_PERIOD: 1m
DD_PROFILING_WAIT_PROFILE: true

DD_INTERNAL_PROFILING_EXTRA_TAGS: experiment:file_tree
13 changes: 8 additions & 5 deletions test/regression/cases/otel_to_otel_logs/experiment.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -5,13 +5,16 @@ environment:
DD_TELEMETRY_ENABLED: true

profiling_environment:
DD_INTERNAL_PROFILING_ENABLED: true
DD_INTERNAL_PROFILING_UNIX_SOCKET: /var/run/datadog/apm.socket
DD_INTERNAL_PROFILING_BLOCK_PROFILE_RATE: 10000
DD_INTERNAL_PROFILING_CPU_DURATION: 1m
DD_INTERNAL_PROFILING_DELTA_PROFILES: true
DD_INTERNAL_PROFILING_ENABLED: true
DD_INTERNAL_PROFILING_ENABLE_GOROUTINE_STACKTRACES: true
DD_INTERNAL_PROFILING_EXTRA_TAGS: experiment:otel_to_otel_logs
DD_INTERNAL_PROFILING_MUTEX_PROFILE_FRACTION: 10
DD_INTERNAL_PROFILING_PERIOD: 1m
DD_INTERNAL_PROFILING_UNIX_SOCKET: /var/run/datadog/apm.socket
DD_PROFILING_EXECUTION_TRACE_ENABLED: true
DD_PROFILING_EXECUTION_TRACE_PERIOD: 1m
DD_PROFILING_WAIT_PROFILE: true
DD_INTERNAL_PROFILING_BLOCK_PROFILE_RATE: 10000
DD_INTERNAL_PROFILING_MUTEX_PROFILE_FRACTION: 10

DD_INTERNAL_PROFILING_EXTRA_TAGS: experiment:otel_to_otel_logs
13 changes: 8 additions & 5 deletions test/regression/cases/tcp_dd_logs_filter_exclude/experiment.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -5,13 +5,16 @@ environment:
DD_TELEMETRY_ENABLED: true

profiling_environment:
DD_INTERNAL_PROFILING_ENABLED: true
DD_INTERNAL_PROFILING_UNIX_SOCKET: /var/run/datadog/apm.socket
DD_INTERNAL_PROFILING_BLOCK_PROFILE_RATE: 10000
DD_INTERNAL_PROFILING_CPU_DURATION: 1m
DD_INTERNAL_PROFILING_DELTA_PROFILES: true
DD_INTERNAL_PROFILING_ENABLED: true
DD_INTERNAL_PROFILING_ENABLE_GOROUTINE_STACKTRACES: true
DD_INTERNAL_PROFILING_EXTRA_TAGS: experiment:tcp_dd_logs_filter_exclude
DD_INTERNAL_PROFILING_MUTEX_PROFILE_FRACTION: 10
DD_INTERNAL_PROFILING_PERIOD: 1m
DD_INTERNAL_PROFILING_UNIX_SOCKET: /var/run/datadog/apm.socket
DD_PROFILING_EXECUTION_TRACE_ENABLED: true
DD_PROFILING_EXECUTION_TRACE_PERIOD: 1m
DD_PROFILING_WAIT_PROFILE: true
DD_INTERNAL_PROFILING_BLOCK_PROFILE_RATE: 10000
DD_INTERNAL_PROFILING_MUTEX_PROFILE_FRACTION: 10

DD_INTERNAL_PROFILING_EXTRA_TAGS: experiment:tcp_dd_logs_filter_exclude
13 changes: 8 additions & 5 deletions test/regression/cases/tcp_syslog_to_blackhole/experiment.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -5,13 +5,16 @@ environment:
DD_TELEMETRY_ENABLED: true

profiling_environment:
DD_INTERNAL_PROFILING_ENABLED: true
DD_INTERNAL_PROFILING_UNIX_SOCKET: /var/run/datadog/apm.socket
DD_INTERNAL_PROFILING_BLOCK_PROFILE_RATE: 10000
DD_INTERNAL_PROFILING_CPU_DURATION: 1m
DD_INTERNAL_PROFILING_DELTA_PROFILES: true
DD_INTERNAL_PROFILING_ENABLED: true
DD_INTERNAL_PROFILING_ENABLE_GOROUTINE_STACKTRACES: true
DD_INTERNAL_PROFILING_EXTRA_TAGS: experiment:tcp_syslog_to_blackhole
DD_INTERNAL_PROFILING_MUTEX_PROFILE_FRACTION: 10
DD_INTERNAL_PROFILING_PERIOD: 1m
DD_INTERNAL_PROFILING_UNIX_SOCKET: /var/run/datadog/apm.socket
DD_PROFILING_EXECUTION_TRACE_ENABLED: true
DD_PROFILING_EXECUTION_TRACE_PERIOD: 1m
DD_PROFILING_WAIT_PROFILE: true
DD_INTERNAL_PROFILING_BLOCK_PROFILE_RATE: 10000
DD_INTERNAL_PROFILING_MUTEX_PROFILE_FRACTION: 10

DD_INTERNAL_PROFILING_EXTRA_TAGS: experiment:tcp_syslog_to_blackhole
13 changes: 8 additions & 5 deletions test/regression/cases/trace_agent_json/experiment.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -5,13 +5,16 @@ environment:
DD_TELEMETRY_ENABLED: true

profiling_environment:
DD_INTERNAL_PROFILING_ENABLED: true
DD_INTERNAL_PROFILING_UNIX_SOCKET: /var/run/datadog/apm.socket
DD_INTERNAL_PROFILING_BLOCK_PROFILE_RATE: 10000
DD_INTERNAL_PROFILING_CPU_DURATION: 1m
DD_INTERNAL_PROFILING_DELTA_PROFILES: true
DD_INTERNAL_PROFILING_ENABLED: true
DD_INTERNAL_PROFILING_ENABLE_GOROUTINE_STACKTRACES: true
DD_INTERNAL_PROFILING_EXTRA_TAGS: experiment:trace_agent_json
DD_INTERNAL_PROFILING_MUTEX_PROFILE_FRACTION: 10
DD_INTERNAL_PROFILING_PERIOD: 1m
DD_INTERNAL_PROFILING_UNIX_SOCKET: /var/run/datadog/apm.socket
DD_PROFILING_EXECUTION_TRACE_ENABLED: true
DD_PROFILING_EXECUTION_TRACE_PERIOD: 1m
DD_PROFILING_WAIT_PROFILE: true
DD_INTERNAL_PROFILING_BLOCK_PROFILE_RATE: 10000
DD_INTERNAL_PROFILING_MUTEX_PROFILE_FRACTION: 10

DD_INTERNAL_PROFILING_EXTRA_TAGS: experiment:trace_agent_json
13 changes: 8 additions & 5 deletions test/regression/cases/trace_agent_msgpack/experiment.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -5,13 +5,16 @@ environment:
DD_TELEMETRY_ENABLED: true

profiling_environment:
DD_INTERNAL_PROFILING_ENABLED: true
DD_INTERNAL_PROFILING_UNIX_SOCKET: /var/run/datadog/apm.socket
DD_INTERNAL_PROFILING_BLOCK_PROFILE_RATE: 10000
DD_INTERNAL_PROFILING_CPU_DURATION: 1m
DD_INTERNAL_PROFILING_DELTA_PROFILES: true
DD_INTERNAL_PROFILING_ENABLED: true
DD_INTERNAL_PROFILING_ENABLE_GOROUTINE_STACKTRACES: true
DD_INTERNAL_PROFILING_EXTRA_TAGS: experiment:trace_agent_msgpack
DD_INTERNAL_PROFILING_MUTEX_PROFILE_FRACTION: 10
DD_INTERNAL_PROFILING_PERIOD: 1m
DD_INTERNAL_PROFILING_UNIX_SOCKET: /var/run/datadog/apm.socket
DD_PROFILING_EXECUTION_TRACE_ENABLED: true
DD_PROFILING_EXECUTION_TRACE_PERIOD: 1m
DD_PROFILING_WAIT_PROFILE: true
DD_INTERNAL_PROFILING_BLOCK_PROFILE_RATE: 10000
DD_INTERNAL_PROFILING_MUTEX_PROFILE_FRACTION: 10

DD_INTERNAL_PROFILING_EXTRA_TAGS: experiment:trace_agent_msgpack
13 changes: 8 additions & 5 deletions test/regression/cases/uds_dogstatsd_to_api/experiment.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -5,13 +5,16 @@ environment:
DD_TELEMETRY_ENABLED: true

profiling_environment:
DD_INTERNAL_PROFILING_ENABLED: true
DD_INTERNAL_PROFILING_UNIX_SOCKET: /var/run/datadog/apm.socket
DD_INTERNAL_PROFILING_BLOCK_PROFILE_RATE: 10000
DD_INTERNAL_PROFILING_CPU_DURATION: 1m
DD_INTERNAL_PROFILING_DELTA_PROFILES: true
DD_INTERNAL_PROFILING_ENABLED: true
DD_INTERNAL_PROFILING_ENABLE_GOROUTINE_STACKTRACES: true
DD_INTERNAL_PROFILING_EXTRA_TAGS: experiment:uds_dogstatsd_to_api
DD_INTERNAL_PROFILING_MUTEX_PROFILE_FRACTION: 10
DD_INTERNAL_PROFILING_PERIOD: 1m
DD_INTERNAL_PROFILING_UNIX_SOCKET: /var/run/datadog/apm.socket
DD_PROFILING_EXECUTION_TRACE_ENABLED: true
DD_PROFILING_EXECUTION_TRACE_PERIOD: 1m
DD_PROFILING_WAIT_PROFILE: true
DD_INTERNAL_PROFILING_BLOCK_PROFILE_RATE: 10000
DD_INTERNAL_PROFILING_MUTEX_PROFILE_FRACTION: 10

DD_INTERNAL_PROFILING_EXTRA_TAGS: experiment:uds_dogstatsd_to_api
Original file line number Diff line number Diff line change
Expand Up @@ -5,13 +5,16 @@ environment:
DD_TELEMETRY_ENABLED: true

profiling_environment:
DD_INTERNAL_PROFILING_ENABLED: true
DD_INTERNAL_PROFILING_UNIX_SOCKET: /var/run/datadog/apm.socket
DD_INTERNAL_PROFILING_BLOCK_PROFILE_RATE: 10000
DD_INTERNAL_PROFILING_CPU_DURATION: 1m
DD_INTERNAL_PROFILING_DELTA_PROFILES: true
DD_INTERNAL_PROFILING_ENABLED: true
DD_INTERNAL_PROFILING_ENABLE_GOROUTINE_STACKTRACES: true
DD_INTERNAL_PROFILING_EXTRA_TAGS: experiment:uds_dogstatsd_to_api_nodist_100MiB
DD_INTERNAL_PROFILING_MUTEX_PROFILE_FRACTION: 10
DD_INTERNAL_PROFILING_PERIOD: 1m
DD_INTERNAL_PROFILING_UNIX_SOCKET: /var/run/datadog/apm.socket
DD_PROFILING_EXECUTION_TRACE_ENABLED: true
DD_PROFILING_EXECUTION_TRACE_PERIOD: 1m
DD_PROFILING_WAIT_PROFILE: true
DD_INTERNAL_PROFILING_BLOCK_PROFILE_RATE: 10000
DD_INTERNAL_PROFILING_MUTEX_PROFILE_FRACTION: 10

DD_INTERNAL_PROFILING_EXTRA_TAGS: experiment:uds_dogstatsd_to_api_nodist_100MiB
Original file line number Diff line number Diff line change
Expand Up @@ -5,13 +5,16 @@ environment:
DD_TELEMETRY_ENABLED: true

profiling_environment:
DD_INTERNAL_PROFILING_ENABLED: true
DD_INTERNAL_PROFILING_UNIX_SOCKET: /var/run/datadog/apm.socket
DD_INTERNAL_PROFILING_BLOCK_PROFILE_RATE: 10000
DD_INTERNAL_PROFILING_CPU_DURATION: 1m
DD_INTERNAL_PROFILING_DELTA_PROFILES: true
DD_INTERNAL_PROFILING_ENABLED: true
DD_INTERNAL_PROFILING_ENABLE_GOROUTINE_STACKTRACES: true
DD_INTERNAL_PROFILING_EXTRA_TAGS: experiment:uds_dogstatsd_to_api_nodist_16MiB
DD_INTERNAL_PROFILING_MUTEX_PROFILE_FRACTION: 10
DD_INTERNAL_PROFILING_PERIOD: 1m
DD_INTERNAL_PROFILING_UNIX_SOCKET: /var/run/datadog/apm.socket
DD_PROFILING_EXECUTION_TRACE_ENABLED: true
DD_PROFILING_EXECUTION_TRACE_PERIOD: 1m
DD_PROFILING_WAIT_PROFILE: true
DD_INTERNAL_PROFILING_BLOCK_PROFILE_RATE: 10000
DD_INTERNAL_PROFILING_MUTEX_PROFILE_FRACTION: 10

DD_INTERNAL_PROFILING_EXTRA_TAGS: experiment:uds_dogstatsd_to_api_nodist_16MiB
Original file line number Diff line number Diff line change
Expand Up @@ -5,13 +5,16 @@ environment:
DD_TELEMETRY_ENABLED: true

profiling_environment:
DD_INTERNAL_PROFILING_ENABLED: true
DD_INTERNAL_PROFILING_UNIX_SOCKET: /var/run/datadog/apm.socket
DD_INTERNAL_PROFILING_BLOCK_PROFILE_RATE: 10000
DD_INTERNAL_PROFILING_CPU_DURATION: 1m
DD_INTERNAL_PROFILING_DELTA_PROFILES: true
DD_INTERNAL_PROFILING_ENABLED: true
DD_INTERNAL_PROFILING_ENABLE_GOROUTINE_STACKTRACES: true
DD_INTERNAL_PROFILING_EXTRA_TAGS: experiment:uds_dogstatsd_to_api_nodist_1MiB
DD_INTERNAL_PROFILING_MUTEX_PROFILE_FRACTION: 10
DD_INTERNAL_PROFILING_PERIOD: 1m
DD_INTERNAL_PROFILING_UNIX_SOCKET: /var/run/datadog/apm.socket
DD_PROFILING_EXECUTION_TRACE_ENABLED: true
DD_PROFILING_EXECUTION_TRACE_PERIOD: 1m
DD_PROFILING_WAIT_PROFILE: true
DD_INTERNAL_PROFILING_BLOCK_PROFILE_RATE: 10000
DD_INTERNAL_PROFILING_MUTEX_PROFILE_FRACTION: 10

DD_INTERNAL_PROFILING_EXTRA_TAGS: experiment:uds_dogstatsd_to_api_nodist_1MiB
Original file line number Diff line number Diff line change
Expand Up @@ -5,13 +5,16 @@ environment:
DD_TELEMETRY_ENABLED: true

profiling_environment:
DD_INTERNAL_PROFILING_ENABLED: true
DD_INTERNAL_PROFILING_UNIX_SOCKET: /var/run/datadog/apm.socket
DD_INTERNAL_PROFILING_BLOCK_PROFILE_RATE: 10000
DD_INTERNAL_PROFILING_CPU_DURATION: 1m
DD_INTERNAL_PROFILING_DELTA_PROFILES: true
DD_INTERNAL_PROFILING_ENABLED: true
DD_INTERNAL_PROFILING_ENABLE_GOROUTINE_STACKTRACES: true
DD_INTERNAL_PROFILING_EXTRA_TAGS: experiment:uds_dogstatsd_to_api_nodist_200MiB
DD_INTERNAL_PROFILING_MUTEX_PROFILE_FRACTION: 10
DD_INTERNAL_PROFILING_PERIOD: 1m
DD_INTERNAL_PROFILING_UNIX_SOCKET: /var/run/datadog/apm.socket
DD_PROFILING_EXECUTION_TRACE_ENABLED: true
DD_PROFILING_EXECUTION_TRACE_PERIOD: 1m
DD_PROFILING_WAIT_PROFILE: true
DD_INTERNAL_PROFILING_BLOCK_PROFILE_RATE: 10000
DD_INTERNAL_PROFILING_MUTEX_PROFILE_FRACTION: 10

DD_INTERNAL_PROFILING_EXTRA_TAGS: experiment:uds_dogstatsd_to_api_nodist_200MiB
Original file line number Diff line number Diff line change
Expand Up @@ -5,13 +5,16 @@ environment:
DD_TELEMETRY_ENABLED: true

profiling_environment:
DD_INTERNAL_PROFILING_ENABLED: true
DD_INTERNAL_PROFILING_UNIX_SOCKET: /var/run/datadog/apm.socket
DD_INTERNAL_PROFILING_BLOCK_PROFILE_RATE: 10000
DD_INTERNAL_PROFILING_CPU_DURATION: 1m
DD_INTERNAL_PROFILING_DELTA_PROFILES: true
DD_INTERNAL_PROFILING_ENABLED: true
DD_INTERNAL_PROFILING_ENABLE_GOROUTINE_STACKTRACES: true
DD_INTERNAL_PROFILING_EXTRA_TAGS: experiment:uds_dogstatsd_to_api_nodist_32MiB
DD_INTERNAL_PROFILING_MUTEX_PROFILE_FRACTION: 10
DD_INTERNAL_PROFILING_PERIOD: 1m
DD_INTERNAL_PROFILING_UNIX_SOCKET: /var/run/datadog/apm.socket
DD_PROFILING_EXECUTION_TRACE_ENABLED: true
DD_PROFILING_EXECUTION_TRACE_PERIOD: 1m
DD_PROFILING_WAIT_PROFILE: true
DD_INTERNAL_PROFILING_BLOCK_PROFILE_RATE: 10000
DD_INTERNAL_PROFILING_MUTEX_PROFILE_FRACTION: 10

DD_INTERNAL_PROFILING_EXTRA_TAGS: experiment:uds_dogstatsd_to_api_nodist_32MiB
Original file line number Diff line number Diff line change
Expand Up @@ -5,13 +5,16 @@ environment:
DD_TELEMETRY_ENABLED: true

profiling_environment:
DD_INTERNAL_PROFILING_ENABLED: true
DD_INTERNAL_PROFILING_UNIX_SOCKET: /var/run/datadog/apm.socket
DD_INTERNAL_PROFILING_BLOCK_PROFILE_RATE: 10000
DD_INTERNAL_PROFILING_CPUDURATION: 1m
DD_INTERNAL_PROFILING_DELTA_PROFILES: true
DD_INTERNAL_PROFILING_ENABLED: true
DD_INTERNAL_PROFILING_ENABLE_GOROUTINE_STACKTRACES: true
DD_INTERNAL_PROFILING_EXTRA_TAGS: experiment:uds_dogstatsd_to_api_nodist_64MiB
DD_INTERNAL_PROFILING_MUTEX_PROFILE_FRACTION: 10
DD_INTERNAL_PROFILING_PERIOD: 1m
DD_INTERNAL_PROFILING_UNIX_SOCKET: /var/run/datadog/apm.socket
DD_PROFILING_EXECUTION_TRACE_ENABLED: true
DD_PROFILING_EXECUTION_TRACE_PERIOD: 1m
DD_PROFILING_WAIT_PROFILE: true
DD_INTERNAL_PROFILING_BLOCK_PROFILE_RATE: 10000
DD_INTERNAL_PROFILING_MUTEX_PROFILE_FRACTION: 10

DD_INTERNAL_PROFILING_EXTRA_TAGS: experiment:uds_dogstatsd_to_api_nodist_64MiB

0 comments on commit 2a2ba63

Please sign in to comment.