From 731f5c33f244abe7431921a97b2f3a24d44e2ecf Mon Sep 17 00:00:00 2001 From: Olivier John Ndjike Nzia Date: Thu, 6 Nov 2025 10:56:34 -0500 Subject: [PATCH] create json-utils --- .../tagprocessor/PayloadTagsProcessor.java | 31 +++++-------- .../PayloadTagsProcessorTest.groovy | 2 +- .../json/JsonPathParserSpec.groovy | 2 + .../payloadtags/json/JsonPathSpec.groovy | 2 + .../payloadtags/json/PathCursorSpec.groovy | 1 + internal-api/build.gradle.kts | 1 + .../main/java/datadog/trace/api/Config.java | 45 ++++++++++++++++--- settings.gradle.kts | 1 + utils/json-utils/build.gradle.kts | 17 +++++++ .../src/main/java/datadog}/json/JsonPath.java | 2 +- .../java/datadog}/json/JsonPathParser.java | 26 ++++++++++- .../java/datadog}/json/JsonStreamParser.java | 2 +- .../main/java/datadog}/json/PathCursor.java | 2 +- 13 files changed, 102 insertions(+), 32 deletions(-) create mode 100644 utils/json-utils/build.gradle.kts rename {dd-trace-core/src/main/java/datadog/trace/payloadtags => utils/json-utils/src/main/java/datadog}/json/JsonPath.java (99%) rename {dd-trace-core/src/main/java/datadog/trace/payloadtags => utils/json-utils/src/main/java/datadog}/json/JsonPathParser.java (91%) rename {dd-trace-core/src/main/java/datadog/trace/payloadtags => utils/json-utils/src/main/java/datadog}/json/JsonStreamParser.java (99%) rename {dd-trace-core/src/main/java/datadog/trace/payloadtags => utils/json-utils/src/main/java/datadog}/json/PathCursor.java (97%) diff --git a/dd-trace-core/src/main/java/datadog/trace/core/tagprocessor/PayloadTagsProcessor.java b/dd-trace-core/src/main/java/datadog/trace/core/tagprocessor/PayloadTagsProcessor.java index a52fa938c3e..68d6d2a83fc 100644 --- a/dd-trace-core/src/main/java/datadog/trace/core/tagprocessor/PayloadTagsProcessor.java +++ b/dd-trace-core/src/main/java/datadog/trace/core/tagprocessor/PayloadTagsProcessor.java @@ -1,5 +1,10 @@ package datadog.trace.core.tagprocessor; +import static datadog.json.JsonPathParser.parseJsonPaths; + +import datadog.json.JsonPath; +import datadog.json.JsonStreamParser; +import datadog.json.PathCursor; import datadog.trace.api.Config; import datadog.trace.api.ConfigDefaults; import datadog.trace.api.TagMap; @@ -7,13 +12,8 @@ import datadog.trace.bootstrap.instrumentation.api.AgentSpanLink; import datadog.trace.core.DDSpanContext; import datadog.trace.payloadtags.PayloadTagsData; -import datadog.trace.payloadtags.json.JsonPath; -import datadog.trace.payloadtags.json.JsonPathParser; -import datadog.trace.payloadtags.json.JsonStreamParser; -import datadog.trace.payloadtags.json.PathCursor; import java.io.InputStream; import java.util.ArrayList; -import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -38,7 +38,7 @@ public static PayloadTagsProcessor create(Config config) { new RedactionRules.Builder() .addRedactionJsonPaths(ConfigDefaults.DEFAULT_CLOUD_COMMON_PAYLOAD_TAGGING) .addRedactionJsonPaths(ConfigDefaults.DEFAULT_CLOUD_REQUEST_PAYLOAD_TAGGING) - .addRedactionJsonPaths(config.getCloudRequestPayloadTagging()) + .addParsedRedactionJsonPaths(config.getCloudRequestPayloadTagging()) .build()); } if (config.isCloudResponsePayloadTaggingEnabled()) { @@ -47,7 +47,7 @@ public static PayloadTagsProcessor create(Config config) { new RedactionRules.Builder() .addRedactionJsonPaths(ConfigDefaults.DEFAULT_CLOUD_COMMON_PAYLOAD_TAGGING) .addRedactionJsonPaths(ConfigDefaults.DEFAULT_CLOUD_RESPONSE_PAYLOAD_TAGGING) - .addRedactionJsonPaths(config.getCloudResponsePayloadTagging()) + .addParsedRedactionJsonPaths(config.getCloudResponsePayloadTagging()) .build()); } if (redactionRulesByTagPrefix.isEmpty()) { @@ -145,20 +145,9 @@ public RedactionRules.Builder addRedactionJsonPaths(List jsonPaths) { return this; } - private static List parseJsonPaths(List rules) { - if (rules.isEmpty() || rules.size() == 1 && rules.get(0).equalsIgnoreCase("all")) { - return Collections.emptyList(); - } - List result = new ArrayList<>(rules.size()); - for (String rule : rules) { - try { - JsonPath jp = JsonPathParser.parse(rule); - result.add(jp); - } catch (Exception ex) { - log.warn("Skipping failed to parse redaction rule '{}'. {}", rule, ex.getMessage()); - } - } - return result; + public RedactionRules.Builder addParsedRedactionJsonPaths(List jsonPaths) { + this.redactionRules.addAll(jsonPaths); + return this; } RedactionRules build() { diff --git a/dd-trace-core/src/test/groovy/datadog/trace/core/tagprocessor/PayloadTagsProcessorTest.groovy b/dd-trace-core/src/test/groovy/datadog/trace/core/tagprocessor/PayloadTagsProcessorTest.groovy index 40ef22a3f4e..dd45c2c07b6 100644 --- a/dd-trace-core/src/test/groovy/datadog/trace/core/tagprocessor/PayloadTagsProcessorTest.groovy +++ b/dd-trace-core/src/test/groovy/datadog/trace/core/tagprocessor/PayloadTagsProcessorTest.groovy @@ -3,7 +3,7 @@ package datadog.trace.core.tagprocessor import com.squareup.moshi.JsonWriter import datadog.trace.payloadtags.PayloadTagsData import datadog.trace.payloadtags.PayloadTagsData.PathAndValue -import datadog.trace.payloadtags.json.PathCursor +import datadog.json.PathCursor import datadog.trace.test.util.DDSpecification import datadog.trace.api.Config import okio.Buffer diff --git a/dd-trace-core/src/test/groovy/datadog/trace/payloadtags/json/JsonPathParserSpec.groovy b/dd-trace-core/src/test/groovy/datadog/trace/payloadtags/json/JsonPathParserSpec.groovy index d2c161426f9..a48788b6db2 100644 --- a/dd-trace-core/src/test/groovy/datadog/trace/payloadtags/json/JsonPathParserSpec.groovy +++ b/dd-trace-core/src/test/groovy/datadog/trace/payloadtags/json/JsonPathParserSpec.groovy @@ -1,5 +1,7 @@ package datadog.trace.payloadtags.json +import datadog.json.JsonPath +import datadog.json.JsonPathParser import spock.lang.Specification class JsonPathParserSpec extends Specification { diff --git a/dd-trace-core/src/test/groovy/datadog/trace/payloadtags/json/JsonPathSpec.groovy b/dd-trace-core/src/test/groovy/datadog/trace/payloadtags/json/JsonPathSpec.groovy index 42293126bdf..bc90e4902f1 100644 --- a/dd-trace-core/src/test/groovy/datadog/trace/payloadtags/json/JsonPathSpec.groovy +++ b/dd-trace-core/src/test/groovy/datadog/trace/payloadtags/json/JsonPathSpec.groovy @@ -1,5 +1,7 @@ package datadog.trace.payloadtags.json +import datadog.json.JsonPath +import datadog.json.PathCursor import spock.lang.Specification class JsonPathSpec extends Specification { diff --git a/dd-trace-core/src/test/groovy/datadog/trace/payloadtags/json/PathCursorSpec.groovy b/dd-trace-core/src/test/groovy/datadog/trace/payloadtags/json/PathCursorSpec.groovy index eba5530beb3..2395b755ee8 100644 --- a/dd-trace-core/src/test/groovy/datadog/trace/payloadtags/json/PathCursorSpec.groovy +++ b/dd-trace-core/src/test/groovy/datadog/trace/payloadtags/json/PathCursorSpec.groovy @@ -1,5 +1,6 @@ package datadog.trace.payloadtags.json +import datadog.json.PathCursor import spock.lang.Specification class PathCursorSpec extends Specification { diff --git a/internal-api/build.gradle.kts b/internal-api/build.gradle.kts index f05ab8f0b2a..97a7901b499 100644 --- a/internal-api/build.gradle.kts +++ b/internal-api/build.gradle.kts @@ -271,6 +271,7 @@ dependencies { api(project(":components:yaml")) api(project(":utils:config-utils")) api(project(":utils:time-utils")) + api(project(":utils:json-utils")) // has to be loaded by system classloader: // it contains annotations that are also present in the instrumented application classes diff --git a/internal-api/src/main/java/datadog/trace/api/Config.java b/internal-api/src/main/java/datadog/trace/api/Config.java index b22778171d5..7fda24d2dac 100644 --- a/internal-api/src/main/java/datadog/trace/api/Config.java +++ b/internal-api/src/main/java/datadog/trace/api/Config.java @@ -2,6 +2,7 @@ import static datadog.environment.JavaVirtualMachine.isJavaVersion; import static datadog.environment.JavaVirtualMachine.isJavaVersionAtLeast; +import static datadog.json.JsonPathParser.parseJsonPaths; import static datadog.trace.api.ConfigDefaults.DEFAULT_ADD_SPAN_POINTERS; import static datadog.trace.api.ConfigDefaults.DEFAULT_AGENT_HOST; import static datadog.trace.api.ConfigDefaults.DEFAULT_AGENT_TIMEOUT; @@ -677,6 +678,7 @@ import datadog.environment.JavaVirtualMachine; import datadog.environment.OperatingSystem; import datadog.environment.SystemProperties; +import datadog.json.JsonPath; import datadog.trace.api.civisibility.CiVisibilityWellKnownTags; import datadog.trace.api.config.GeneralConfig; import datadog.trace.api.config.OtlpConfig; @@ -822,6 +824,7 @@ public static String getHostName() { private final String agentUnixDomainSocket; private final String agentNamedPipe; private final int agentTimeout; + /** Should be set to {@code true} when running in agentless mode in a JVM without TLS */ private final boolean forceClearTextHttpForIntakeClient; @@ -1283,8 +1286,8 @@ public static String getHostName() { private final String agentlessLogSubmissionProduct; private final Set cloudPayloadTaggingServices; - @Nullable private final List cloudRequestPayloadTagging; - @Nullable private final List cloudResponsePayloadTagging; + @Nullable private final List cloudRequestPayloadTagging; + @Nullable private final List cloudResponsePayloadTagging; private final int cloudPayloadTaggingMaxDepth; private final int cloudPayloadTaggingMaxTags; @@ -2861,10 +2864,40 @@ PROFILING_DATADOG_PROFILER_ENABLED, isDatadogProfilerSafeInCurrentEnvironment()) this.cloudPayloadTaggingServices = configProvider.getSet( TRACE_CLOUD_PAYLOAD_TAGGING_SERVICES, DEFAULT_TRACE_CLOUD_PAYLOAD_TAGGING_SERVICES); - this.cloudRequestPayloadTagging = + + List cloudReqPayloadTaggingConf = configProvider.getList(TRACE_CLOUD_REQUEST_PAYLOAD_TAGGING, null); - this.cloudResponsePayloadTagging = + + if (null == cloudReqPayloadTaggingConf) { + // if no configuration is provided, disable payload tagging + this.cloudRequestPayloadTagging = null; + } else if (cloudReqPayloadTaggingConf.size() == 1 + && cloudReqPayloadTaggingConf.get(0).equalsIgnoreCase("all")) { + // if "all" is specified enable all JSON paths + this.cloudRequestPayloadTagging = Collections.emptyList(); + } else { + // parse and validate JSON paths. if none are valid, disable payload tagging + List validRequestJsonPaths = parseJsonPaths(cloudReqPayloadTaggingConf); + this.cloudRequestPayloadTagging = + validRequestJsonPaths.isEmpty() ? null : validRequestJsonPaths; + } + + List cloudRespPayloadTaggingConf = configProvider.getList(TRACE_CLOUD_RESPONSE_PAYLOAD_TAGGING, null); + if (null == cloudRespPayloadTaggingConf) { + // if no configuration is provided, disable payload tagging + this.cloudResponsePayloadTagging = null; + } else if (cloudRespPayloadTaggingConf.size() == 1 + && cloudRespPayloadTaggingConf.get(0).equalsIgnoreCase("all")) { + // if "all" is specified enable all JSON paths + this.cloudResponsePayloadTagging = Collections.emptyList(); + } else { + // parse and validate JSON paths. if none are valid, disable payload tagging + List validResponseJsonPaths = parseJsonPaths(cloudRespPayloadTaggingConf); + this.cloudResponsePayloadTagging = + validResponseJsonPaths.isEmpty() ? null : validResponseJsonPaths; + } + this.cloudPayloadTaggingMaxDepth = configProvider.getInteger(TRACE_CLOUD_PAYLOAD_TAGGING_MAX_DEPTH, 10); this.cloudPayloadTaggingMaxTags = @@ -5315,7 +5348,7 @@ public boolean isCloudPayloadTaggingEnabled() { return isCloudRequestPayloadTaggingEnabled() || isCloudResponsePayloadTaggingEnabled(); } - public List getCloudRequestPayloadTagging() { + public List getCloudRequestPayloadTagging() { return cloudRequestPayloadTagging == null ? Collections.emptyList() : cloudRequestPayloadTagging; @@ -5325,7 +5358,7 @@ public boolean isCloudRequestPayloadTaggingEnabled() { return cloudRequestPayloadTagging != null; } - public List getCloudResponsePayloadTagging() { + public List getCloudResponsePayloadTagging() { return cloudResponsePayloadTagging == null ? Collections.emptyList() : cloudResponsePayloadTagging; diff --git a/settings.gradle.kts b/settings.gradle.kts index 92aba9c108e..70e4769c42f 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -152,6 +152,7 @@ include( ":utils:test-utils", ":utils:time-utils", ":utils:version-utils", + "utils:json-utils", ) // smoke tests diff --git a/utils/json-utils/build.gradle.kts b/utils/json-utils/build.gradle.kts new file mode 100644 index 00000000000..e014b999941 --- /dev/null +++ b/utils/json-utils/build.gradle.kts @@ -0,0 +1,17 @@ +plugins { + `java-library` +} + +apply(from = "$rootDir/gradle/java.gradle") + +dependencies { + implementation(libs.slf4j) + implementation(libs.moshi) + implementation(libs.okio) + + modules { + module("com.squareup.okio:okio") { + replacedBy("com.datadoghq.okio:okio") // embed our patched fork + } + } +} diff --git a/dd-trace-core/src/main/java/datadog/trace/payloadtags/json/JsonPath.java b/utils/json-utils/src/main/java/datadog/json/JsonPath.java similarity index 99% rename from dd-trace-core/src/main/java/datadog/trace/payloadtags/json/JsonPath.java rename to utils/json-utils/src/main/java/datadog/json/JsonPath.java index a65a1726db7..7422db85e7e 100644 --- a/dd-trace-core/src/main/java/datadog/trace/payloadtags/json/JsonPath.java +++ b/utils/json-utils/src/main/java/datadog/json/JsonPath.java @@ -1,4 +1,4 @@ -package datadog.trace.payloadtags.json; +package datadog.json; import java.util.ArrayList; import java.util.Collection; diff --git a/dd-trace-core/src/main/java/datadog/trace/payloadtags/json/JsonPathParser.java b/utils/json-utils/src/main/java/datadog/json/JsonPathParser.java similarity index 91% rename from dd-trace-core/src/main/java/datadog/trace/payloadtags/json/JsonPathParser.java rename to utils/json-utils/src/main/java/datadog/json/JsonPathParser.java index 2d75d00fffc..33c834cab39 100644 --- a/dd-trace-core/src/main/java/datadog/trace/payloadtags/json/JsonPathParser.java +++ b/utils/json-utils/src/main/java/datadog/json/JsonPathParser.java @@ -1,7 +1,13 @@ -package datadog.trace.payloadtags.json; +package datadog.json; import static java.lang.Character.isDigit; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + public class JsonPathParser { public static final class ParseError extends Exception { @@ -26,6 +32,24 @@ public ParseError(CharSequence path, int position, String error) { private static final char DOUBLE_QUOTE = '"'; private static final char ESC = '\\'; + private static final Logger log = LoggerFactory.getLogger(JsonPathParser.class); + + public static List parseJsonPaths(List rules) { + if (rules.isEmpty() || rules.size() == 1 && rules.get(0).equalsIgnoreCase("all")) { + return Collections.emptyList(); + } + List result = new ArrayList<>(rules.size()); + for (String rule : rules) { + try { + JsonPath jp = JsonPathParser.parse(rule); + result.add(jp); + } catch (Exception ex) { + log.warn("Skipping failed to parse redaction rule '{}'. {}", rule, ex.getMessage()); + } + } + return result; + } + public static JsonPath parse(String path) throws ParseError { Cursor cur = new Cursor(path); diff --git a/dd-trace-core/src/main/java/datadog/trace/payloadtags/json/JsonStreamParser.java b/utils/json-utils/src/main/java/datadog/json/JsonStreamParser.java similarity index 99% rename from dd-trace-core/src/main/java/datadog/trace/payloadtags/json/JsonStreamParser.java rename to utils/json-utils/src/main/java/datadog/json/JsonStreamParser.java index 916e65afb72..ef3f3e0f350 100644 --- a/dd-trace-core/src/main/java/datadog/trace/payloadtags/json/JsonStreamParser.java +++ b/utils/json-utils/src/main/java/datadog/json/JsonStreamParser.java @@ -1,4 +1,4 @@ -package datadog.trace.payloadtags.json; +package datadog.json; import com.squareup.moshi.JsonReader; import java.io.ByteArrayInputStream; diff --git a/dd-trace-core/src/main/java/datadog/trace/payloadtags/json/PathCursor.java b/utils/json-utils/src/main/java/datadog/json/PathCursor.java similarity index 97% rename from dd-trace-core/src/main/java/datadog/trace/payloadtags/json/PathCursor.java rename to utils/json-utils/src/main/java/datadog/json/PathCursor.java index 1b23b315ccb..0d209fefae1 100644 --- a/dd-trace-core/src/main/java/datadog/trace/payloadtags/json/PathCursor.java +++ b/utils/json-utils/src/main/java/datadog/json/PathCursor.java @@ -1,4 +1,4 @@ -package datadog.trace.payloadtags.json; +package datadog.json; import java.util.Arrays;