Skip to content

Commit 2be1b73

Browse files
gary-huanggary-huang
committed
add APIs for llm obs sdk (#8135)
* add APIs for llm obs * add llm message class to support llm spans * follow java convention of naming Id instead of ID * add codeowners
1 parent 7c80dbe commit 2be1b73

File tree

6 files changed

+392
-0
lines changed

6 files changed

+392
-0
lines changed

‎.github/CODEOWNERS

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -74,3 +74,9 @@ dd-trace-core/src/main/java/datadog/trace/core/datastreams @Dat
7474
dd-trace-core/src/test/groovy/datadog/trace/core/datastreams @DataDog/data-streams-monitoring
7575
internal-api/src/main/java/datadog/trace/api/datastreams @DataDog/data-streams-monitoring
7676
internal-api/src/test/groovy/datadog/trace/api/datastreams @DataDog/data-streams-monitoring
77+
78+
# @DataDog/ml-observability
79+
dd-trace-api/src/main/java/datadog/trace/api/llmobs/ @DataDog/ml-observability
80+
dd-java-agent/agent-llmobs/ @DataDog/ml-observability
81+
dd-trace-core/src/main/java/datadog/trace/llmobs/ @DataDog/ml-observability
82+
dd-trace-core/src/test/groovy/datadog/trace/llmobs/ @DataDog/ml-observability

‎dd-trace-api/build.gradle

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,12 @@ excludedClassesCoverage += [
3232
'datadog.trace.api.profiling.ProfilingScope',
3333
'datadog.trace.api.profiling.ProfilingContext',
3434
'datadog.trace.api.profiling.ProfilingContextAttribute.NoOp',
35+
'datadog.trace.api.llmobs.LLMObs',
36+
'datadog.trace.api.llmobs.LLMObs.LLMMessage',
37+
'datadog.trace.api.llmobs.LLMObs.ToolCall',
38+
'datadog.trace.api.llmobs.LLMObsSpan',
39+
'datadog.trace.api.llmobs.noop.NoOpLLMObsSpan',
40+
'datadog.trace.api.llmobs.noop.NoOpLLMObsSpanFactory',
3541
'datadog.trace.api.experimental.DataStreamsCheckpointer',
3642
'datadog.trace.api.experimental.DataStreamsCheckpointer.NoOp',
3743
'datadog.trace.api.experimental.DataStreamsContextCarrier',

‎dd-trace-api/src/main/java/datadog/trace/api/llmobs/LLMObs.java

Lines changed: 136 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,136 @@
1+
package datadog.trace.api.llmobs;
2+
3+
import datadog.trace.api.llmobs.noop.NoOpLLMObsSpanFactory;
4+
import java.util.List;
5+
import java.util.Map;
6+
import javax.annotation.Nullable;
7+
8+
public class LLMObs {
9+
protected LLMObs() {}
10+
11+
protected static LLMObsSpanFactory SPAN_FACTORY = NoOpLLMObsSpanFactory.INSTANCE;
12+
13+
public static LLMObsSpan startLLMSpan(
14+
String spanName,
15+
String modelName,
16+
String modelProvider,
17+
@Nullable String mlApp,
18+
@Nullable String sessionId) {
19+
20+
return SPAN_FACTORY.startLLMSpan(spanName, modelName, modelProvider, mlApp, sessionId);
21+
}
22+
23+
public static LLMObsSpan startAgentSpan(
24+
String spanName, @Nullable String mlApp, @Nullable String sessionId) {
25+
26+
return SPAN_FACTORY.startAgentSpan(spanName, mlApp, sessionId);
27+
}
28+
29+
public static LLMObsSpan startToolSpan(
30+
String spanName, @Nullable String mlApp, @Nullable String sessionId) {
31+
32+
return SPAN_FACTORY.startToolSpan(spanName, mlApp, sessionId);
33+
}
34+
35+
public static LLMObsSpan startTaskSpan(
36+
String spanName, @Nullable String mlApp, @Nullable String sessionId) {
37+
38+
return SPAN_FACTORY.startTaskSpan(spanName, mlApp, sessionId);
39+
}
40+
41+
public static LLMObsSpan startWorkflowSpan(
42+
String spanName, @Nullable String mlApp, @Nullable String sessionId) {
43+
44+
return SPAN_FACTORY.startWorkflowSpan(spanName, mlApp, sessionId);
45+
}
46+
47+
public interface LLMObsSpanFactory {
48+
LLMObsSpan startLLMSpan(
49+
String spanName,
50+
String modelName,
51+
String modelProvider,
52+
@Nullable String mlApp,
53+
@Nullable String sessionId);
54+
55+
LLMObsSpan startAgentSpan(String spanName, @Nullable String mlApp, @Nullable String sessionId);
56+
57+
LLMObsSpan startToolSpan(String spanName, @Nullable String mlApp, @Nullable String sessionId);
58+
59+
LLMObsSpan startTaskSpan(String spanName, @Nullable String mlApp, @Nullable String sessionId);
60+
61+
LLMObsSpan startWorkflowSpan(
62+
String spanName, @Nullable String mlApp, @Nullable String sessionId);
63+
}
64+
65+
public static class ToolCall {
66+
private String name;
67+
private String type;
68+
private String toolId;
69+
private Map<String, Object> arguments;
70+
71+
public static ToolCall from(
72+
String name, String type, String toolId, Map<String, Object> arguments) {
73+
return new ToolCall(name, type, toolId, arguments);
74+
}
75+
76+
private ToolCall(String name, String type, String toolId, Map<String, Object> arguments) {
77+
this.name = name;
78+
this.type = type;
79+
this.toolId = toolId;
80+
this.arguments = arguments;
81+
}
82+
83+
public String getName() {
84+
return name;
85+
}
86+
87+
public String getType() {
88+
return type;
89+
}
90+
91+
public String getToolId() {
92+
return toolId;
93+
}
94+
95+
public Map<String, Object> getArguments() {
96+
return arguments;
97+
}
98+
}
99+
100+
public static class LLMMessage {
101+
private String role;
102+
private String content;
103+
private List<ToolCall> toolCalls;
104+
105+
public static LLMMessage from(String role, String content, List<ToolCall> toolCalls) {
106+
return new LLMMessage(role, content, toolCalls);
107+
}
108+
109+
public static LLMMessage from(String role, String content) {
110+
return new LLMMessage(role, content);
111+
}
112+
113+
private LLMMessage(String role, String content, List<ToolCall> toolCalls) {
114+
this.role = role;
115+
this.content = content;
116+
this.toolCalls = toolCalls;
117+
}
118+
119+
private LLMMessage(String role, String content) {
120+
this.role = role;
121+
this.content = content;
122+
}
123+
124+
public String getRole() {
125+
return role;
126+
}
127+
128+
public String getContent() {
129+
return content;
130+
}
131+
132+
public List<ToolCall> getToolCalls() {
133+
return toolCalls;
134+
}
135+
}
136+
}

‎dd-trace-api/src/main/java/datadog/trace/api/llmobs/LLMObsSpan.java

Lines changed: 145 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,145 @@
1+
package datadog.trace.api.llmobs;
2+
3+
import java.util.List;
4+
import java.util.Map;
5+
6+
/** This interface represent an individual LLM Obs span. */
7+
public interface LLMObsSpan {
8+
9+
/**
10+
* Annotate the span with inputs and outputs for LLM spans
11+
*
12+
* @param inputMessages The input messages of the span in the form of a list
13+
* @param outputMessages The output messages of the span in the form of a list
14+
*/
15+
void annotateIO(List<LLMObs.LLMMessage> inputMessages, List<LLMObs.LLMMessage> outputMessages);
16+
17+
/**
18+
* Annotate the span with inputs and outputs
19+
*
20+
* @param inputData The input data of the span in the form of a string
21+
* @param outputData The output data of the span in the form of a string
22+
*/
23+
void annotateIO(String inputData, String outputData);
24+
25+
/**
26+
* Annotate the span with metadata
27+
*
28+
* @param metadata A map of JSON serializable key-value pairs that contains metadata information
29+
* relevant to the input or output operation described by the span
30+
*/
31+
void setMetadata(Map<String, Object> metadata);
32+
33+
/**
34+
* Annotate the span with metrics
35+
*
36+
* @param metrics A map of JSON serializable keys and numeric values that users can add as metrics
37+
* relevant to the operation described by the span (input_tokens, output_tokens, total_tokens,
38+
* etc.).
39+
*/
40+
void setMetrics(Map<String, Number> metrics);
41+
42+
/**
43+
* Annotate the span with a single metric key value pair for the span’s context (number of tokens
44+
* document length, etc).
45+
*
46+
* @param key the name of the metric
47+
* @param value the value of the metric
48+
*/
49+
void setMetric(CharSequence key, int value);
50+
51+
/**
52+
* Annotate the span with a single metric key value pair for the span’s context (number of tokens
53+
* document length, etc).
54+
*
55+
* @param key the name of the metric
56+
* @param value the value of the metric
57+
*/
58+
void setMetric(CharSequence key, long value);
59+
60+
/**
61+
* Annotate the span with a single metric key value pair for the span’s context (number of tokens
62+
* document length, etc).
63+
*
64+
* @param key the name of the metric
65+
* @param value the value of the metric
66+
*/
67+
void setMetric(CharSequence key, double value);
68+
69+
/**
70+
* Annotate the span with tags
71+
*
72+
* @param tags An map of JSON serializable key-value pairs that users can add as tags regarding
73+
* the span’s context (session, environment, system, versioning, etc.).
74+
*/
75+
void setTags(Map<String, Object> tags);
76+
77+
/**
78+
* Annotate the span with a single tag key value pair as a tag regarding the span’s context
79+
* (session, environment, system, versioning, etc.).
80+
*
81+
* @param key the key of the tag
82+
* @param value the value of the tag
83+
*/
84+
void setTag(String key, String value);
85+
86+
/**
87+
* Annotate the span with a single tag key value pair as a tag regarding the span’s context
88+
* (session, environment, system, versioning, etc.).
89+
*
90+
* @param key the key of the tag
91+
* @param value the value of the tag
92+
*/
93+
void setTag(String key, boolean value);
94+
95+
/**
96+
* Annotate the span with a single tag key value pair as a tag regarding the span’s context
97+
* (session, environment, system, versioning, etc.).
98+
*
99+
* @param key the key of the tag
100+
* @param value the value of the tag
101+
*/
102+
void setTag(String key, int value);
103+
104+
/**
105+
* Annotate the span with a single tag key value pair as a tag regarding the span’s context
106+
* (session, environment, system, versioning, etc.).
107+
*
108+
* @param key the key of the tag
109+
* @param value the value of the tag
110+
*/
111+
void setTag(String key, long value);
112+
113+
/**
114+
* Annotate the span with a single tag key value pair as a tag regarding the span’s context
115+
* (session, environment, system, versioning, etc.).
116+
*
117+
* @param key the key of the tag
118+
* @param value the value of the tag
119+
*/
120+
void setTag(String key, double value);
121+
122+
/**
123+
* Annotate the span to indicate that an error occurred
124+
*
125+
* @param error whether an error occurred
126+
*/
127+
void setError(boolean error);
128+
129+
/**
130+
* Annotate the span with an error message
131+
*
132+
* @param errorMessage the message of the error
133+
*/
134+
void setErrorMessage(String errorMessage);
135+
136+
/**
137+
* Annotate the span with a throwable
138+
*
139+
* @param throwable the errored throwable
140+
*/
141+
void addThrowable(Throwable throwable);
142+
143+
/** Finishes (closes) a span */
144+
void finish();
145+
}

‎dd-trace-api/src/main/java/datadog/trace/api/llmobs/noop/NoOpLLMObsSpan.java

Lines changed: 61 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,61 @@
1+
package datadog.trace.api.llmobs.noop;
2+
3+
import datadog.trace.api.llmobs.LLMObs;
4+
import datadog.trace.api.llmobs.LLMObsSpan;
5+
import java.util.List;
6+
import java.util.Map;
7+
8+
public class NoOpLLMObsSpan implements LLMObsSpan {
9+
public static final LLMObsSpan INSTANCE = new NoOpLLMObsSpan();
10+
11+
@Override
12+
public void annotateIO(List<LLMObs.LLMMessage> inputData, List<LLMObs.LLMMessage> outputData) {}
13+
14+
@Override
15+
public void annotateIO(String inputData, String outputData) {}
16+
17+
@Override
18+
public void setMetadata(Map<String, Object> metadata) {}
19+
20+
@Override
21+
public void setMetrics(Map<String, Number> metrics) {}
22+
23+
@Override
24+
public void setMetric(CharSequence key, int value) {}
25+
26+
@Override
27+
public void setMetric(CharSequence key, long value) {}
28+
29+
@Override
30+
public void setMetric(CharSequence key, double value) {}
31+
32+
@Override
33+
public void setTags(Map<String, Object> tags) {}
34+
35+
@Override
36+
public void setTag(String key, String value) {}
37+
38+
@Override
39+
public void setTag(String key, boolean value) {}
40+
41+
@Override
42+
public void setTag(String key, int value) {}
43+
44+
@Override
45+
public void setTag(String key, long value) {}
46+
47+
@Override
48+
public void setTag(String key, double value) {}
49+
50+
@Override
51+
public void setError(boolean error) {}
52+
53+
@Override
54+
public void setErrorMessage(String errorMessage) {}
55+
56+
@Override
57+
public void addThrowable(Throwable throwable) {}
58+
59+
@Override
60+
public void finish() {}
61+
}

0 commit comments

Comments
 (0)