Skip to content

Commit

Permalink
fix(js/plugins/google-cloud)!: rename io to inputAndOutput (#1674)
Browse files Browse the repository at this point in the history
BREAKING CHANGE: IO generally refers to interfaces between a computer and the outside world. Reading/writing from disk, reading/writing from the network, etc (and also of course peripherals like mice, keyboards, monitors, etc). Renaming this removes ambiguity that we are referring to something different.
  • Loading branch information
MichaelDoyle authored Jan 28, 2025
1 parent 0272f27 commit 300f41b
Show file tree
Hide file tree
Showing 10 changed files with 52 additions and 33 deletions.
2 changes: 1 addition & 1 deletion docs/plugins/google-cloud.md
Original file line number Diff line number Diff line change
Expand Up @@ -197,7 +197,7 @@ and logs.
Provides an override that disables exporting traces while still exprting metrics
and logs.

#### disableLoggingIO
#### disableLoggingInputAndOutput

Provides an override that disables collecting input and output logs.

Expand Down
34 changes: 27 additions & 7 deletions js/plugins/google-cloud/src/gcpOpenTelemetry.ts
Original file line number Diff line number Diff line change
Expand Up @@ -128,7 +128,7 @@ export class GcpOpenTelemetry {
credentials: this.config.credentials,
})
: new InMemorySpanExporter(),
this.config.exportIO,
this.config.exportInputAndOutput,
this.config.projectId,
getErrorHandler(
(err) => {
Expand Down Expand Up @@ -245,7 +245,7 @@ class MetricExporterWrapper extends MetricExporter {
class AdjustingTraceExporter implements SpanExporter {
constructor(
private exporter: SpanExporter,
private logIO: boolean,
private logInputAndOutput: boolean,
private projectId?: string,
private errorHandler?: (error: Error) => void
) {}
Expand Down Expand Up @@ -354,26 +354,46 @@ class AdjustingTraceExporter implements SpanExporter {
if (isRoot) {
// Report top level feature request and latency only for root spans
// Log input to and output from to the feature
featuresTelemetry.tick(span, unused, this.logIO, this.projectId);
featuresTelemetry.tick(
span,
unused,
this.logInputAndOutput,
this.projectId
);
// Report executions and latency for all flow paths only on the root span
pathsTelemetry.tick(span, paths, this.logIO, this.projectId);
pathsTelemetry.tick(span, paths, this.logInputAndOutput, this.projectId);
// Set root status explicitly
span.attributes['genkit:rootState'] = span.attributes['genkit:state'];
}
if (type === 'action' && subtype === 'model') {
// Report generate metrics () for all model actions
generateTelemetry.tick(span, unused, this.logIO, this.projectId);
generateTelemetry.tick(
span,
unused,
this.logInputAndOutput,
this.projectId
);
}
if (type === 'action' && subtype === 'tool') {
// TODO: Report input and output for tool actions
}
if (type === 'action' || type === 'flow' || type == 'flowStep') {
// Report request and latency metrics for all actions
actionTelemetry.tick(span, unused, this.logIO, this.projectId);
actionTelemetry.tick(
span,
unused,
this.logInputAndOutput,
this.projectId
);
}
if (type === 'userEngagement') {
// Report user acceptance and feedback metrics
engagementTelemetry.tick(span, unused, this.logIO, this.projectId);
engagementTelemetry.tick(
span,
unused,
this.logInputAndOutput,
this.projectId
);
}
}

Expand Down
14 changes: 7 additions & 7 deletions js/plugins/google-cloud/src/telemetry/action.ts
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,7 @@ class ActionTelemetry implements Telemetry {
tick(
span: ReadableSpan,
paths: Set<PathMetadata>,
logIO: boolean,
logInputAndOutput: boolean,
projectId?: string
): void {
const attributes = span.attributes;
Expand All @@ -78,14 +78,14 @@ class ActionTelemetry implements Telemetry {
logger.warn(`Unknown action state; ${state}`);
}

if (subtype === 'tool' && logIO) {
if (subtype === 'tool' && logInputAndOutput) {
const input = attributes['genkit:input'] as string;
const output = attributes['genkit:output'] as string;
const sessionId = attributes['genkit:sessionId'] as string;
const threadName = attributes['genkit:threadName'] as string;

if (input) {
this.recordIO(
this.writeLog(
span,
'Input',
featureName,
Expand All @@ -97,7 +97,7 @@ class ActionTelemetry implements Telemetry {
);
}
if (output) {
this.recordIO(
this.writeLog(
span,
'Output',
featureName,
Expand Down Expand Up @@ -149,12 +149,12 @@ class ActionTelemetry implements Telemetry {
this.actionLatencies.record(latencyMs, dimensions);
}

private recordIO(
private writeLog(
span: ReadableSpan,
tag: string,
featureName: string,
qualifiedPath: string,
input: string,
content: string,
projectId?: string,
sessionId?: string,
threadName?: string
Expand All @@ -170,7 +170,7 @@ class ActionTelemetry implements Telemetry {
};
logger.logStructured(`${tag}[${path}, ${featureName}]`, {
...sharedMetadata,
content: input,
content,
});
}
}
Expand Down
5 changes: 2 additions & 3 deletions js/plugins/google-cloud/src/telemetry/defaults.ts
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,7 @@ export const TelemetryConfigs = {
metricExportTimeoutMillis: 5_000,
disableMetrics: false,
disableTraces: false,
exportIO: !overrides.disableLoggingIO,
exportInputAndOutput: !overrides.disableLoggingInputAndOutput,
export: !!overrides.forceDevExport, // false
};
return { ...defaults, ...overrides };
Expand All @@ -61,8 +61,7 @@ export const TelemetryConfigs = {
metricExportTimeoutMillis: 300_000,
disableMetrics: false,
disableTraces: false,
disableLoggingIO: false,
exportIO: !overrides.disableLoggingIO,
exportInputAndOutput: !overrides.disableLoggingInputAndOutput,
export: true,
};
return { ...defaults, ...overrides };
Expand Down
2 changes: 1 addition & 1 deletion js/plugins/google-cloud/src/telemetry/engagement.ts
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,7 @@ class EngagementTelemetry implements Telemetry {
tick(
span: ReadableSpan,
paths: Set<PathMetadata>,
logIO: boolean,
logInputAndOutput: boolean,
projectId?: string
): void {
const subtype = span.attributes['genkit:metadata:subtype'] as string;
Expand Down
14 changes: 7 additions & 7 deletions js/plugins/google-cloud/src/telemetry/feature.ts
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,7 @@ class FeaturesTelemetry implements Telemetry {
tick(
span: ReadableSpan,
paths: Set<PathMetadata>,
logIO: boolean,
logInputAndOutput: boolean,
projectId?: string
): void {
const attributes = span.attributes;
Expand Down Expand Up @@ -76,14 +76,14 @@ class FeaturesTelemetry implements Telemetry {
return;
}

if (logIO) {
if (logInputAndOutput) {
const input = attributes['genkit:input'] as string;
const output = attributes['genkit:output'] as string;
const sessionId = attributes['genkit:sessionId'] as string;
const threadName = attributes['genkit:threadName'] as string;

if (input) {
this.recordIO(
this.writeLog(
span,
'Input',
name,
Expand All @@ -95,7 +95,7 @@ class FeaturesTelemetry implements Telemetry {
);
}
if (output) {
this.recordIO(
this.writeLog(
span,
'Output',
name,
Expand Down Expand Up @@ -136,12 +136,12 @@ class FeaturesTelemetry implements Telemetry {
this.featureLatencies.record(latencyMs, dimensions);
}

private recordIO(
private writeLog(
span: ReadableSpan,
tag: string,
featureName: string,
qualifiedPath: string,
input: string,
content: string,
projectId?: string,
sessionId?: string,
threadName?: string
Expand All @@ -157,7 +157,7 @@ class FeaturesTelemetry implements Telemetry {
};
logger.logStructured(`${tag}[${path}, ${featureName}]`, {
...sharedMetadata,
content: input,
content,
});
}
}
Expand Down
6 changes: 3 additions & 3 deletions js/plugins/google-cloud/src/telemetry/generate.ts
Original file line number Diff line number Diff line change
Expand Up @@ -132,7 +132,7 @@ class GenerateTelemetry implements Telemetry {
tick(
span: ReadableSpan,
paths: Set<PathMetadata>,
logIO: boolean,
logInputAndOutput: boolean,
projectId?: string
): void {
const attributes = span.attributes;
Expand Down Expand Up @@ -177,7 +177,7 @@ class GenerateTelemetry implements Telemetry {
threadName
);

if (logIO) {
if (logInputAndOutput) {
this.recordGenerateActionInputLogs(
span,
modelName,
Expand All @@ -191,7 +191,7 @@ class GenerateTelemetry implements Telemetry {
}
}

if (output && logIO) {
if (output && logInputAndOutput) {
this.recordGenerateActionOutputLogs(
span,
modelName,
Expand Down
2 changes: 1 addition & 1 deletion js/plugins/google-cloud/src/telemetry/path.ts
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,7 @@ class PathsTelemetry implements Telemetry {
tick(
span: ReadableSpan,
paths: Set<PathMetadata>,
logIO: boolean,
logInputAndOutput: boolean,
projectId?: string
): void {
const attributes = span.attributes;
Expand Down
4 changes: 2 additions & 2 deletions js/plugins/google-cloud/src/types.ts
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,7 @@ export interface GcpTelemetryConfigOptions {
disableTraces?: boolean;

/** When true, inputs and outputs are not logged to GCP */
disableLoggingIO?: boolean;
disableLoggingInputAndOutput?: boolean;

/** When true, telemetry data will be exported, even for local runs. Defaults to not exporting development traces. */
forceDevExport?: boolean;
Expand All @@ -67,7 +67,7 @@ export interface GcpTelemetryConfig {
instrumentations: Instrumentation[];
disableMetrics: boolean;
disableTraces: boolean;
exportIO: boolean;
exportInputAndOutput: boolean;
export: boolean;
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -74,7 +74,7 @@ describe('GoogleCloudLogs no I/O', () => {
forceDevExport: false,
metricExportIntervalMillis: 100,
metricExportTimeoutMillis: 100,
disableLoggingIO: true,
disableLoggingInputAndOutput: true,
});
ai = genkit({});
// Wait for the telemetry plugin to be initialized
Expand Down

0 comments on commit 300f41b

Please sign in to comment.