From 4b91b8b63b6d3b8dfe8e32058239dc7dadf71b83 Mon Sep 17 00:00:00 2001 From: Elisabeth Rulke Date: Thu, 27 Aug 2026 14:55:48 -0400 Subject: [PATCH 1/6] [cli] add opt-out telemetry and AI-agent attribution --- .changeset/spotty-pandas-tell.md | 12 ++ packages/sandbox/docs/index.md | 1 + packages/sandbox/package.json | 1 + packages/sandbox/src/app.ts | 2 + packages/sandbox/src/client.ts | 47 +++-- packages/sandbox/src/commands/telemetry.ts | 63 +++++++ packages/sandbox/src/index.ts | 8 +- packages/sandbox/src/sandbox.ts | 10 +- packages/sandbox/src/telemetry/agent.ts | 18 ++ packages/sandbox/src/telemetry/index.ts | 172 ++++++++++++++++++ .../sandbox/src/telemetry/telemetry.test.ts | 134 ++++++++++++++ pnpm-lock.yaml | 9 + 12 files changed, 461 insertions(+), 16 deletions(-) create mode 100644 .changeset/spotty-pandas-tell.md create mode 100644 packages/sandbox/src/commands/telemetry.ts create mode 100644 packages/sandbox/src/telemetry/agent.ts create mode 100644 packages/sandbox/src/telemetry/index.ts create mode 100644 packages/sandbox/src/telemetry/telemetry.test.ts diff --git a/.changeset/spotty-pandas-tell.md b/.changeset/spotty-pandas-tell.md new file mode 100644 index 00000000..92b1d01e --- /dev/null +++ b/.changeset/spotty-pandas-tell.md @@ -0,0 +1,12 @@ +--- +"sandbox": minor +--- + +Add opt-out usage telemetry and AI-agent attribution. The CLI now reports +anonymous usage events (subcommand, CLI version, platform, and the AI agent +driving the invocation, detected via `detect-agent`) to Vercel's telemetry +bridge, and tags API requests with the detected agent in the user-agent +header. Manage collection with `sandbox telemetry status|enable|disable`, +`VERCEL_SANDBOX_TELEMETRY_DISABLED=1`, or inspect events without sending via +`VERCEL_TELEMETRY_DEBUG=1`. Running as `vercel sandbox` respects the Vercel +CLI's own telemetry preference. diff --git a/packages/sandbox/docs/index.md b/packages/sandbox/docs/index.md index 92123151..798409b3 100644 --- a/packages/sandbox/docs/index.md +++ b/packages/sandbox/docs/index.md @@ -23,6 +23,7 @@ Commands: snapshot Take a snapshot of the filesystem of a sandbox snapshots Manage sandbox snapshots sessions Manage sandbox sessions + telemetry Manage telemetry collection status login Log in to the Sandbox CLI logout Log out of the Sandbox CLI diff --git a/packages/sandbox/package.json b/packages/sandbox/package.json index e7564985..2e31c012 100644 --- a/packages/sandbox/package.json +++ b/packages/sandbox/package.json @@ -55,6 +55,7 @@ "chalk": "^5.6.0", "cmd-ts": "0.15.0", "date-fns": "^4.1.0", + "detect-agent": "1.2.0", "dotenv-flow": "^4.1.0", "listr2": "^9.0.2", "ms": "^2.1.3", diff --git a/packages/sandbox/src/app.ts b/packages/sandbox/src/app.ts index 5dbdb0c4..291ea85a 100644 --- a/packages/sandbox/src/app.ts +++ b/packages/sandbox/src/app.ts @@ -16,6 +16,7 @@ import { snapshot } from "./commands/snapshot"; import { snapshots } from "./commands/snapshots"; import { sessions } from "./commands/sessions"; import { config } from "./commands/config"; +import { telemetryCommand } from "./commands/telemetry"; export const app = (opts?: { withoutAuth?: boolean; appName?: string }) => { const appName = opts?.appName ?? "sandbox"; @@ -38,6 +39,7 @@ export const app = (opts?: { withoutAuth?: boolean; appName?: string }) => { snapshot, snapshots, sessions, + telemetry: telemetryCommand, ...(!opts?.withoutAuth && { login, logout, diff --git a/packages/sandbox/src/client.ts b/packages/sandbox/src/client.ts index 86bcf9cb..624ffb89 100644 --- a/packages/sandbox/src/client.ts +++ b/packages/sandbox/src/client.ts @@ -2,6 +2,8 @@ import { Sandbox, APIError, Snapshot } from "@vercel/sandbox"; import { version } from "./pkg"; import { withFreshAuthRetry } from "./util/fresh-auth-retry"; import { formatApiError } from "./util/format-error"; +import { telemetry } from "./telemetry"; +import { detectAgentName } from "./telemetry/agent"; /** * A {@link Sandbox} wrapper that adds user-agent headers and error handling. @@ -11,21 +13,25 @@ export const sandboxClient: Pick< "get" | "list" | "create" | "fork" > = { get: (params) => - withErrorHandling(() => - Sandbox.get({ fetch: fetchWithUserAgent, resume: false, ...params }), - ), + withErrorHandling(() => { + telemetry.updateTeamId(teamIdOf(params)); + return Sandbox.get({ fetch: fetchWithUserAgent, resume: false, ...params }); + }), create: (params) => - withErrorHandling(() => - Sandbox.create({ fetch: fetchWithUserAgent, ...params }), - ), + withErrorHandling(() => { + telemetry.updateTeamId(teamIdOf(params)); + return Sandbox.create({ fetch: fetchWithUserAgent, ...params }); + }), fork: (params) => - withErrorHandling(() => - Sandbox.fork({ fetch: fetchWithUserAgent, ...params }), - ), + withErrorHandling(() => { + telemetry.updateTeamId(teamIdOf(params)); + return Sandbox.fork({ fetch: fetchWithUserAgent, ...params }); + }), list: (params) => - withErrorHandling(() => - Sandbox.list({ fetch: fetchWithUserAgent, ...params } as typeof params), - ), + withErrorHandling(() => { + telemetry.updateTeamId(teamIdOf(params)); + return Sandbox.list({ fetch: fetchWithUserAgent, ...params } as typeof params); + }), }; export const snapshotClient: Pick< @@ -41,7 +47,15 @@ export const snapshotClient: Pick< withErrorHandling(() => Snapshot.tree({ fetch: fetchWithUserAgent, ...params })), }; -const fetchWithUserAgent: typeof globalThis.fetch = (input, init) => { +function teamIdOf(params: unknown): string | undefined { + if (params && typeof params === "object" && "teamId" in params) { + const { teamId } = params as { teamId?: unknown }; + if (typeof teamId === "string") return teamId; + } + return undefined; +} + +const fetchWithUserAgent: typeof globalThis.fetch = async (input, init) => { const headers = new Headers( init?.headers ?? (input && typeof input === "object" && "headers" in input @@ -50,6 +64,13 @@ const fetchWithUserAgent: typeof globalThis.fetch = (input, init) => { ); let agent = `vercel/sandbox-cli/${version}`; + // Attribute API traffic to the AI agent driving this invocation, if any, + // so the server side can record it once ingestion support lands. + const aiAgent = await detectAgentName(); + if (aiAgent) { + agent += ` agent/${aiAgent}`; + } + const existingAgent = headers.get("user-agent"); if (existingAgent) { agent += ` ${existingAgent}`; diff --git a/packages/sandbox/src/commands/telemetry.ts b/packages/sandbox/src/commands/telemetry.ts new file mode 100644 index 00000000..a401f14d --- /dev/null +++ b/packages/sandbox/src/commands/telemetry.ts @@ -0,0 +1,63 @@ +import * as cmd from "cmd-ts"; +import chalk from "chalk"; +import { + readTelemetryConfig, + writeTelemetryConfig, + telemetry, +} from "../telemetry"; + +function printStatus(): void { + const enabled = telemetry.enabled; + const status = enabled ? chalk.green("Enabled") : chalk.red("Disabled"); + process.stderr.write(`Telemetry status: ${status}\n\n`); + if (enabled) { + process.stderr.write( + "The Vercel Sandbox CLI collects anonymous usage data to improve the product.\n" + + `Opt out with ${chalk.cyan("sandbox telemetry disable")} or by setting ${chalk.cyan("VERCEL_SANDBOX_TELEMETRY_DISABLED=1")}.\n` + + `Inspect what is collected by setting ${chalk.cyan("VERCEL_TELEMETRY_DEBUG=1")}; events are printed and not sent.\n`, + ); + } else { + process.stderr.write( + `Re-enable with ${chalk.cyan("sandbox telemetry enable")}.\n`, + ); + } +} + +const statusCommand = cmd.command({ + name: "status", + description: "Show whether telemetry collection is enabled", + args: {}, + async handler() { + printStatus(); + }, +}); + +const enableCommand = cmd.command({ + name: "enable", + description: "Enable telemetry collection", + args: {}, + async handler() { + writeTelemetryConfig(true); + printStatus(); + }, +}); + +const disableCommand = cmd.command({ + name: "disable", + description: "Disable telemetry collection", + args: {}, + async handler() { + writeTelemetryConfig(false); + printStatus(); + }, +}); + +export const telemetryCommand = cmd.subcommands({ + name: "telemetry", + description: "Manage telemetry collection status", + cmds: { + status: statusCommand, + enable: enableCommand, + disable: disableCommand, + }, +}); diff --git a/packages/sandbox/src/index.ts b/packages/sandbox/src/index.ts index 4083be6e..67bb3e80 100644 --- a/packages/sandbox/src/index.ts +++ b/packages/sandbox/src/index.ts @@ -1,11 +1,17 @@ import { run as runCmd } from "cmd-ts"; import { app } from "./app"; +import { telemetry } from "./telemetry"; export function createApp(opts: { withoutAuth: boolean; appName: string }) { const instance = app(opts); return { async run(args: string[]) { - await runCmd(instance, args); + await telemetry.trackInvocation({ appName: opts.appName, argv: args }); + try { + await runCmd(instance, args); + } finally { + await telemetry.flush(); + } }, }; } diff --git a/packages/sandbox/src/sandbox.ts b/packages/sandbox/src/sandbox.ts index 1d86be3b..6febed12 100644 --- a/packages/sandbox/src/sandbox.ts +++ b/packages/sandbox/src/sandbox.ts @@ -4,6 +4,7 @@ import dotenv from "dotenv-flow"; import { printTopLevelError } from "./util/format-error"; import { steerShCommand } from "./util/steer-sh"; import { vercelFormatter } from "cmd-ts/batteries/vercel-formatter"; +import { telemetry } from "./telemetry"; dotenv.config({ silent: true, @@ -12,10 +13,15 @@ dotenv.config({ async function main() { setDefaultHelpFormatter(vercelFormatter); + const argv = process.argv.slice(2); + await telemetry.trackInvocation({ appName: "sandbox", argv }); + try { - steerShCommand(process.argv.slice(2)); - await run(app(), process.argv.slice(2)); + steerShCommand(argv); + await run(app(), argv); + await telemetry.flush(); } catch (e) { + await telemetry.flush(); await printTopLevelError(e); process.exit(1); } diff --git a/packages/sandbox/src/telemetry/agent.ts b/packages/sandbox/src/telemetry/agent.ts new file mode 100644 index 00000000..81a9a081 --- /dev/null +++ b/packages/sandbox/src/telemetry/agent.ts @@ -0,0 +1,18 @@ +import { determineAgent } from "detect-agent"; + +let agentNamePromise: Promise | undefined; + +/** + * Detects the AI agent driving this process, if any. Memoized because the + * result is used both for telemetry events and for the user-agent header + * on every API request. + */ +export function detectAgentName(): Promise { + if (!agentNamePromise) { + agentNamePromise = determineAgent().then( + (result) => (result.isAgent ? result.agent.name : undefined), + () => undefined, + ); + } + return agentNamePromise; +} diff --git a/packages/sandbox/src/telemetry/index.ts b/packages/sandbox/src/telemetry/index.ts new file mode 100644 index 00000000..b2babd93 --- /dev/null +++ b/packages/sandbox/src/telemetry/index.ts @@ -0,0 +1,172 @@ +import { randomUUID } from "node:crypto"; +import { mkdirSync, readFileSync, writeFileSync } from "node:fs"; +import { join } from "node:path"; +import os from "node:os"; +import xdgAppPaths from "xdg-app-paths"; +import createDebugger from "debug"; +import { version } from "../pkg"; +import { detectAgentName } from "./agent"; + +const debug = createDebugger("sandbox:telemetry"); + +const BRIDGE_URL = "https://telemetry.vercel.com/api/sandbox-cli/v1/events"; +const FLUSH_TIMEOUT_MS = 1_500; + +interface Event { + id: string; + event_time: number; + key: string; + value: string; +} + +function configFilePath(): string { + return join(xdgAppPaths("sandbox-cli").config(), "telemetry.json"); +} + +function readJson(path: string): Record | undefined { + try { + return JSON.parse(readFileSync(path, "utf8")); + } catch { + return undefined; + } +} + +export function readTelemetryConfig(): { enabled?: boolean } | undefined { + return readJson(configFilePath()); +} + +export function writeTelemetryConfig(enabled: boolean): void { + const dir = xdgAppPaths("sandbox-cli").config(); + mkdirSync(dir, { recursive: true }); + writeFileSync(configFilePath(), JSON.stringify({ enabled }, null, 2)); +} + +/** + * Reads the Vercel CLI's global telemetry preference so that running as + * `vercel sandbox` respects an existing `vercel telemetry disable`. + */ +function vercelCliTelemetryDisabled(): boolean { + const config = readJson(join(xdgAppPaths("com.vercel.cli").config(), "config.json")); + const telemetry = config?.telemetry as { enabled?: boolean } | undefined; + return telemetry?.enabled === false; +} + +// Keep in sync with the commands registered in app.ts (including aliases). +const SUBCOMMANDS = new Set([ + "list", + "ls", + "create", + "sh", + "fork", + "config", + "copy", + "cp", + "exec", + "connect", + "ssh", + "shell", + "stop", + "remove", + "rm", + "run", + "snapshot", + "snapshots", + "sessions", + "login", + "logout", + "telemetry", +]); + +export class Telemetry { + private events: Event[] = []; + private readonly sessionId = randomUUID(); + private teamId = "NO_TEAM_ID"; + private embedded = false; + + get enabled(): boolean { + if ( + process.env.VERCEL_SANDBOX_TELEMETRY_DISABLED || + process.env.VERCEL_TELEMETRY_DISABLED + ) { + return false; + } + if (this.embedded && vercelCliTelemetryDisabled()) { + return false; + } + return readTelemetryConfig()?.enabled ?? true; + } + + get isDebug(): boolean { + return Boolean(process.env.VERCEL_TELEMETRY_DEBUG); + } + + track(key: string, value: string | undefined): void { + if (!value) return; + this.events.push({ id: randomUUID(), event_time: Date.now(), key, value }); + } + + updateTeamId(teamId: string | undefined): void { + if (teamId) this.teamId = teamId; + } + + async trackInvocation(opts: { appName: string; argv: string[] }): Promise { + this.embedded = opts.appName !== "sandbox"; + // Only ever record known subcommand names, so option values that happen + // to precede the subcommand (tokens, paths) can never end up in an event. + const subcommand = opts.argv.find((arg) => SUBCOMMANDS.has(arg)); + this.track("subcommand", subcommand); + this.track("agent", await detectAgentName()); + this.track("version", version); + this.track("platform", os.platform()); + this.track("arch", os.arch()); + this.track("ci", process.env.CI ? "TRUE" : undefined); + this.track("embedded", this.embedded ? opts.appName : undefined); + } + + /** + * Sends buffered events to the telemetry bridge. Never throws and never + * blocks the CLI for longer than FLUSH_TIMEOUT_MS. With + * VERCEL_TELEMETRY_DEBUG set, events are printed to stderr and not sent. + */ + async flush(): Promise { + if (this.events.length === 0) return; + + const events = this.events.map((event) => ({ + ...event, + team_id: this.teamId, + session_id: this.sessionId, + })); + this.events = []; + + if (this.isDebug) { + for (const event of events) { + process.stderr.write(`[telemetry] ${JSON.stringify(event)}\n`); + } + return; + } + + if (!this.enabled) return; + + try { + const response = await fetch( + process.env.VERCEL_SANDBOX_TELEMETRY_BRIDGE_URL || BRIDGE_URL, + { + method: "POST", + headers: { + "content-type": "application/json", + "client-id": "sandbox-cli", + "x-sandbox-cli-topic-id": "generic", + "x-sandbox-cli-session-id": this.sessionId, + }, + body: JSON.stringify(events), + signal: AbortSignal.timeout(FLUSH_TIMEOUT_MS), + }, + ); + debug("telemetry bridge responded with %d", response.status); + } catch (error) { + debug("failed to send telemetry events: %o", error); + } + } +} + +export const telemetry = new Telemetry(); diff --git a/packages/sandbox/src/telemetry/telemetry.test.ts b/packages/sandbox/src/telemetry/telemetry.test.ts new file mode 100644 index 00000000..4c1b4643 --- /dev/null +++ b/packages/sandbox/src/telemetry/telemetry.test.ts @@ -0,0 +1,134 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +const configRoot = mkdtempSync(join(tmpdir(), "sandbox-telemetry-test-")); + +vi.mock("xdg-app-paths", () => ({ + default: (name: string) => ({ + config: () => join(configRoot, name), + cache: () => join(configRoot, name, "cache"), + }), +})); + +import { Telemetry, writeTelemetryConfig } from "./index"; + +describe("telemetry", () => { + const fetchMock = vi.fn(async () => new Response(null, { status: 204 })); + + beforeEach(() => { + vi.stubGlobal("fetch", fetchMock); + vi.stubEnv("VERCEL_TELEMETRY_DEBUG", ""); + vi.stubEnv("VERCEL_TELEMETRY_DISABLED", ""); + vi.stubEnv("VERCEL_SANDBOX_TELEMETRY_DISABLED", ""); + }); + + afterEach(() => { + vi.unstubAllGlobals(); + vi.unstubAllEnvs(); + fetchMock.mockClear(); + rmSync(configRoot, { recursive: true, force: true }); + }); + + it("sends buffered events with session and team metadata", async () => { + const telemetry = new Telemetry(); + telemetry.track("subcommand", "create"); + telemetry.updateTeamId("team_123"); + await telemetry.flush(); + + expect(fetchMock).toHaveBeenCalledOnce(); + const [url, init] = fetchMock.mock.calls[0] as unknown as [ + string, + RequestInit, + ]; + expect(url).toBe("https://telemetry.vercel.com/api/sandbox-cli/v1/events"); + const headers = new Headers(init.headers); + expect(headers.get("client-id")).toBe("sandbox-cli"); + expect(headers.get("x-sandbox-cli-topic-id")).toBe("generic"); + expect(headers.get("x-sandbox-cli-session-id")).toBeTruthy(); + + const events = JSON.parse(String(init.body)); + expect(events).toHaveLength(1); + expect(events[0]).toMatchObject({ + key: "subcommand", + value: "create", + team_id: "team_123", + }); + expect(events[0].id).toBeTruthy(); + expect(events[0].session_id).toBeTruthy(); + }); + + it("does not send anything when disabled via environment", async () => { + vi.stubEnv("VERCEL_SANDBOX_TELEMETRY_DISABLED", "1"); + const telemetry = new Telemetry(); + telemetry.track("subcommand", "create"); + await telemetry.flush(); + + expect(fetchMock).not.toHaveBeenCalled(); + }); + + it("does not send anything when disabled via config file", async () => { + writeTelemetryConfig(false); + const telemetry = new Telemetry(); + telemetry.track("subcommand", "create"); + await telemetry.flush(); + + expect(fetchMock).not.toHaveBeenCalled(); + }); + + it("prints events to stderr instead of sending in debug mode", async () => { + vi.stubEnv("VERCEL_TELEMETRY_DEBUG", "1"); + const stderr = vi + .spyOn(process.stderr, "write") + .mockImplementation(() => true); + + const telemetry = new Telemetry(); + telemetry.track("subcommand", "create"); + await telemetry.flush(); + + expect(fetchMock).not.toHaveBeenCalled(); + expect(stderr).toHaveBeenCalledWith( + expect.stringContaining("[telemetry]"), + ); + stderr.mockRestore(); + }); + + it("swallows network errors from the bridge", async () => { + fetchMock.mockRejectedValueOnce(new Error("network down")); + const telemetry = new Telemetry(); + telemetry.track("subcommand", "create"); + await expect(telemetry.flush()).resolves.toBeUndefined(); + }); + + it("tracks the invocation subcommand and environment facts", async () => { + const telemetry = new Telemetry(); + await telemetry.trackInvocation({ + appName: "sandbox", + argv: ["--token", "create"], + }); + await telemetry.flush(); + + const events = JSON.parse(String(fetchMock.mock.calls[0]?.[1]?.body)); + const keys = events.map((event: { key: string }) => event.key); + expect(keys).toContain("subcommand"); + expect(keys).toContain("version"); + expect(keys).toContain("platform"); + expect(keys).toContain("arch"); + expect(keys).not.toContain("embedded"); + }); + + it("marks embedded invocations", async () => { + const telemetry = new Telemetry(); + await telemetry.trackInvocation({ + appName: "vercel sandbox", + argv: ["ls"], + }); + await telemetry.flush(); + + const events = JSON.parse(String(fetchMock.mock.calls[0]?.[1]?.body)); + expect(events).toContainEqual( + expect.objectContaining({ key: "embedded", value: "vercel sandbox" }), + ); + }); +}); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 700feba2..eab145d1 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -340,6 +340,9 @@ importers: date-fns: specifier: ^4.1.0 version: 4.1.0 + detect-agent: + specifier: 1.2.0 + version: 1.2.0 dotenv-flow: specifier: ^4.1.0 version: 4.1.0 @@ -3477,6 +3480,10 @@ packages: resolution: {integrity: sha512-2sJGJTaXIIaR1w4iJSNoN0hnMY7Gpc/n8D4qSCJw8QqFWXf7cuAgnEHxBpweaVcPevC2l3KpjYCx3NypQQgaJg==} engines: {node: '>= 0.8', npm: 1.2.8000 || >= 1.4.16} + detect-agent@1.2.0: + resolution: {integrity: sha512-fW/515FHIogLopGDTSMc4XXkJV6mkVC10gVPoenb7cgY7PZAF/jakLDdDUBQ9QpJEA9AYgEg3/B9WqLTUtz3Ag==} + engines: {node: '>=20'} + detect-indent@6.1.0: resolution: {integrity: sha512-reYkTUJAZb9gUuZ2RvVCNhVHdg62RHnJ7WJl8ftMi4diZ6NWlciOzQN88pUhSELEwflJht4oQDv0F0BMlwaYtA==} engines: {node: '>=8'} @@ -9425,6 +9432,8 @@ snapshots: destroy@1.2.0: {} + detect-agent@1.2.0: {} + detect-indent@6.1.0: {} detect-libc@2.0.4: {} From 8d3f2e5bf50500db1267936c3b577e80096d719c Mon Sep 17 00:00:00 2001 From: Elisabeth Rulke Date: Thu, 27 Aug 2026 15:13:41 -0400 Subject: [PATCH 2/6] [cli] fix fetch mock typing in telemetry tests --- .../sandbox/src/telemetry/telemetry.test.ts | 25 +++++++++++-------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/packages/sandbox/src/telemetry/telemetry.test.ts b/packages/sandbox/src/telemetry/telemetry.test.ts index 4c1b4643..70c0b23f 100644 --- a/packages/sandbox/src/telemetry/telemetry.test.ts +++ b/packages/sandbox/src/telemetry/telemetry.test.ts @@ -15,7 +15,15 @@ vi.mock("xdg-app-paths", () => ({ import { Telemetry, writeTelemetryConfig } from "./index"; describe("telemetry", () => { - const fetchMock = vi.fn(async () => new Response(null, { status: 204 })); + const fetchMock = vi.fn( + async (..._args: Parameters) => + new Response(null, { status: 204 }), + ); + + function sentEvents(): Array> { + const init = fetchMock.mock.calls[0]?.[1]; + return JSON.parse(String(init?.body)); + } beforeEach(() => { vi.stubGlobal("fetch", fetchMock); @@ -38,17 +46,14 @@ describe("telemetry", () => { await telemetry.flush(); expect(fetchMock).toHaveBeenCalledOnce(); - const [url, init] = fetchMock.mock.calls[0] as unknown as [ - string, - RequestInit, - ]; + const [url, init] = fetchMock.mock.calls[0]!; expect(url).toBe("https://telemetry.vercel.com/api/sandbox-cli/v1/events"); - const headers = new Headers(init.headers); + const headers = new Headers(init?.headers); expect(headers.get("client-id")).toBe("sandbox-cli"); expect(headers.get("x-sandbox-cli-topic-id")).toBe("generic"); expect(headers.get("x-sandbox-cli-session-id")).toBeTruthy(); - const events = JSON.parse(String(init.body)); + const events = sentEvents(); expect(events).toHaveLength(1); expect(events[0]).toMatchObject({ key: "subcommand", @@ -109,8 +114,7 @@ describe("telemetry", () => { }); await telemetry.flush(); - const events = JSON.parse(String(fetchMock.mock.calls[0]?.[1]?.body)); - const keys = events.map((event: { key: string }) => event.key); + const keys = sentEvents().map((event) => event.key); expect(keys).toContain("subcommand"); expect(keys).toContain("version"); expect(keys).toContain("platform"); @@ -126,8 +130,7 @@ describe("telemetry", () => { }); await telemetry.flush(); - const events = JSON.parse(String(fetchMock.mock.calls[0]?.[1]?.body)); - expect(events).toContainEqual( + expect(sentEvents()).toContainEqual( expect.objectContaining({ key: "embedded", value: "vercel sandbox" }), ); }); From b23b9bad98e67f6883f0c5648376b0b651608890 Mon Sep 17 00:00:00 2001 From: Elisabeth Rulke Date: Thu, 27 Aug 2026 17:35:44 -0400 Subject: [PATCH 3/6] [sdk] tag API requests with the detected AI agent --- .changeset/tall-lions-shake.md | 8 ++++ packages/sandbox/src/client.test.ts | 37 +++++++++++++++++++ packages/sandbox/src/client.ts | 14 ++++--- packages/vercel-sandbox/package.json | 1 + .../src/api-client/api-client.ts | 8 +++- .../vercel-sandbox/src/utils/detect-agent.ts | 17 +++++++++ pnpm-lock.yaml | 3 ++ 7 files changed, 82 insertions(+), 6 deletions(-) create mode 100644 .changeset/tall-lions-shake.md create mode 100644 packages/sandbox/src/client.test.ts create mode 100644 packages/vercel-sandbox/src/utils/detect-agent.ts diff --git a/.changeset/tall-lions-shake.md b/.changeset/tall-lions-shake.md new file mode 100644 index 00000000..398b7c96 --- /dev/null +++ b/.changeset/tall-lions-shake.md @@ -0,0 +1,8 @@ +--- +"@vercel/sandbox": minor +--- + +Tag API requests with the AI agent driving the process, when one is +detected via `detect-agent`, as an `agent/` phrase in the user-agent +header. No agent detected means no change to the header. The SDK sends no +telemetry events; this is request metadata only. diff --git a/packages/sandbox/src/client.test.ts b/packages/sandbox/src/client.test.ts new file mode 100644 index 00000000..15cc3998 --- /dev/null +++ b/packages/sandbox/src/client.test.ts @@ -0,0 +1,37 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { sandboxClient } from "./client"; + +describe("fetchWithUserAgent", () => { + const fetchMock = vi.fn( + async (..._args: Parameters) => + new Response("{}", { + status: 200, + headers: { "content-type": "application/json" }, + }), + ); + + beforeEach(() => { + vi.stubGlobal("fetch", fetchMock); + vi.stubEnv("AI_AGENT", "test-agent"); + }); + + afterEach(() => { + vi.unstubAllGlobals(); + vi.unstubAllEnvs(); + fetchMock.mockClear(); + }); + + it("sends exactly one agent phrase when the SDK already stamped one", async () => { + await sandboxClient + .list({ token: "fake", teamId: "team_fake", projectId: "prj_fake" }) + .catch(() => undefined); + + expect(fetchMock).toHaveBeenCalled(); + const init = fetchMock.mock.calls[0]?.[1]; + const userAgent = new Headers(init?.headers).get("user-agent") ?? ""; + + expect(userAgent).toMatch(/^vercel\/sandbox-cli\//); + expect(userAgent).toContain("vercel/sandbox/"); + expect(userAgent.match(/ agent\//g)).toHaveLength(1); + }); +}); diff --git a/packages/sandbox/src/client.ts b/packages/sandbox/src/client.ts index 624ffb89..84ec8c82 100644 --- a/packages/sandbox/src/client.ts +++ b/packages/sandbox/src/client.ts @@ -64,14 +64,18 @@ const fetchWithUserAgent: typeof globalThis.fetch = async (input, init) => { ); let agent = `vercel/sandbox-cli/${version}`; + const existingAgent = headers.get("user-agent"); + // Attribute API traffic to the AI agent driving this invocation, if any, - // so the server side can record it once ingestion support lands. - const aiAgent = await detectAgentName(); - if (aiAgent) { - agent += ` agent/${aiAgent}`; + // so the server side can record it once ingestion support lands. The SDK + // stamps its own agent phrase, so skip ours when one is already present. + if (!existingAgent?.includes(" agent/")) { + const aiAgent = await detectAgentName(); + if (aiAgent) { + agent += ` agent/${aiAgent}`; + } } - const existingAgent = headers.get("user-agent"); if (existingAgent) { agent += ` ${existingAgent}`; } diff --git a/packages/vercel-sandbox/package.json b/packages/vercel-sandbox/package.json index dbdb7d12..2b05ed71 100644 --- a/packages/vercel-sandbox/package.json +++ b/packages/vercel-sandbox/package.json @@ -56,6 +56,7 @@ "@vercel/oidc": "3.2.0", "@workflow/serde": "4.1.0-beta.2", "async-retry": "1.3.3", + "detect-agent": "1.2.0", "jose": "6.2.3", "jsonlines": "0.1.1", "lru-cache": "^10.4.3", diff --git a/packages/vercel-sandbox/src/api-client/api-client.ts b/packages/vercel-sandbox/src/api-client/api-client.ts index b4448237..e8b355fd 100644 --- a/packages/vercel-sandbox/src/api-client/api-client.ts +++ b/packages/vercel-sandbox/src/api-client/api-client.ts @@ -38,6 +38,7 @@ import { getVercelOidcToken } from "@vercel/oidc"; import { NetworkPolicy } from "../network-policy.js"; import { toAPINetworkPolicy } from "../utils/network-policy.js"; import { getPrivateParams, WithPrivate } from "../utils/types.js"; +import { detectAgentName } from "../utils/detect-agent.js"; import type { RUNTIMES, SandboxRegion } from "../constants.js"; interface Claims { @@ -125,12 +126,17 @@ export class APIClient extends BaseClient { protected async request(path: string, params?: RequestParams) { await this.ensureValidToken(); + // Attribute traffic to the AI agent driving this process, if any, so the + // server side can record it once ingestion support lands. + const aiAgent = await detectAgentName(); + const agentSuffix = aiAgent ? ` agent/${aiAgent}` : ""; + return super.request(path, { ...params, query: { teamId: this.teamId, ...params?.query }, headers: { "content-type": "application/json", - "user-agent": `vercel/sandbox/${VERSION} (Node.js/${process.version}; ${os.platform()}/${os.arch()})`, + "user-agent": `vercel/sandbox/${VERSION}${agentSuffix} (Node.js/${process.version}; ${os.platform()}/${os.arch()})`, ...params?.headers, }, }); diff --git a/packages/vercel-sandbox/src/utils/detect-agent.ts b/packages/vercel-sandbox/src/utils/detect-agent.ts new file mode 100644 index 00000000..dfcab095 --- /dev/null +++ b/packages/vercel-sandbox/src/utils/detect-agent.ts @@ -0,0 +1,17 @@ +import { determineAgent } from "detect-agent"; + +let agentNamePromise: Promise | undefined; + +/** + * Detects the AI agent driving this process, if any. Memoized so the + * environment is inspected once per process, not once per request. + */ +export function detectAgentName(): Promise { + if (!agentNamePromise) { + agentNamePromise = determineAgent().then( + (result) => (result.isAgent ? result.agent.name : undefined), + () => undefined, + ); + } + return agentNamePromise; +} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index eab145d1..ee1c6c2c 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -382,6 +382,9 @@ importers: async-retry: specifier: 1.3.3 version: 1.3.3 + detect-agent: + specifier: 1.2.0 + version: 1.2.0 jose: specifier: 6.2.3 version: 6.2.3 From 9beef8e2d7dd14ffb4d07fc3aa45374fe5d76b05 Mon Sep 17 00:00:00 2001 From: Elisabeth Rulke Date: Fri, 28 Aug 2026 12:57:11 -0400 Subject: [PATCH 4/6] [cli] address telemetry review: session ids, exit code, scope, correct vercel config path --- packages/sandbox/package.json | 3 +- packages/sandbox/src/client.ts | 58 +++++++++++++------ packages/sandbox/src/index.ts | 4 ++ packages/sandbox/src/sandbox.ts | 2 + packages/sandbox/src/telemetry/agent.ts | 2 +- packages/sandbox/src/telemetry/index.ts | 45 ++++++++++++-- .../sandbox/src/telemetry/telemetry.test.ts | 50 +++++++++++++++- pnpm-lock.yaml | 29 +++++++--- 8 files changed, 155 insertions(+), 38 deletions(-) diff --git a/packages/sandbox/package.json b/packages/sandbox/package.json index 2e31c012..777caba2 100644 --- a/packages/sandbox/package.json +++ b/packages/sandbox/package.json @@ -50,12 +50,13 @@ "@types/ms": "^2.1.0", "@types/node": "^22.15.12", "@types/ws": "^8.18.1", + "@vercel/cli-config": "0.2.4", + "@vercel/detect-agent": "1.2.5", "@vercel/oidc": "^3.2.0", "@vercel/sandbox": "workspace:*", "chalk": "^5.6.0", "cmd-ts": "0.15.0", "date-fns": "^4.1.0", - "detect-agent": "1.2.0", "dotenv-flow": "^4.1.0", "listr2": "^9.0.2", "ms": "^2.1.3", diff --git a/packages/sandbox/src/client.ts b/packages/sandbox/src/client.ts index 624ffb89..0303723f 100644 --- a/packages/sandbox/src/client.ts +++ b/packages/sandbox/src/client.ts @@ -13,23 +13,29 @@ export const sandboxClient: Pick< "get" | "list" | "create" | "fork" > = { get: (params) => - withErrorHandling(() => { - telemetry.updateTeamId(teamIdOf(params)); - return Sandbox.get({ fetch: fetchWithUserAgent, resume: false, ...params }); + withErrorHandling(async () => { + updateScope(params); + const sandbox = await Sandbox.get({ fetch: fetchWithUserAgent, resume: false, ...params }); + trackSession(sandbox, "attached"); + return sandbox; }), create: (params) => - withErrorHandling(() => { - telemetry.updateTeamId(teamIdOf(params)); - return Sandbox.create({ fetch: fetchWithUserAgent, ...params }); + withErrorHandling(async () => { + updateScope(params); + const sandbox = await Sandbox.create({ fetch: fetchWithUserAgent, ...params }); + trackSession(sandbox, "created"); + return sandbox; }), fork: (params) => - withErrorHandling(() => { - telemetry.updateTeamId(teamIdOf(params)); - return Sandbox.fork({ fetch: fetchWithUserAgent, ...params }); + withErrorHandling(async () => { + updateScope(params); + const sandbox = await Sandbox.fork({ fetch: fetchWithUserAgent, ...params }); + trackSession(sandbox, "created"); + return sandbox; }), list: (params) => withErrorHandling(() => { - telemetry.updateTeamId(teamIdOf(params)); + updateScope(params); return Sandbox.list({ fetch: fetchWithUserAgent, ...params } as typeof params); }), }; @@ -47,14 +53,27 @@ export const snapshotClient: Pick< withErrorHandling(() => Snapshot.tree({ fetch: fetchWithUserAgent, ...params })), }; -function teamIdOf(params: unknown): string | undefined { - if (params && typeof params === "object" && "teamId" in params) { - const { teamId } = params as { teamId?: unknown }; - if (typeof teamId === "string") return teamId; +function scopeField(params: unknown, field: string): string | undefined { + if (params && typeof params === "object" && field in params) { + const value = (params as Record)[field]; + if (typeof value === "string") return value; } return undefined; } +function updateScope(params: unknown): void { + telemetry.updateTeamId(scopeField(params, "teamId")); + telemetry.updateProjectId(scopeField(params, "projectId")); +} + +function trackSession(sandbox: Sandbox, origin: "created" | "attached"): void { + try { + telemetry.trackSandboxSession(sandbox.currentSession().sessionId, origin); + } catch { + // No active session on this instance; nothing to record. + } +} + const fetchWithUserAgent: typeof globalThis.fetch = async (input, init) => { const headers = new Headers( init?.headers ?? @@ -65,10 +84,13 @@ const fetchWithUserAgent: typeof globalThis.fetch = async (input, init) => { let agent = `vercel/sandbox-cli/${version}`; // Attribute API traffic to the AI agent driving this invocation, if any, - // so the server side can record it once ingestion support lands. - const aiAgent = await detectAgentName(); - if (aiAgent) { - agent += ` agent/${aiAgent}`; + // so the server side can record it once ingestion support lands. Gated on + // the telemetry setting so opting out covers agent attribution too. + if (telemetry.enabled) { + const aiAgent = await detectAgentName(); + if (aiAgent) { + agent += ` agent/${aiAgent}`; + } } const existingAgent = headers.get("user-agent"); diff --git a/packages/sandbox/src/index.ts b/packages/sandbox/src/index.ts index 67bb3e80..74ff657f 100644 --- a/packages/sandbox/src/index.ts +++ b/packages/sandbox/src/index.ts @@ -9,6 +9,10 @@ export function createApp(opts: { withoutAuth: boolean; appName: string }) { await telemetry.trackInvocation({ appName: opts.appName, argv: args }); try { await runCmd(instance, args); + telemetry.trackExitCode(0); + } catch (error) { + telemetry.trackExitCode(1); + throw error; } finally { await telemetry.flush(); } diff --git a/packages/sandbox/src/sandbox.ts b/packages/sandbox/src/sandbox.ts index 6febed12..a9c42d96 100644 --- a/packages/sandbox/src/sandbox.ts +++ b/packages/sandbox/src/sandbox.ts @@ -19,8 +19,10 @@ async function main() { try { steerShCommand(argv); await run(app(), argv); + telemetry.trackExitCode(0); await telemetry.flush(); } catch (e) { + telemetry.trackExitCode(1); await telemetry.flush(); await printTopLevelError(e); process.exit(1); diff --git a/packages/sandbox/src/telemetry/agent.ts b/packages/sandbox/src/telemetry/agent.ts index 81a9a081..e5841a93 100644 --- a/packages/sandbox/src/telemetry/agent.ts +++ b/packages/sandbox/src/telemetry/agent.ts @@ -1,4 +1,4 @@ -import { determineAgent } from "detect-agent"; +import { determineAgent } from "@vercel/detect-agent"; let agentNamePromise: Promise | undefined; diff --git a/packages/sandbox/src/telemetry/index.ts b/packages/sandbox/src/telemetry/index.ts index b2babd93..c2003074 100644 --- a/packages/sandbox/src/telemetry/index.ts +++ b/packages/sandbox/src/telemetry/index.ts @@ -3,6 +3,7 @@ import { mkdirSync, readFileSync, writeFileSync } from "node:fs"; import { join } from "node:path"; import os from "node:os"; import xdgAppPaths from "xdg-app-paths"; +import { getConfigFilePath, getGlobalPathConfig } from "@vercel/cli-config/paths"; import createDebugger from "debug"; import { version } from "../pkg"; import { detectAgentName } from "./agent"; @@ -44,11 +45,17 @@ export function writeTelemetryConfig(enabled: boolean): void { /** * Reads the Vercel CLI's global telemetry preference so that running as * `vercel sandbox` respects an existing `vercel telemetry disable`. + * getConfigFilePath resolves the same directory (and legacy fallbacks) + * the Vercel CLI actually writes to. */ function vercelCliTelemetryDisabled(): boolean { - const config = readJson(join(xdgAppPaths("com.vercel.cli").config(), "config.json")); - const telemetry = config?.telemetry as { enabled?: boolean } | undefined; - return telemetry?.enabled === false; + try { + const config = readJson(getConfigFilePath(getGlobalPathConfig())); + const telemetry = config?.telemetry as { enabled?: boolean } | undefined; + return telemetry?.enabled === false; + } catch { + return false; + } } // Keep in sync with the commands registered in app.ts (including aliases). @@ -81,6 +88,7 @@ export class Telemetry { private events: Event[] = []; private readonly sessionId = randomUUID(); private teamId = "NO_TEAM_ID"; + private projectId: string | undefined; private embedded = false; get enabled(): boolean { @@ -109,11 +117,32 @@ export class Telemetry { if (teamId) this.teamId = teamId; } + updateProjectId(projectId: string | undefined): void { + if (projectId) this.projectId = projectId; + } + + /** + * Records the `sbx_` session id an invocation touched, and whether it + * created that session or attached to an existing one, so invocations can + * be joined to the usage-fact pipeline downstream. + */ + trackSandboxSession(sessionId: string | undefined, origin: "created" | "attached"): void { + this.track("sandbox_session_id", sessionId); + if (sessionId) this.track("sandbox_session_origin", origin); + } + + trackExitCode(code: number): void { + this.track("exit_code", String(code)); + } + async trackInvocation(opts: { appName: string; argv: string[] }): Promise { this.embedded = opts.appName !== "sandbox"; - // Only ever record known subcommand names, so option values that happen - // to precede the subcommand (tokens, paths) can never end up in an event. - const subcommand = opts.argv.find((arg) => SUBCOMMANDS.has(arg)); + // cmd-ts only dispatches on argv[0], so anchor there and only ever record + // known subcommand names: an option value elsewhere in argv (a token, a + // path, or a name that collides with a command) can never end up in an + // event, and a leading flag records nothing rather than something wrong. + const first = opts.argv[0]; + const subcommand = first && SUBCOMMANDS.has(first) ? first : undefined; this.track("subcommand", subcommand); this.track("agent", await detectAgentName()); this.track("version", version); @@ -135,6 +164,10 @@ export class Telemetry { ...event, team_id: this.teamId, session_id: this.sessionId, + project_id: this.projectId ?? null, + // Set by the Vercel CLI's sandbox wrapper so events from embedded runs + // can join the Vercel CLI's own invocation-grained telemetry. + vercel_cli_invocation_id: process.env.VERCEL_CLI_INVOCATION_ID ?? null, })); this.events = []; diff --git a/packages/sandbox/src/telemetry/telemetry.test.ts b/packages/sandbox/src/telemetry/telemetry.test.ts index 70c0b23f..2b783678 100644 --- a/packages/sandbox/src/telemetry/telemetry.test.ts +++ b/packages/sandbox/src/telemetry/telemetry.test.ts @@ -110,18 +110,62 @@ describe("telemetry", () => { const telemetry = new Telemetry(); await telemetry.trackInvocation({ appName: "sandbox", - argv: ["--token", "create"], + argv: ["create", "--token", "secret"], }); await telemetry.flush(); - const keys = sentEvents().map((event) => event.key); - expect(keys).toContain("subcommand"); + const events = sentEvents(); + expect(events).toContainEqual( + expect.objectContaining({ key: "subcommand", value: "create" }), + ); + const keys = events.map((event) => event.key); expect(keys).toContain("version"); expect(keys).toContain("platform"); expect(keys).toContain("arch"); expect(keys).not.toContain("embedded"); }); + it("records no subcommand when argv does not start with a known command", async () => { + const telemetry = new Telemetry(); + // A registered command name appearing as an option value must not win: + // cmd-ts only dispatches on argv[0]. + await telemetry.trackInvocation({ + appName: "sandbox", + argv: ["--scope", "create", "exec", "my-box"], + }); + await telemetry.flush(); + + const keys = sentEvents().map((event) => event.key); + expect(keys).not.toContain("subcommand"); + }); + + it("carries project id and the Vercel CLI invocation id on every event", async () => { + vi.stubEnv("VERCEL_CLI_INVOCATION_ID", "inv_123"); + const telemetry = new Telemetry(); + telemetry.updateProjectId("prj_123"); + telemetry.track("subcommand", "create"); + await telemetry.flush(); + + expect(sentEvents()[0]).toMatchObject({ + project_id: "prj_123", + vercel_cli_invocation_id: "inv_123", + }); + }); + + it("records sandbox session ids with their origin", async () => { + const telemetry = new Telemetry(); + telemetry.trackSandboxSession("sbx_abc", "created"); + await telemetry.flush(); + + const events = sentEvents(); + expect(events).toContainEqual( + expect.objectContaining({ key: "sandbox_session_id", value: "sbx_abc" }), + ); + expect(events).toContainEqual( + expect.objectContaining({ key: "sandbox_session_origin", value: "created" }), + ); + }); + it("marks embedded invocations", async () => { const telemetry = new Telemetry(); await telemetry.trackInvocation({ diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index eab145d1..75aef320 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -328,6 +328,12 @@ importers: '@types/ws': specifier: ^8.18.1 version: 8.18.1 + '@vercel/cli-config': + specifier: 0.2.4 + version: 0.2.4 + '@vercel/detect-agent': + specifier: 1.2.5 + version: 1.2.5 '@vercel/oidc': specifier: ^3.2.0 version: 3.2.0 @@ -340,9 +346,6 @@ importers: date-fns: specifier: ^4.1.0 version: 4.1.0 - detect-agent: - specifier: 1.2.0 - version: 1.2.0 dotenv-flow: specifier: ^4.1.0 version: 4.1.0 @@ -2580,6 +2583,13 @@ packages: '@vercel/cli-auth@0.0.1': resolution: {integrity: sha512-CnqiuMlZ4pjs2LCPYiR6aLKPPd3Xb8SBI1Y7eotXKgpx6qgrGNY+E7EIyUt5ErGHJGIrCZyGG5WEo4bHtVmz2Q==} + '@vercel/cli-config@0.2.4': + resolution: {integrity: sha512-kZ5SojbrV06GHoU6QIWGwDXLov+s9rWZ7QqdqKfJfBGCNUieGfgaCjeeenNy8Y+QC0bwC0dZ2B4l5Hvdmrgpdw==} + + '@vercel/detect-agent@1.2.5': + resolution: {integrity: sha512-0krENrjuitlW8s6TJu0MlqCevyCU7K7JK63jZAf7xZ6n17tx+vUEwzHT3sTxawtwZxaW21hu+oFUpoOrm49FsQ==} + engines: {node: '>=14'} + '@vercel/functions@3.4.3': resolution: {integrity: sha512-kA14KIUVgAY6VXbhZ5jjY+s0883cV3cZqIU3WhrSRxuJ9KvxatMjtmzl0K23HK59oOUjYl7HaE/eYMmhmqpZzw==} engines: {node: '>= 20'} @@ -3480,10 +3490,6 @@ packages: resolution: {integrity: sha512-2sJGJTaXIIaR1w4iJSNoN0hnMY7Gpc/n8D4qSCJw8QqFWXf7cuAgnEHxBpweaVcPevC2l3KpjYCx3NypQQgaJg==} engines: {node: '>= 0.8', npm: 1.2.8000 || >= 1.4.16} - detect-agent@1.2.0: - resolution: {integrity: sha512-fW/515FHIogLopGDTSMc4XXkJV6mkVC10gVPoenb7cgY7PZAF/jakLDdDUBQ9QpJEA9AYgEg3/B9WqLTUtz3Ag==} - engines: {node: '>=20'} - detect-indent@6.1.0: resolution: {integrity: sha512-reYkTUJAZb9gUuZ2RvVCNhVHdg62RHnJ7WJl8ftMi4diZ6NWlciOzQN88pUhSELEwflJht4oQDv0F0BMlwaYtA==} engines: {node: '>=8'} @@ -8217,6 +8223,13 @@ snapshots: xdg-app-paths: 5.1.0 zod: 4.1.11 + '@vercel/cli-config@0.2.4': + dependencies: + xdg-app-paths: 5.1.0 + zod: 4.1.11 + + '@vercel/detect-agent@1.2.5': {} + '@vercel/functions@3.4.3(@aws-sdk/credential-provider-web-identity@3.972.13)': dependencies: '@vercel/oidc': 3.2.0 @@ -9432,8 +9445,6 @@ snapshots: destroy@1.2.0: {} - detect-agent@1.2.0: {} - detect-indent@6.1.0: {} detect-libc@2.0.4: {} From 25a4b6cc48c7b1b2d10f1bd8511b588264cc674f Mon Sep 17 00:00:00 2001 From: Elisabeth Rulke Date: Fri, 28 Aug 2026 13:03:49 -0400 Subject: [PATCH 5/6] [sdk] address review: use @vercel/detect-agent, honor telemetry opt-out env --- packages/vercel-sandbox/package.json | 2 +- packages/vercel-sandbox/src/utils/detect-agent.ts | 10 +++++++++- pnpm-lock.yaml | 6 +++--- 3 files changed, 13 insertions(+), 5 deletions(-) diff --git a/packages/vercel-sandbox/package.json b/packages/vercel-sandbox/package.json index 2b05ed71..8b36fea7 100644 --- a/packages/vercel-sandbox/package.json +++ b/packages/vercel-sandbox/package.json @@ -53,10 +53,10 @@ }, "license": "Apache-2.0", "dependencies": { + "@vercel/detect-agent": "1.2.5", "@vercel/oidc": "3.2.0", "@workflow/serde": "4.1.0-beta.2", "async-retry": "1.3.3", - "detect-agent": "1.2.0", "jose": "6.2.3", "jsonlines": "0.1.1", "lru-cache": "^10.4.3", diff --git a/packages/vercel-sandbox/src/utils/detect-agent.ts b/packages/vercel-sandbox/src/utils/detect-agent.ts index dfcab095..9662fdc9 100644 --- a/packages/vercel-sandbox/src/utils/detect-agent.ts +++ b/packages/vercel-sandbox/src/utils/detect-agent.ts @@ -1,12 +1,20 @@ -import { determineAgent } from "detect-agent"; +import { determineAgent } from "@vercel/detect-agent"; let agentNamePromise: Promise | undefined; /** * Detects the AI agent driving this process, if any. Memoized so the * environment is inspected once per process, not once per request. + * Attribution honors the telemetry opt-out variables: disabling telemetry + * also stops the agent phrase from being added to the user-agent header. */ export function detectAgentName(): Promise { + if ( + process.env.VERCEL_TELEMETRY_DISABLED || + process.env.VERCEL_SANDBOX_TELEMETRY_DISABLED + ) { + return Promise.resolve(undefined); + } if (!agentNamePromise) { agentNamePromise = determineAgent().then( (result) => (result.isAgent ? result.agent.name : undefined), diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index d7bcd34b..016bee56 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -376,6 +376,9 @@ importers: packages/vercel-sandbox: dependencies: + '@vercel/detect-agent': + specifier: 1.2.5 + version: 1.2.5 '@vercel/oidc': specifier: 3.2.0 version: 3.2.0 @@ -385,9 +388,6 @@ importers: async-retry: specifier: 1.3.3 version: 1.3.3 - detect-agent: - specifier: 1.2.0 - version: 1.2.0 jose: specifier: 6.2.3 version: 6.2.3 From 8bcf7da40670243898798dfbede93b0df17e249c Mon Sep 17 00:00:00 2001 From: Elisabeth Rulke Date: Mon, 31 Aug 2026 08:37:34 -0700 Subject: [PATCH 6/6] [cli] enforce config-file opt-out by stripping the SDK agent phrase --- packages/sandbox/src/client.test.ts | 33 +++++++++++++++++++++++++++++ packages/sandbox/src/client.ts | 24 +++++++++++++-------- 2 files changed, 48 insertions(+), 9 deletions(-) diff --git a/packages/sandbox/src/client.test.ts b/packages/sandbox/src/client.test.ts index 15cc3998..a5487cca 100644 --- a/packages/sandbox/src/client.test.ts +++ b/packages/sandbox/src/client.test.ts @@ -1,5 +1,19 @@ import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +const configRoot = mkdtempSync(join(tmpdir(), "sandbox-client-test-")); + +vi.mock("xdg-app-paths", () => ({ + default: (name: string) => ({ + config: () => join(configRoot, name), + cache: () => join(configRoot, name, "cache"), + }), +})); + import { sandboxClient } from "./client"; +import { writeTelemetryConfig } from "./telemetry"; describe("fetchWithUserAgent", () => { const fetchMock = vi.fn( @@ -19,6 +33,7 @@ describe("fetchWithUserAgent", () => { vi.unstubAllGlobals(); vi.unstubAllEnvs(); fetchMock.mockClear(); + rmSync(configRoot, { recursive: true, force: true }); }); it("sends exactly one agent phrase when the SDK already stamped one", async () => { @@ -34,4 +49,22 @@ describe("fetchWithUserAgent", () => { expect(userAgent).toContain("vercel/sandbox/"); expect(userAgent.match(/ agent\//g)).toHaveLength(1); }); + + it("strips the SDK's agent phrase on a config-file opt-out", async () => { + // `sandbox telemetry disable` writes the config file but sets no env + // vars, so the SDK (which gates on env only) still stamps its phrase. + // The wrapper must enforce the opt-out by stripping it from the header. + writeTelemetryConfig(false); + + await sandboxClient + .list({ token: "fake", teamId: "team_fake", projectId: "prj_fake" }) + .catch(() => undefined); + + const init = fetchMock.mock.calls[0]?.[1]; + const userAgent = new Headers(init?.headers).get("user-agent") ?? ""; + + expect(userAgent).toMatch(/^vercel\/sandbox-cli\//); + expect(userAgent).toContain("vercel/sandbox/"); + expect(userAgent).not.toContain(" agent/"); + }); }); diff --git a/packages/sandbox/src/client.ts b/packages/sandbox/src/client.ts index 32a75f3a..0ba90f24 100644 --- a/packages/sandbox/src/client.ts +++ b/packages/sandbox/src/client.ts @@ -83,17 +83,23 @@ const fetchWithUserAgent: typeof globalThis.fetch = async (input, init) => { ); let agent = `vercel/sandbox-cli/${version}`; - const existingAgent = headers.get("user-agent"); + let existingAgent = headers.get("user-agent"); - // Attribute API traffic to the AI agent driving this invocation, if any, - // so the server side can record it once ingestion support lands. Gated on - // the telemetry setting so opting out covers agent attribution too; the - // SDK stamps its own phrase, so skip ours when one is already present. - if (telemetry.enabled && !existingAgent?.includes(" agent/")) { - const aiAgent = await detectAgentName(); - if (aiAgent) { - agent += ` agent/${aiAgent}`; + if (telemetry.enabled) { + // Attribute API traffic to the AI agent driving this invocation, if any, + // so the server side can record it once ingestion support lands. The SDK + // stamps its own phrase, so skip ours when one is already present. + if (!existingAgent?.includes(" agent/")) { + const aiAgent = await detectAgentName(); + if (aiAgent) { + agent += ` agent/${aiAgent}`; + } } + } else if (existingAgent) { + // The SDK gates its stamp on env vars only, so a config-file opt-out + // (`sandbox telemetry disable`) must be enforced here: strip any agent + // phrase from the header rather than trusting upstream gates. + existingAgent = existingAgent.replace(/ agent\/\S+/g, ""); } if (existingAgent) {