From 54623528f8bf4a15e27c7ff282a8c56bd79aacd9 Mon Sep 17 00:00:00 2001 From: Calvin Remsburg Date: Wed, 27 May 2026 20:30:04 -0500 Subject: [PATCH] fix(redteam): suppress spurious multi_turn_error_message when multi_turn=false (#202) Adds sanitizeTargetMetadata() at the SDK normalizer boundary plus the probe path so targets probe / create / update / get no longer surface the "Multi turn configuration JSON not provided" message when the operator never opted into multi-turn. Field is preserved when multi_turn=true, where it reflects a real configuration error. --- ...dteam-targets-suppress-multi-turn-error.md | 5 ++ src/airs/redteam.ts | 25 +++++- tests/unit/airs/redteam.spec.ts | 85 +++++++++++++++++++ 3 files changed, 113 insertions(+), 2 deletions(-) create mode 100644 .changeset/fix-redteam-targets-suppress-multi-turn-error.md diff --git a/.changeset/fix-redteam-targets-suppress-multi-turn-error.md b/.changeset/fix-redteam-targets-suppress-multi-turn-error.md new file mode 100644 index 0000000..c088b79 --- /dev/null +++ b/.changeset/fix-redteam-targets-suppress-multi-turn-error.md @@ -0,0 +1,5 @@ +--- +"@cdot65/prisma-airs-cli": patch +--- + +Suppress the spurious `target_metadata.multi_turn_error_message` ("Multi turn configuration JSON not provided") that `airs redteam targets probe`, `create`, `update`, and `get` were showing whenever `multi_turn: false`. The field is only kept when `multi_turn: true`, where it indicates an actual misconfiguration. diff --git a/src/airs/redteam.ts b/src/airs/redteam.ts index 8b9e8e8..f79bf89 100644 --- a/src/airs/redteam.ts +++ b/src/airs/redteam.ts @@ -45,6 +45,22 @@ function normalizeJob(raw: Record): RedTeamJob { }; } +/** + * Drop noisy "you didn't opt in" fields from `target_metadata` when the + * corresponding feature is disabled. `multi_turn_error_message` always comes + * back populated, but it's only a real error when `multi_turn === true`. + */ +export function sanitizeTargetMetadata | undefined>( + metadata: T, +): T { + if (!metadata) return metadata; + if (metadata.multi_turn === false && 'multi_turn_error_message' in metadata) { + const { multi_turn_error_message: _drop, ...rest } = metadata; + return rest as T; + } + return metadata; +} + /** Normalize an SDK target response into a RedTeamTargetDetail. */ function normalizeTargetDetail(raw: Record): RedTeamTargetDetail { return { @@ -65,7 +81,7 @@ function normalizeTargetDetail(raw: Record): RedTeamTargetDetai connectionParams: raw.connection_params as Record | undefined, background: raw.target_background as RedTeamTargetDetail['background'], additionalContext: raw.additional_context as RedTeamTargetDetail['additionalContext'], - metadata: raw.target_metadata as RedTeamTargetDetail['metadata'], + metadata: sanitizeTargetMetadata(raw.target_metadata as RedTeamTargetDetail['metadata']), }; } @@ -274,7 +290,12 @@ export class SdkRedTeamService implements RedTeamService { async probeTarget(request: Record): Promise> { const response = await this.client.targets.probe(request as never); - return response as unknown as Record; + const raw = response as unknown as Record; + const meta = raw.target_metadata; + if (meta && typeof meta === 'object') { + raw.target_metadata = sanitizeTargetMetadata(meta as Record); + } + return raw; } async getTargetProfile(uuid: string): Promise> { diff --git a/tests/unit/airs/redteam.spec.ts b/tests/unit/airs/redteam.spec.ts index dd88a82..a6370eb 100644 --- a/tests/unit/airs/redteam.spec.ts +++ b/tests/unit/airs/redteam.spec.ts @@ -753,6 +753,91 @@ describe('SdkRedTeamService', () => { expect(result).toEqual({ status: 'connected', latency_ms: 42 }); expect(mockTargetsProbe).toHaveBeenCalledWith({ api_endpoint: 'https://example.com' }); }); + + it('strips multi_turn_error_message from target_metadata when multi_turn is false', async () => { + mockTargetsProbe.mockResolvedValue({ + status: 'connected', + target_metadata: { + multi_turn: false, + multi_turn_error_message: 'Multi turn configuration JSON not provided', + rate_limit: 50, + }, + }); + const result = await service.probeTarget({ api_endpoint: 'https://example.com' }); + const meta = result.target_metadata as Record; + expect(meta).not.toHaveProperty('multi_turn_error_message'); + expect(meta.multi_turn).toBe(false); + expect(meta.rate_limit).toBe(50); + }); + + it('keeps multi_turn_error_message when multi_turn is true (real error)', async () => { + mockTargetsProbe.mockResolvedValue({ + status: 'connected', + target_metadata: { + multi_turn: true, + multi_turn_error_message: 'something actually wrong', + }, + }); + const result = await service.probeTarget({ api_endpoint: 'https://example.com' }); + const meta = result.target_metadata as Record; + expect(meta.multi_turn_error_message).toBe('something actually wrong'); + }); + }); + + describe('multi_turn_error_message sanitization on normalized target paths', () => { + it('getTarget: drops multi_turn_error_message when multi_turn is false', async () => { + mockTargetsGet.mockResolvedValue({ + uuid: 't-1', + name: 'Target', + status: 'active', + active: true, + target_metadata: { + multi_turn: false, + multi_turn_error_message: 'Multi turn configuration JSON not provided', + rate_limit: 25, + }, + }); + const result = await service.getTarget('t-1'); + expect(result.metadata).not.toHaveProperty('multi_turn_error_message'); + expect(result.metadata).toEqual({ multi_turn: false, rate_limit: 25 }); + }); + + it('getTarget: keeps multi_turn_error_message when multi_turn is true', async () => { + mockTargetsGet.mockResolvedValue({ + uuid: 't-1', + name: 'Target', + status: 'active', + active: true, + target_metadata: { + multi_turn: true, + multi_turn_error_message: 'real failure', + }, + }); + const result = await service.getTarget('t-1'); + expect(result.metadata).toEqual({ + multi_turn: true, + multi_turn_error_message: 'real failure', + }); + }); + + it('createTarget: drops multi_turn_error_message when multi_turn is false', async () => { + mockTargetsCreate.mockResolvedValue({ + uuid: 't-new', + name: 'New', + status: 'active', + active: true, + target_metadata: { + multi_turn: false, + multi_turn_error_message: 'Multi turn configuration JSON not provided', + }, + }); + const result = await service.createTarget({ + name: 'New', + target_type: 'REST', + connection_params: {}, + }); + expect(result.metadata).toEqual({ multi_turn: false }); + }); }); describe('getTargetProfile', () => {