From e2cfd0e89302461bd02861233b1f828736777aca Mon Sep 17 00:00:00 2001 From: diegosouzapw <8016841+diegosouzapw@users.noreply.github.com> Date: Mon, 17 Aug 2026 14:56:11 -0300 Subject: [PATCH] feat(accounting): normalize provider usage evidence Add a fail-closed physical-usage contract that distinguishes provider reports, estimates and byte-only evidence without changing the cache-weighted baseline model. Co-authored-by: Alexandre Teixeira Inspired-by: https://github.com/teamchong/pxpipe/pull/222 --- CHANGELOG.md | 7 + src/core/accounting.ts | 224 +++++++++++++++++++++++++ src/core/index.ts | 8 + tests/accounting-normalization.test.ts | 186 ++++++++++++++++++++ 4 files changed, 425 insertions(+) create mode 100644 src/core/accounting.ts create mode 100644 tests/accounting-normalization.test.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index b089dbd..b942f4a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,13 @@ Format: [Keep a Changelog](https://keepachangelog.com/en/1.1.0/) · semantic ver ## [Unreleased] +### Added + +- **feat(accounting):** provider-neutral physical-usage normalization with + explicit evidence grades, fail-closed counter validation, cache-bucket + semantics for Anthropic and OpenAI-compatible usage, and honest negative + reductions. (thanks @alteixeira20) + ### Fixed - **fix(render):** glyph surgery so the Spleen 5×8 `K` no longer reads as `H`. diff --git a/src/core/accounting.ts b/src/core/accounting.ts new file mode 100644 index 0000000..d677db5 --- /dev/null +++ b/src/core/accounting.ts @@ -0,0 +1,224 @@ +/** + * Provider-neutral physical-usage normalization. + * + * This module classifies the evidence available on one request. It deliberately + * does not replace baseline.ts: cache-weighted counterfactual cost and savings + * remain a separate economic calculation. + */ + +export type AccountingProvider = 'anthropic' | 'openai' | 'xai' | 'unknown'; +export type SavingsEvidence = + | 'provider-reported' + | 'estimated' + | 'bytes-only' + | 'unavailable'; + +export interface AccountingInput { + provider: AccountingProvider; + model?: string; + originalBytes?: number; + transformedBytes?: number; + estimatedOriginalInputTokens?: number; + estimatedTransformedInputTokens?: number; + providerBaselineInputTokens?: number; + providerInputTokens?: number; + providerOutputTokens?: number; + cacheReadTokens?: number; + cacheWriteTokens?: number; + imageTokens?: number; + proxyAddedLatencyMs?: number; + modelLatencyMs?: number; + fallbackCount?: number; + bypassReason?: string; +} + +export interface NormalizedAccounting { + provider: AccountingProvider; + model?: string; + bytes: { + original?: number; + transformed?: number; + reduced?: number; + compressionRatio?: number; + }; + tokens: { + providerReportedOriginalInput?: number; + providerReportedActualInput?: number; + providerReportedReduced?: number; + estimatedOriginalInput?: number; + estimatedActualInput?: number; + estimatedReduced?: number; + cacheRead?: number; + cacheWrite?: number; + image?: number; + output?: number; + total?: number; + }; + savings: { + evidence: SavingsEvidence; + inputTokensReduced?: number; + inputReductionRatio?: number; + }; + latency: { + proxyAddedMs?: number; + modelMs?: number; + }; + fallbackCount: number; + bypassReason?: string; +} + +function nonNegativeCount(value: number | undefined): number | undefined { + return typeof value === 'number' && Number.isSafeInteger(value) && value >= 0 + ? value + : undefined; +} + +function nonNegativeMeasure(value: number | undefined): number | undefined { + return typeof value === 'number' && Number.isFinite(value) && value >= 0 + ? value + : undefined; +} + +function difference( + before: number | undefined, + after: number | undefined, +): number | undefined { + if (before === undefined || after === undefined) return undefined; + return before - after; +} + +function ratio( + reduced: number | undefined, + baseline: number | undefined, +): number | undefined { + if (reduced === undefined || baseline === undefined || baseline <= 0) return undefined; + return reduced / baseline; +} + +function safeSum(values: readonly number[]): number | undefined { + let total = 0; + for (const value of values) { + total += value; + if (!Number.isSafeInteger(total) || total < 0) return undefined; + } + return total; +} + +function suppliedInvalidCount(value: number | undefined): boolean { + return value !== undefined && nonNegativeCount(value) === undefined; +} + +/** + * Normalize provider usage into physical input tokens. + * + * Anthropic's input, cache-create and cache-read values are disjoint buckets. + * OpenAI and xAI are consumed through OmniGlyph's OpenAI-compatible usage + * contract, where cached tokens are a diagnostic subset of input; adding the + * subset would double count. Unknown providers fail closed whenever cache + * semantics would have to be guessed. + */ +export function providerActualInputTokens(input: AccountingInput): number | undefined { + const reported = nonNegativeCount(input.providerInputTokens); + if (reported === undefined) return undefined; + + const cacheRead = nonNegativeCount(input.cacheReadTokens); + const cacheWrite = nonNegativeCount(input.cacheWriteTokens); + if (input.provider === 'anthropic') { + if ( + suppliedInvalidCount(input.cacheReadTokens) || + suppliedInvalidCount(input.cacheWriteTokens) + ) return undefined; + return safeSum([reported, cacheRead ?? 0, cacheWrite ?? 0]); + } + + if (input.provider === 'unknown') { + if (input.cacheReadTokens !== undefined || input.cacheWriteTokens !== undefined) { + return undefined; + } + return reported; + } + + return reported; +} + +export function normalizeAccounting(input: AccountingInput): NormalizedAccounting { + const originalBytes = nonNegativeCount(input.originalBytes); + const transformedBytes = nonNegativeCount(input.transformedBytes); + const bytesReduced = difference(originalBytes, transformedBytes); + + const providerOriginal = nonNegativeCount(input.providerBaselineInputTokens); + const providerActual = providerActualInputTokens(input); + const providerReduced = difference(providerOriginal, providerActual); + + const estimatedOriginal = nonNegativeCount(input.estimatedOriginalInputTokens); + const estimatedActual = nonNegativeCount(input.estimatedTransformedInputTokens); + const estimatedReduced = difference(estimatedOriginal, estimatedActual); + + let evidence: SavingsEvidence = 'unavailable'; + let inputTokensReduced: number | undefined; + let inputReductionRatio: number | undefined; + if (providerReduced !== undefined) { + evidence = 'provider-reported'; + inputTokensReduced = providerReduced; + inputReductionRatio = ratio(providerReduced, providerOriginal); + } else if (estimatedReduced !== undefined) { + evidence = 'estimated'; + inputTokensReduced = estimatedReduced; + inputReductionRatio = ratio(estimatedReduced, estimatedOriginal); + } else if (bytesReduced !== undefined) { + evidence = 'bytes-only'; + } + + const cacheRead = nonNegativeCount(input.cacheReadTokens); + const cacheWrite = nonNegativeCount(input.cacheWriteTokens); + const image = nonNegativeCount(input.imageTokens); + const output = nonNegativeCount(input.providerOutputTokens); + const total = providerActual !== undefined && output !== undefined + ? safeSum([providerActual, output]) + : undefined; + const proxyAddedMs = nonNegativeMeasure(input.proxyAddedLatencyMs); + const modelMs = nonNegativeMeasure(input.modelLatencyMs); + + return { + provider: input.provider, + ...(input.model ? { model: input.model } : {}), + bytes: { + ...(originalBytes !== undefined ? { original: originalBytes } : {}), + ...(transformedBytes !== undefined ? { transformed: transformedBytes } : {}), + ...(bytesReduced !== undefined ? { reduced: bytesReduced } : {}), + ...(originalBytes !== undefined && originalBytes > 0 && transformedBytes !== undefined + ? { compressionRatio: transformedBytes / originalBytes } + : {}), + }, + tokens: { + ...(providerOriginal !== undefined + ? { providerReportedOriginalInput: providerOriginal } + : {}), + ...(providerActual !== undefined + ? { providerReportedActualInput: providerActual } + : {}), + ...(providerReduced !== undefined ? { providerReportedReduced: providerReduced } : {}), + ...(estimatedOriginal !== undefined + ? { estimatedOriginalInput: estimatedOriginal } + : {}), + ...(estimatedActual !== undefined ? { estimatedActualInput: estimatedActual } : {}), + ...(estimatedReduced !== undefined ? { estimatedReduced } : {}), + ...(cacheRead !== undefined ? { cacheRead } : {}), + ...(cacheWrite !== undefined ? { cacheWrite } : {}), + ...(image !== undefined ? { image } : {}), + ...(output !== undefined ? { output } : {}), + ...(total !== undefined ? { total } : {}), + }, + savings: { + evidence, + ...(inputTokensReduced !== undefined ? { inputTokensReduced } : {}), + ...(inputReductionRatio !== undefined ? { inputReductionRatio } : {}), + }, + latency: { + ...(proxyAddedMs !== undefined ? { proxyAddedMs } : {}), + ...(modelMs !== undefined ? { modelMs } : {}), + }, + fallbackCount: nonNegativeCount(input.fallbackCount) ?? 0, + ...(input.bypassReason ? { bypassReason: input.bypassReason } : {}), + }; +} diff --git a/src/core/index.ts b/src/core/index.ts index 93fdb4e..e99c5d4 100644 --- a/src/core/index.ts +++ b/src/core/index.ts @@ -48,3 +48,11 @@ export { CACHE_CREATE_RATE, CACHE_READ_RATE, } from './baseline.js'; +export { + normalizeAccounting, + providerActualInputTokens, + type AccountingInput, + type AccountingProvider, + type NormalizedAccounting, + type SavingsEvidence, +} from './accounting.js'; diff --git a/tests/accounting-normalization.test.ts b/tests/accounting-normalization.test.ts new file mode 100644 index 0000000..1aad9da --- /dev/null +++ b/tests/accounting-normalization.test.ts @@ -0,0 +1,186 @@ +import { describe, expect, it } from 'vitest'; + +import { + normalizeAccounting, + providerActualInputTokens, +} from '../src/core/index.js'; + +describe('provider input normalization', () => { + it('sums Anthropic disjoint input and cache buckets', () => { + expect(providerActualInputTokens({ + provider: 'anthropic', + providerInputTokens: 100, + cacheReadTokens: 700, + cacheWriteTokens: 200, + })).toBe(1_000); + }); + + it.each(['openai', 'xai'] as const)( + 'does not double-count the cache subset for %s usage', + (provider) => { + expect(providerActualInputTokens({ + provider, + providerInputTokens: 1_000, + cacheReadTokens: 700, + })).toBe(1_000); + }, + ); + + it.each([-1, Number.NaN, 1.5, Number.MAX_SAFE_INTEGER + 1])( + 'fails closed on malformed Anthropic cache-read counter %s', + (cacheReadTokens) => { + expect(providerActualInputTokens({ + provider: 'anthropic', + providerInputTokens: 1_000, + cacheReadTokens, + })).toBeUndefined(); + }, + ); + + it('fails closed when Anthropic bucket summation exceeds safe integer precision', () => { + expect(providerActualInputTokens({ + provider: 'anthropic', + providerInputTokens: Number.MAX_SAFE_INTEGER, + cacheWriteTokens: 1, + })).toBeUndefined(); + }); + + it('does not guess cache semantics for an unknown provider', () => { + expect(providerActualInputTokens({ + provider: 'unknown', + providerInputTokens: 1_000, + cacheReadTokens: 700, + })).toBeUndefined(); + expect(providerActualInputTokens({ + provider: 'unknown', + providerInputTokens: 1_000, + })).toBe(1_000); + }); +}); + +describe('normalized accounting evidence', () => { + it('prefers provider-reported reduction over estimates', () => { + const result = normalizeAccounting({ + provider: 'anthropic', + model: 'claude-fable-5', + originalBytes: 100_000, + transformedBytes: 25_000, + providerBaselineInputTokens: 20_000, + providerInputTokens: 1_000, + cacheReadTokens: 2_000, + cacheWriteTokens: 500, + providerOutputTokens: 300, + estimatedOriginalInputTokens: 25_000, + estimatedTransformedInputTokens: 4_000, + imageTokens: 1_200, + proxyAddedLatencyMs: 85, + modelLatencyMs: 2_400, + fallbackCount: 1, + }); + + expect(result.savings).toEqual({ + evidence: 'provider-reported', + inputTokensReduced: 16_500, + inputReductionRatio: 0.825, + }); + expect(result.tokens.providerReportedActualInput).toBe(3_500); + expect(result.tokens.total).toBe(3_800); + expect(result.tokens.estimatedReduced).toBe(21_000); + expect(result.bytes).toEqual({ + original: 100_000, + transformed: 25_000, + reduced: 75_000, + compressionRatio: 0.25, + }); + }); + + it('labels savings as estimated when no provider baseline exists', () => { + const result = normalizeAccounting({ + provider: 'openai', + estimatedOriginalInputTokens: 10_000, + estimatedTransformedInputTokens: 4_000, + }); + expect(result.savings).toEqual({ + evidence: 'estimated', + inputTokensReduced: 6_000, + inputReductionRatio: 0.6, + }); + }); + + it('falls back to estimates when provider evidence is malformed', () => { + const result = normalizeAccounting({ + provider: 'anthropic', + providerBaselineInputTokens: 10_000, + providerInputTokens: 2_000, + cacheReadTokens: -1, + estimatedOriginalInputTokens: 10_000, + estimatedTransformedInputTokens: 4_000, + }); + expect(result.tokens.providerReportedActualInput).toBeUndefined(); + expect(result.tokens.providerReportedReduced).toBeUndefined(); + expect(result.savings.evidence).toBe('estimated'); + expect(result.savings.inputTokensReduced).toBe(6_000); + }); + + it('reports bytes-only evidence without inventing token savings', () => { + const result = normalizeAccounting({ + provider: 'unknown', + originalBytes: 1_000, + transformedBytes: 400, + bypassReason: 'provider_usage_unavailable', + }); + expect(result.savings).toEqual({ evidence: 'bytes-only' }); + expect(result.bytes.reduced).toBe(600); + expect(result.bypassReason).toBe('provider_usage_unavailable'); + }); + + it('never converts malformed discrete counters into savings', () => { + const result = normalizeAccounting({ + provider: 'openai', + providerBaselineInputTokens: -1, + providerInputTokens: 10.5, + estimatedOriginalInputTokens: Number.NaN, + estimatedTransformedInputTokens: 20, + originalBytes: 100.5, + transformedBytes: 20, + fallbackCount: 1.5, + }); + expect(result.savings.evidence).toBe('unavailable'); + expect(result.bytes.original).toBeUndefined(); + expect(result.tokens.providerReportedActualInput).toBeUndefined(); + expect(result.tokens.estimatedOriginalInput).toBeUndefined(); + expect(result.fallbackCount).toBe(0); + }); + + it('allows fractional latency while rejecting negative latency', () => { + const result = normalizeAccounting({ + provider: 'openai', + proxyAddedLatencyMs: 12.75, + modelLatencyMs: -1, + }); + expect(result.latency).toEqual({ proxyAddedMs: 12.75 }); + }); + + it('preserves measured expansion as a negative reduction', () => { + const result = normalizeAccounting({ + provider: 'openai', + providerBaselineInputTokens: 1_000, + providerInputTokens: 1_250, + }); + expect(result.savings).toEqual({ + evidence: 'provider-reported', + inputTokensReduced: -250, + inputReductionRatio: -0.25, + }); + }); + + it('does not emit an unsafe total-token sum', () => { + const result = normalizeAccounting({ + provider: 'openai', + providerInputTokens: Number.MAX_SAFE_INTEGER, + providerOutputTokens: 1, + }); + expect(result.tokens.providerReportedActualInput).toBe(Number.MAX_SAFE_INTEGER); + expect(result.tokens.total).toBeUndefined(); + }); +});