From 21abfe7ea5649b58ec9d832d4a47fc8ecb18e1cb Mon Sep 17 00:00:00 2001 From: Pau Date: Tue, 7 Jul 2026 15:48:04 +0200 Subject: [PATCH 1/5] feat(billing): add HTTP event-name suffix env var Adds BILLING_EVENTS_HTTP_EVENT_NAME_SUFFIX, an optional suffix appended to the Orb event_name emitted by the HTTP-fed billing path. Default is empty so behavior is unchanged in every environment where the var is not set. Prepares the ground for the S3-fed cutover: once dev flips this to "_http" (and clears the S3 suffix), HTTP-emitted events keep flowing into Orb under a distinct name and remain queryable via /prices/evaluate as a defensive backup, without competing with the S3-emitted events for the canonical billable-metric filter. Co-Authored-By: Claude Opus 4.7 --- packages/billing/lib/clients/orb/adapters.ts | 3 ++- .../billing/lib/clients/orb/adapters.unit.test.ts | 12 ++++++++++++ packages/utils/lib/environment/parse.ts | 4 ++++ 3 files changed, 18 insertions(+), 1 deletion(-) diff --git a/packages/billing/lib/clients/orb/adapters.ts b/packages/billing/lib/clients/orb/adapters.ts index bab20a2f03..a8daf1207f 100644 --- a/packages/billing/lib/clients/orb/adapters.ts +++ b/packages/billing/lib/clients/orb/adapters.ts @@ -2,6 +2,7 @@ import { uuidv7 } from 'uuidv7'; import { Err, Ok } from '@nangohq/utils'; +import { envs } from '../../envs.js'; import { putOrbCustomerSchema } from './types.js'; import type { BillingAddress, BillingCustomer, BillingEvent, BillingInvoicingDetails, Result, UsageMetric } from '@nangohq/types'; @@ -24,7 +25,7 @@ export function toOrbEvent(event: BillingEvent): Orb.Events.EventIngestParams.Ev } return { - event_name: event.type, + event_name: `${event.type}${envs.BILLING_EVENTS_HTTP_EVENT_NAME_SUFFIX ?? ''}`, idempotency_key: idempotencyKey || uuidv7(), external_customer_id: accountId.toString(), timestamp: timestamp.toISOString(), diff --git a/packages/billing/lib/clients/orb/adapters.unit.test.ts b/packages/billing/lib/clients/orb/adapters.unit.test.ts index be6de11117..9476cf6f17 100644 --- a/packages/billing/lib/clients/orb/adapters.unit.test.ts +++ b/packages/billing/lib/clients/orb/adapters.unit.test.ts @@ -1,5 +1,6 @@ import { describe, expect, it, vi } from 'vitest'; +import { envs } from '../../envs.js'; import { fromOrbAddress, fromOrbCustomer, orbMetricToUsageMetric, toOrbEvent, toOrbPutCustomerPayload } from './adapters.js'; import type { BillingEvent, BillingInvoicingDetails } from '@nangohq/types'; @@ -66,6 +67,17 @@ describe('toOrbEvent', () => { expect(result.external_customer_id).toBe('42'); expect(result.timestamp).toBe('2024-01-15T10:00:00.000Z'); }); + + it('appends BILLING_EVENTS_HTTP_EVENT_NAME_SUFFIX to event_name when set', () => { + const original = envs.BILLING_EVENTS_HTTP_EVENT_NAME_SUFFIX; + try { + (envs as any).BILLING_EVENTS_HTTP_EVENT_NAME_SUFFIX = '_http'; + const event: BillingEvent = { type: 'proxy', properties: { ...baseProperties } as any }; + expect(toOrbEvent(event).event_name).toBe('proxy_http'); + } finally { + (envs as any).BILLING_EVENTS_HTTP_EVENT_NAME_SUFFIX = original; + } + }); }); // ─── toOrbPutCustomerPayload ────────────────────────────────────────────────── diff --git a/packages/utils/lib/environment/parse.ts b/packages/utils/lib/environment/parse.ts index 463f54052f..eacebce3b9 100644 --- a/packages/utils/lib/environment/parse.ts +++ b/packages/utils/lib/environment/parse.ts @@ -350,6 +350,10 @@ export const ENVS = z.object({ BILLING_EVENTS_S3_BUCKET: z.string().optional(), BILLING_EVENTS_S3_WRITER_ROLE_ARN: z.string().optional(), BILLING_EVENTS_S3_EVENT_NAME_SUFFIX: z.string().optional(), + // Temporary. Appended to HTTP-fed Orb event_name during the S3-fed + // cutover so HTTP emissions remain queryable as a defensive backup. + // Remove once the HTTP emission path is retired. + BILLING_EVENTS_HTTP_EVENT_NAME_SUFFIX: z.string().optional(), BILLING_EVENTS_S3_REGION: z.string().optional().default('us-west-2'), // ClickHouse From 93d90b11ff11fff70a87f24864719de54cb2be0c Mon Sep 17 00:00:00 2001 From: Pau Date: Thu, 9 Jul 2026 10:12:30 +0200 Subject: [PATCH 2/5] feat(billing): gate HTTP/S3 event-name suffix swap on a cutover date MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Introduces BILLING_EVENTS_CUTOVER_AT — an ISO 8601 timestamp that flips which pipeline emits events under the canonical (unsuffixed) name customer billable metrics filter on: - Before the timestamp (or when it's unset): S3 events keep their pre-cutover suffix (BILLING_EVENTS_S3_EVENT_NAME_SUFFIX); HTTP events stay unsuffixed and continue to feed customer plans. - At and after the timestamp: S3 events become unsuffixed (canonical); HTTP events pick up BILLING_EVENTS_HTTP_EVENT_NAME_SUFFIX ("_http" per the deploy config) and become the defensive shadow. Deployment can now merge and roll out ahead of the cutover date and the suffix swap happens automatically at the configured instant. Rollback = unset (or clear, or push to a future date) the env var and redeploy. Malformed dates fail hard at Zod parse time via z.string().datetime() so a typo can't silently defer the cutover. Co-Authored-By: Claude Opus 4.7 --- packages/billing/lib/clients/orb/adapters.ts | 12 +++++++++- .../lib/clients/orb/adapters.unit.test.ts | 23 ++++++++++++++++--- .../lib/crons/billingEventsS3Export.ts | 12 +++++++++- packages/utils/lib/environment/parse.ts | 18 ++++++++++++--- 4 files changed, 57 insertions(+), 8 deletions(-) diff --git a/packages/billing/lib/clients/orb/adapters.ts b/packages/billing/lib/clients/orb/adapters.ts index a8daf1207f..cbbc5beaba 100644 --- a/packages/billing/lib/clients/orb/adapters.ts +++ b/packages/billing/lib/clients/orb/adapters.ts @@ -8,6 +8,15 @@ import { putOrbCustomerSchema } from './types.js'; import type { BillingAddress, BillingCustomer, BillingEvent, BillingInvoicingDetails, Result, UsageMetric } from '@nangohq/types'; import type Orb from 'orb-billing'; +// True once the S3-fed pipeline is authoritative for billing (see +// BILLING_EVENTS_CUTOVER_AT in packages/utils). While this returns false, +// HTTP-emitted events stay unsuffixed so customer billable metrics keep +// matching them. Once it flips true, HTTP events pick up the "_http" +// suffix and become the defensive shadow. +function cutoverActive(): boolean { + return !!envs.BILLING_EVENTS_CUTOVER_AT && new Date() >= new Date(envs.BILLING_EVENTS_CUTOVER_AT); +} + export function toOrbEvent(event: BillingEvent): Orb.Events.EventIngestParams.Event { const { idempotencyKey, timestamp, accountId, ...rest } = event.properties; @@ -24,8 +33,9 @@ export function toOrbEvent(event: BillingEvent): Orb.Events.EventIngestParams.Ev } } + const suffix = cutoverActive() ? (envs.BILLING_EVENTS_HTTP_EVENT_NAME_SUFFIX ?? '') : ''; return { - event_name: `${event.type}${envs.BILLING_EVENTS_HTTP_EVENT_NAME_SUFFIX ?? ''}`, + event_name: `${event.type}${suffix}`, idempotency_key: idempotencyKey || uuidv7(), external_customer_id: accountId.toString(), timestamp: timestamp.toISOString(), diff --git a/packages/billing/lib/clients/orb/adapters.unit.test.ts b/packages/billing/lib/clients/orb/adapters.unit.test.ts index 9476cf6f17..4b369d8a37 100644 --- a/packages/billing/lib/clients/orb/adapters.unit.test.ts +++ b/packages/billing/lib/clients/orb/adapters.unit.test.ts @@ -68,14 +68,31 @@ describe('toOrbEvent', () => { expect(result.timestamp).toBe('2024-01-15T10:00:00.000Z'); }); - it('appends BILLING_EVENTS_HTTP_EVENT_NAME_SUFFIX to event_name when set', () => { - const original = envs.BILLING_EVENTS_HTTP_EVENT_NAME_SUFFIX; + it('appends the HTTP suffix to event_name once the cutover has passed', () => { + const originalSuffix = envs.BILLING_EVENTS_HTTP_EVENT_NAME_SUFFIX; + const originalCutover = envs.BILLING_EVENTS_CUTOVER_AT; try { (envs as any).BILLING_EVENTS_HTTP_EVENT_NAME_SUFFIX = '_http'; + (envs as any).BILLING_EVENTS_CUTOVER_AT = '2000-01-01T00:00:00Z'; const event: BillingEvent = { type: 'proxy', properties: { ...baseProperties } as any }; expect(toOrbEvent(event).event_name).toBe('proxy_http'); } finally { - (envs as any).BILLING_EVENTS_HTTP_EVENT_NAME_SUFFIX = original; + (envs as any).BILLING_EVENTS_HTTP_EVENT_NAME_SUFFIX = originalSuffix; + (envs as any).BILLING_EVENTS_CUTOVER_AT = originalCutover; + } + }); + + it('does not append the HTTP suffix before the cutover instant', () => { + const originalSuffix = envs.BILLING_EVENTS_HTTP_EVENT_NAME_SUFFIX; + const originalCutover = envs.BILLING_EVENTS_CUTOVER_AT; + try { + (envs as any).BILLING_EVENTS_HTTP_EVENT_NAME_SUFFIX = '_http'; + (envs as any).BILLING_EVENTS_CUTOVER_AT = '9999-01-01T00:00:00Z'; + const event: BillingEvent = { type: 'proxy', properties: { ...baseProperties } as any }; + expect(toOrbEvent(event).event_name).toBe('proxy'); + } finally { + (envs as any).BILLING_EVENTS_HTTP_EVENT_NAME_SUFFIX = originalSuffix; + (envs as any).BILLING_EVENTS_CUTOVER_AT = originalCutover; } }); }); diff --git a/packages/metering/lib/crons/billingEventsS3Export.ts b/packages/metering/lib/crons/billingEventsS3Export.ts index 25239db48d..92fbc233a6 100644 --- a/packages/metering/lib/crons/billingEventsS3Export.ts +++ b/packages/metering/lib/crons/billingEventsS3Export.ts @@ -17,7 +17,16 @@ const cronMinute = envs.CRON_BILLING_EVENTS_S3_HOURLY_EXPORT_MINUTE; const bucket = envs.BILLING_EVENTS_S3_BUCKET; const roleArn = envs.BILLING_EVENTS_S3_WRITER_ROLE_ARN; const region = envs.BILLING_EVENTS_S3_REGION; -const eventNameSuffix = envs.BILLING_EVENTS_S3_EVENT_NAME_SUFFIX ?? ''; + +// True once the S3-fed pipeline is authoritative for billing (see +// BILLING_EVENTS_CUTOVER_AT in packages/utils). While this returns false, +// S3 events keep their pre-cutover suffix (e.g. "_shadow") so they don't +// collide with the HTTP-fed canonical events. Once it flips true, S3 +// events land under the canonical (unsuffixed) name customer billable +// metrics filter on. +function cutoverActive(): boolean { + return !!envs.BILLING_EVENTS_CUTOVER_AT && new Date() >= new Date(envs.BILLING_EVENTS_CUTOVER_AT); +} const LOCK_KEY = 'lock:cron:billingEventsS3Export'; // Cron fires hourly; lock should expire well before the next tick. @@ -176,6 +185,7 @@ export async function exec(): Promise { return; } const day = yesterdayUTC(); + const eventNameSuffix = cutoverActive() ? '' : (envs.BILLING_EVENTS_S3_EVENT_NAME_SUFFIX ?? ''); let anyFailure = false; try { for (const metric of METRICS) { diff --git a/packages/utils/lib/environment/parse.ts b/packages/utils/lib/environment/parse.ts index eacebce3b9..17d28b21d5 100644 --- a/packages/utils/lib/environment/parse.ts +++ b/packages/utils/lib/environment/parse.ts @@ -349,11 +349,23 @@ export const ENVS = z.object({ BILLING_INGEST_MAX_RETRY: z.coerce.number().optional().default(3), BILLING_EVENTS_S3_BUCKET: z.string().optional(), BILLING_EVENTS_S3_WRITER_ROLE_ARN: z.string().optional(), + // Temporary. Applied to S3-emitted Orb event_name only BEFORE the + // S3-fed cutover (i.e. while BILLING_EVENTS_CUTOVER_AT is unset or in + // the future). After the cutover, S3 events land unsuffixed under the + // canonical event names. BILLING_EVENTS_S3_EVENT_NAME_SUFFIX: z.string().optional(), - // Temporary. Appended to HTTP-fed Orb event_name during the S3-fed - // cutover so HTTP emissions remain queryable as a defensive backup. - // Remove once the HTTP emission path is retired. + // Temporary. Applied to HTTP-fed Orb event_name only AFTER the + // S3-fed cutover (i.e. once BILLING_EVENTS_CUTOVER_AT has passed) so + // HTTP emissions remain queryable as a defensive backup. Remove once + // the HTTP emission path is retired. BILLING_EVENTS_HTTP_EVENT_NAME_SUFFIX: z.string().optional(), + // Temporary. ISO 8601 timestamp at which the S3-fed pipeline becomes + // authoritative for billing. Before this instant, S3 events are + // shadow (suffixed) and HTTP events are canonical (unsuffixed). At + // and after this instant, the two swap roles. Unset (or set to a + // future date) to defer or roll back the cutover. Remove once the + // HTTP emission path is retired. + BILLING_EVENTS_CUTOVER_AT: z.string().datetime().optional(), BILLING_EVENTS_S3_REGION: z.string().optional().default('us-west-2'), // ClickHouse From 290b323afb4f4426c55e8d0f780691d21cbcc4db Mon Sep 17 00:00:00 2001 From: Pau Date: Thu, 9 Jul 2026 16:47:22 +0200 Subject: [PATCH 3/5] refactor(billing): hardcode "_http" and "_s3" suffixes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Per review feedback: the cutover suffixes were never going to vary at runtime — always "_http" on the HTTP path post-cutover and "_s3" on the S3 path pre-cutover. Two env vars just for holding two string constants was unnecessary config surface for a temporary mechanism. Drops BILLING_EVENTS_S3_EVENT_NAME_SUFFIX (pre-existing) and BILLING_EVENTS_HTTP_EVENT_NAME_SUFFIX (added earlier in this PR), inlines the two literals at the emission sites, and updates the unit tests to key solely on BILLING_EVENTS_CUTOVER_AT — the one env var that genuinely varies per environment. Dev's Orb workspace previously had the shadow S3 events named "_poc"; that value is being retired here. Post-deploy, dev shadow-S3 events will land under "_s3" like prod, so any dev Orb billable-metric filter still matching on "_poc" needs a one-time filter update. Dev's cutover date is in the past so the pre-cutover suffix is only briefly observed during initial deploy. Co-Authored-By: Claude Opus 4.7 --- packages/billing/lib/clients/orb/adapters.ts | 3 +-- .../lib/clients/orb/adapters.unit.test.ts | 10 ++------- .../lib/crons/billingEventsS3Export.ts | 2 +- packages/utils/lib/environment/parse.ts | 21 ++++++------------- 4 files changed, 10 insertions(+), 26 deletions(-) diff --git a/packages/billing/lib/clients/orb/adapters.ts b/packages/billing/lib/clients/orb/adapters.ts index cbbc5beaba..8daa24a08e 100644 --- a/packages/billing/lib/clients/orb/adapters.ts +++ b/packages/billing/lib/clients/orb/adapters.ts @@ -33,9 +33,8 @@ export function toOrbEvent(event: BillingEvent): Orb.Events.EventIngestParams.Ev } } - const suffix = cutoverActive() ? (envs.BILLING_EVENTS_HTTP_EVENT_NAME_SUFFIX ?? '') : ''; return { - event_name: `${event.type}${suffix}`, + event_name: `${event.type}${cutoverActive() ? '_http' : ''}`, idempotency_key: idempotencyKey || uuidv7(), external_customer_id: accountId.toString(), timestamp: timestamp.toISOString(), diff --git a/packages/billing/lib/clients/orb/adapters.unit.test.ts b/packages/billing/lib/clients/orb/adapters.unit.test.ts index 4b369d8a37..6022d09e41 100644 --- a/packages/billing/lib/clients/orb/adapters.unit.test.ts +++ b/packages/billing/lib/clients/orb/adapters.unit.test.ts @@ -68,30 +68,24 @@ describe('toOrbEvent', () => { expect(result.timestamp).toBe('2024-01-15T10:00:00.000Z'); }); - it('appends the HTTP suffix to event_name once the cutover has passed', () => { - const originalSuffix = envs.BILLING_EVENTS_HTTP_EVENT_NAME_SUFFIX; + it('appends the "_http" suffix to event_name once the cutover has passed', () => { const originalCutover = envs.BILLING_EVENTS_CUTOVER_AT; try { - (envs as any).BILLING_EVENTS_HTTP_EVENT_NAME_SUFFIX = '_http'; (envs as any).BILLING_EVENTS_CUTOVER_AT = '2000-01-01T00:00:00Z'; const event: BillingEvent = { type: 'proxy', properties: { ...baseProperties } as any }; expect(toOrbEvent(event).event_name).toBe('proxy_http'); } finally { - (envs as any).BILLING_EVENTS_HTTP_EVENT_NAME_SUFFIX = originalSuffix; (envs as any).BILLING_EVENTS_CUTOVER_AT = originalCutover; } }); - it('does not append the HTTP suffix before the cutover instant', () => { - const originalSuffix = envs.BILLING_EVENTS_HTTP_EVENT_NAME_SUFFIX; + it('does not append the "_http" suffix before the cutover instant', () => { const originalCutover = envs.BILLING_EVENTS_CUTOVER_AT; try { - (envs as any).BILLING_EVENTS_HTTP_EVENT_NAME_SUFFIX = '_http'; (envs as any).BILLING_EVENTS_CUTOVER_AT = '9999-01-01T00:00:00Z'; const event: BillingEvent = { type: 'proxy', properties: { ...baseProperties } as any }; expect(toOrbEvent(event).event_name).toBe('proxy'); } finally { - (envs as any).BILLING_EVENTS_HTTP_EVENT_NAME_SUFFIX = originalSuffix; (envs as any).BILLING_EVENTS_CUTOVER_AT = originalCutover; } }); diff --git a/packages/metering/lib/crons/billingEventsS3Export.ts b/packages/metering/lib/crons/billingEventsS3Export.ts index 92fbc233a6..5758a886c3 100644 --- a/packages/metering/lib/crons/billingEventsS3Export.ts +++ b/packages/metering/lib/crons/billingEventsS3Export.ts @@ -185,7 +185,7 @@ export async function exec(): Promise { return; } const day = yesterdayUTC(); - const eventNameSuffix = cutoverActive() ? '' : (envs.BILLING_EVENTS_S3_EVENT_NAME_SUFFIX ?? ''); + const eventNameSuffix = cutoverActive() ? '' : '_s3'; let anyFailure = false; try { for (const metric of METRICS) { diff --git a/packages/utils/lib/environment/parse.ts b/packages/utils/lib/environment/parse.ts index 17d28b21d5..dd0cd8eb36 100644 --- a/packages/utils/lib/environment/parse.ts +++ b/packages/utils/lib/environment/parse.ts @@ -349,22 +349,13 @@ export const ENVS = z.object({ BILLING_INGEST_MAX_RETRY: z.coerce.number().optional().default(3), BILLING_EVENTS_S3_BUCKET: z.string().optional(), BILLING_EVENTS_S3_WRITER_ROLE_ARN: z.string().optional(), - // Temporary. Applied to S3-emitted Orb event_name only BEFORE the - // S3-fed cutover (i.e. while BILLING_EVENTS_CUTOVER_AT is unset or in - // the future). After the cutover, S3 events land unsuffixed under the - // canonical event names. - BILLING_EVENTS_S3_EVENT_NAME_SUFFIX: z.string().optional(), - // Temporary. Applied to HTTP-fed Orb event_name only AFTER the - // S3-fed cutover (i.e. once BILLING_EVENTS_CUTOVER_AT has passed) so - // HTTP emissions remain queryable as a defensive backup. Remove once - // the HTTP emission path is retired. - BILLING_EVENTS_HTTP_EVENT_NAME_SUFFIX: z.string().optional(), // Temporary. ISO 8601 timestamp at which the S3-fed pipeline becomes - // authoritative for billing. Before this instant, S3 events are - // shadow (suffixed) and HTTP events are canonical (unsuffixed). At - // and after this instant, the two swap roles. Unset (or set to a - // future date) to defer or roll back the cutover. Remove once the - // HTTP emission path is retired. + // authoritative for billing. Before this instant, S3 events ship as + // "_s3" shadow and HTTP events ship canonical (unsuffixed). At + // and after this instant, the two swap roles — HTTP events pick up + // the "_http" suffix and S3 events become canonical. Unset (or set + // to a future date) to defer or roll back the cutover. Remove once + // the HTTP emission path is retired. BILLING_EVENTS_CUTOVER_AT: z.string().datetime().optional(), BILLING_EVENTS_S3_REGION: z.string().optional().default('us-west-2'), From 8f7c551400c0b4bec894f7de86eed5cb0407d2e4 Mon Sep 17 00:00:00 2001 From: Pau Date: Fri, 10 Jul 2026 11:40:17 +0200 Subject: [PATCH 4/5] fix(billing): key cutover suffix on the event's logical timestamp MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The two suffix checks were reading wall-clock time. That's wrong at the seam: the Aug 1 00:15 UTC S3 export cron fires for July 31 data, so under the old check it saw "wall clock past cutover → drop the suffix" and shipped July 31 rows under the canonical (unsuffixed) name — the same name customer plans already had HTTP-fed events under for July 31. Result: July 31 double-counts on the July invoice. Fixes both sides symmetrically: - adapters.ts (HTTP path): decision keys on the event's own timestamp, so a batched or late-emitted event whose logical time is pre-cutover stays unsuffixed regardless of when the flush happens to hit. - billingEventsS3Export.ts (S3 path): decision keys on the DATA day (`yesterdayUTC()` at cron time), so the Aug 1 firing exporting July 31 correctly ships as "_s3", and the Aug 2 firing exporting Aug 1 correctly ships unsuffixed. Adds a seam test that pins the per-event-timestamp semantic: two events straddling the cutover instant, one before and one after, verified to produce the right suffix independently. Co-Authored-By: Claude Opus 4.7 --- packages/billing/lib/clients/orb/adapters.ts | 15 ++++++----- .../lib/clients/orb/adapters.unit.test.ts | 25 +++++++++++++++++-- .../lib/crons/billingEventsS3Export.ts | 17 ++++++------- 3 files changed, 38 insertions(+), 19 deletions(-) diff --git a/packages/billing/lib/clients/orb/adapters.ts b/packages/billing/lib/clients/orb/adapters.ts index 8daa24a08e..2920d0731b 100644 --- a/packages/billing/lib/clients/orb/adapters.ts +++ b/packages/billing/lib/clients/orb/adapters.ts @@ -8,13 +8,12 @@ import { putOrbCustomerSchema } from './types.js'; import type { BillingAddress, BillingCustomer, BillingEvent, BillingInvoicingDetails, Result, UsageMetric } from '@nangohq/types'; import type Orb from 'orb-billing'; -// True once the S3-fed pipeline is authoritative for billing (see -// BILLING_EVENTS_CUTOVER_AT in packages/utils). While this returns false, -// HTTP-emitted events stay unsuffixed so customer billable metrics keep -// matching them. Once it flips true, HTTP events pick up the "_http" -// suffix and become the defensive shadow. -function cutoverActive(): boolean { - return !!envs.BILLING_EVENTS_CUTOVER_AT && new Date() >= new Date(envs.BILLING_EVENTS_CUTOVER_AT); +// Keyed on the EVENT's timestamp, not the wall clock, so a batched or +// late-emitted event whose logical time is pre-cutover ships under the +// pre-cutover name and vice versa. See BILLING_EVENTS_CUTOVER_AT in +// packages/utils. +function cutoverAppliesTo(eventTimestamp: Date): boolean { + return !!envs.BILLING_EVENTS_CUTOVER_AT && eventTimestamp >= new Date(envs.BILLING_EVENTS_CUTOVER_AT); } export function toOrbEvent(event: BillingEvent): Orb.Events.EventIngestParams.Event { @@ -34,7 +33,7 @@ export function toOrbEvent(event: BillingEvent): Orb.Events.EventIngestParams.Ev } return { - event_name: `${event.type}${cutoverActive() ? '_http' : ''}`, + event_name: `${event.type}${cutoverAppliesTo(timestamp) ? '_http' : ''}`, idempotency_key: idempotencyKey || uuidv7(), external_customer_id: accountId.toString(), timestamp: timestamp.toISOString(), diff --git a/packages/billing/lib/clients/orb/adapters.unit.test.ts b/packages/billing/lib/clients/orb/adapters.unit.test.ts index 6022d09e41..8eb3465a1b 100644 --- a/packages/billing/lib/clients/orb/adapters.unit.test.ts +++ b/packages/billing/lib/clients/orb/adapters.unit.test.ts @@ -68,7 +68,7 @@ describe('toOrbEvent', () => { expect(result.timestamp).toBe('2024-01-15T10:00:00.000Z'); }); - it('appends the "_http" suffix to event_name once the cutover has passed', () => { + it('appends "_http" when the event timestamp is at or after the cutover', () => { const originalCutover = envs.BILLING_EVENTS_CUTOVER_AT; try { (envs as any).BILLING_EVENTS_CUTOVER_AT = '2000-01-01T00:00:00Z'; @@ -79,7 +79,7 @@ describe('toOrbEvent', () => { } }); - it('does not append the "_http" suffix before the cutover instant', () => { + it('does not append "_http" when the event timestamp is before the cutover', () => { const originalCutover = envs.BILLING_EVENTS_CUTOVER_AT; try { (envs as any).BILLING_EVENTS_CUTOVER_AT = '9999-01-01T00:00:00Z'; @@ -89,6 +89,27 @@ describe('toOrbEvent', () => { (envs as any).BILLING_EVENTS_CUTOVER_AT = originalCutover; } }); + + it('keys the suffix on each event timestamp independently — a batched pre-cutover event stays unsuffixed even when processed after cutover', () => { + // Same cutover instant, two events on either side of it: verifies the + // suffix is decided per-event, not from wall-clock at processing time. + const originalCutover = envs.BILLING_EVENTS_CUTOVER_AT; + try { + (envs as any).BILLING_EVENTS_CUTOVER_AT = '2024-06-01T00:00:00Z'; + const beforeCutover: BillingEvent = { + type: 'proxy', + properties: { ...baseProperties, timestamp: new Date('2024-05-31T23:59:59.999Z') } as any + }; + const atCutover: BillingEvent = { + type: 'proxy', + properties: { ...baseProperties, timestamp: new Date('2024-06-01T00:00:00.000Z') } as any + }; + expect(toOrbEvent(beforeCutover).event_name).toBe('proxy'); + expect(toOrbEvent(atCutover).event_name).toBe('proxy_http'); + } finally { + (envs as any).BILLING_EVENTS_CUTOVER_AT = originalCutover; + } + }); }); // ─── toOrbPutCustomerPayload ────────────────────────────────────────────────── diff --git a/packages/metering/lib/crons/billingEventsS3Export.ts b/packages/metering/lib/crons/billingEventsS3Export.ts index 5758a886c3..986c4b99fc 100644 --- a/packages/metering/lib/crons/billingEventsS3Export.ts +++ b/packages/metering/lib/crons/billingEventsS3Export.ts @@ -18,14 +18,13 @@ const bucket = envs.BILLING_EVENTS_S3_BUCKET; const roleArn = envs.BILLING_EVENTS_S3_WRITER_ROLE_ARN; const region = envs.BILLING_EVENTS_S3_REGION; -// True once the S3-fed pipeline is authoritative for billing (see -// BILLING_EVENTS_CUTOVER_AT in packages/utils). While this returns false, -// S3 events keep their pre-cutover suffix (e.g. "_shadow") so they don't -// collide with the HTTP-fed canonical events. Once it flips true, S3 -// events land under the canonical (unsuffixed) name customer billable -// metrics filter on. -function cutoverActive(): boolean { - return !!envs.BILLING_EVENTS_CUTOVER_AT && new Date() >= new Date(envs.BILLING_EVENTS_CUTOVER_AT); +// Keyed on the DATA DAY, not the wall clock. Matters at the seam: the +// Aug 1 00:15 UTC firing exports July 31 data, and July 31 is still +// pre-cutover — so those rows must ship as "_s3" shadow, not canonical. +// See BILLING_EVENTS_CUTOVER_AT in packages/utils. +function dayIsPostCutover(day: string): boolean { + if (!envs.BILLING_EVENTS_CUTOVER_AT) return false; + return new Date(`${day}T00:00:00Z`) >= new Date(envs.BILLING_EVENTS_CUTOVER_AT); } const LOCK_KEY = 'lock:cron:billingEventsS3Export'; @@ -185,7 +184,7 @@ export async function exec(): Promise { return; } const day = yesterdayUTC(); - const eventNameSuffix = cutoverActive() ? '' : '_s3'; + const eventNameSuffix = dayIsPostCutover(day) ? '' : '_s3'; let anyFailure = false; try { for (const metric of METRICS) { From a1cc3341dc04db1e63fd0b1f8df968299a1256cb Mon Sep 17 00:00:00 2001 From: Pau Date: Tue, 14 Jul 2026 13:04:06 +0200 Subject: [PATCH 5/5] fix(billing): reconcile cutover suffix with data_transfer opt-out MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit After merging master, NangoHQ/nango#6758's `addEventNameSuffix` helper (which special-cases `data_transfer` to never suffix) and this branch's `dayIsPostCutover(day)` cutover check needed to be fused — otherwise `addEventNameSuffix` referenced a module-scope `eventNameSuffix` this branch had removed. Folds the cutover decision into `addEventNameSuffix` by threading `day` through, so every metric except `data_transfer` picks up `_s3` or `''` based on whether its data day is pre- or post-cutover. `data_transfer` still returns unsuffixed unconditionally, matching #6758's intent that it never had a shadow name to migrate off. Co-Authored-By: Claude Opus 4.7 --- packages/metering/lib/crons/billingEventsS3Export.ts | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/packages/metering/lib/crons/billingEventsS3Export.ts b/packages/metering/lib/crons/billingEventsS3Export.ts index ca5d1ee1e4..7edf71ddc3 100644 --- a/packages/metering/lib/crons/billingEventsS3Export.ts +++ b/packages/metering/lib/crons/billingEventsS3Export.ts @@ -222,13 +222,12 @@ export function billingEventsS3ExportCron(): void { }); } -function addEventNameSuffix(eventName: MetricSpec['canonicalEventName']): string { - if (eventName == 'data_transfer') { +function addEventNameSuffix(eventName: MetricSpec['canonicalEventName'], day: string): string { + if (eventName === 'data_transfer') { // data_transfer is a new event and thus doesn't need the suffix to support live traffic cutoff. return eventName; } - - return `${eventName}${eventNameSuffix}`; + return `${eventName}${dayIsPostCutover(day) ? '' : '_s3'}`; } export async function exec(): Promise { @@ -241,11 +240,10 @@ export async function exec(): Promise { return; } const day = yesterdayUTC(); - const eventNameSuffix = dayIsPostCutover(day) ? '' : '_s3'; let anyFailure = false; try { for (const metric of METRICS) { - const eventName = addEventNameSuffix(metric.canonicalEventName); + const eventName = addEventNameSuffix(metric.canonicalEventName, day); const key = objectKey({ day, eventName }); const start = process.hrtime.bigint(); // Tracks which step is in flight so a catch can tag the failure