Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 4 additions & 1 deletion .github/actions/deploy-backend-stack/action.yml
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ description: >-

inputs:
backend_sync_memory:
description: 'backend-sync memory: 4Gi prod right-size (2026-10-24), 6Gi/8Gi C023 trial values; dev callers keep 8Gi'
description: 'backend-sync memory: 6Gi prod right-size (2026-10-24 costcut), 4Gi/8Gi fallback trial values; dev callers keep 8Gi'
required: false
default: '8Gi'
firebase_probe_signer_service_account:
Expand Down Expand Up @@ -690,13 +690,16 @@ runs:
project_id: ${{ inputs.project_id }}
image: ${{ steps.service-images.outputs.backend }}
no_traffic: true
# 2026-10-24 costcut mirror: prod backend keeps its live shape in source (it was
# revision-inherited only, so a fresh service would deploy defaults). Dev inherits.
flags: >-
--revision-suffix=${{ steps.image-tag.outputs.revision_suffix }}-base
${{ inputs.transcription_candidate_tag != '' && format('--tag={0}', inputs.transcription_candidate_tag) || '' }}
${{ inputs.deploy_profile == 'auto-dev' && format('--tag={0}', inputs.candidate_tag) || '' }}
--remove-env-vars=MEMORY_ENABLED_USERS,MEMORY_CANONICAL_PROMOTION_CRON_ENABLED,MEMORY_CANONICAL_PROMOTION_CRON_INTERVAL_HOURS,MEMORY_CANONICAL_PROMOTION_FAST_TRACK_ENABLED,BASIC_TIER_MINUTES_LIMIT_PER_MONTH,BASIC_TIER_WORDS_TRANSCRIBED_LIMIT_PER_MONTH,BASIC_TIER_INSIGHTS_GAINED_LIMIT_PER_MONTH,BASIC_TIER_MEMORIES_CREATED_LIMIT_PER_MONTH,CONVERSATION_RELEVANCE_JEV_UID_ALLOWLIST,CONVERSATION_RELEVANCE_JEV_ENABLED,CONVERSATION_RELEVANCE_JEV_SHADOW_PERCENT,CONVERSATION_RELEVANCE_JEV_SHADOW_DAILY_CAP,CONVERSATION_RELEVANCE_KEEP_ALL_PERCENT,CONVERSATION_RELEVANCE_JEV_RESCUE,MEMORY_OWNER_JEV_FLIP_ENABLED,MEMORY_OWNER_JEV_FLIP_PERCENT,MEMORY_OWNER_JEV_SHADOW_PERCENT,MEMORY_OWNER_JEV_SHADOW_DAILY_CAP,CONVERSATION_SMART_MERGE_UID_ALLOWLIST,CONVERSATION_SMART_MERGE_WALLCLOCK_GAP_MODE,DESKTOP_PENDANT_CALL_LINK_MODE,DESKTOP_PENDANT_CALL_LINK_PERCENT,MEETING_NOTES_EPISODE_SELECTION,MEETING_NOTES_EPISODE_JEV_THRESHOLD,MEETING_NOTES_EPISODE_SELECTION_EFFORT,MEETING_NOTES_EPISODE_SELECTION_TIMEOUT_SECONDS,CONVERSATION_CALENDAR_CONTEXT_READ_ENABLED,CONVERSATION_OCR_CONTEXT_ENABLED,MEETING_NOTES_SCREEN_FRAMES_CONTEXT_ENABLED,SCREEN_FRAME_EGRESS_ENABLED,MEMORY_BELIEF_MODEL_ENABLED,FREE_TIER_MEMORY_SUPPRESSION,FREE_TIER_MEMORY_SUPPRESSION_COHORT,KNOWLEDGE_LEDGER_DRAIN_ENABLED,KNOWLEDGE_LEDGER_DRAIN_UID_ALLOWLIST,MEMORY_DAILY_MEMORY_SWEEP_ENABLED,MEMORY_DAILY_MEMORY_SWEEP_KILL_SWITCH,MEMORY_DAILY_MEMORY_SWEEP_MODEL_ENABLED,MEMORY_DAILY_MEMORY_SWEEP_MODEL_NAME,MEMORY_DAILY_MEMORY_SWEEP_MAX_MODEL_CANDIDATES,MEMORY_DAILY_MEMORY_SWEEP_MAX_MODEL_COST_USD,MEMORY_DAILY_MEMORY_SWEEP_COHORT_ENABLED,MEMORY_DAILY_MEMORY_SWEEP_COHORT_FLAG,MEMORY_DAILY_MEMORY_SWEEP_COHORT_NAME,MEMORY_DAILY_MEMORY_SWEEP_COHORT_TIMEOUT_SECONDS,MEMORY_DAILY_MEMORY_SWEEP_TIMEZONE_RECONCILIATION_ENABLED,MEMORY_DAILY_MEMORY_SWEEP_STAGGER_SECONDS,DEEPGRAM_SELF_HOSTED_ENABLED,DEEPGRAM_SELF_HOSTED_URL${{ steps.runtime-env.outputs.backend_remove_env_extra }}
${{ steps.runtime-env.outputs.cloud_run_flags }}
${{ steps.runtime-env.outputs.backend_identity_flags }}
${{ inputs.runtime_env == 'prod' && '--cpu=2 --memory=8Gi --concurrency=20 --min-instances=3 --max-instances=100' || '' }}
env_vars: |-
${{ steps.runtime-env.outputs.backend_env_vars }}
OMI_BACKEND_RELEASE_SHA=${{ inputs.admitted_sha }}
Expand Down
4 changes: 2 additions & 2 deletions .github/workflows/gcp_backend.yml
Original file line number Diff line number Diff line change
Expand Up @@ -711,8 +711,8 @@ jobs:
LADDER_V1_ARCHITECT_YEAR_PRICE_ID: ${{ vars.LADDER_V1_ARCHITECT_YEAR_PRICE_ID }}
with:
deploy_profile: manual
# 2026-10-24 right-sizing: prod backend-sync runs 4Gi (live 05:54Z); dev keeps 8Gi.
backend_sync_memory: ${{ vars.BACKEND_SYNC_MEMORY || (github.event.inputs.environment == 'prod' && '4Gi' || '8Gi') }}
# 2026-10-24 costcut (live 22:45Z): prod backend-sync runs 6Gi; dev keeps 8Gi.
backend_sync_memory: ${{ vars.BACKEND_SYNC_MEMORY || (github.event.inputs.environment == 'prod' && '6Gi' || '8Gi') }}
# Read only for a development dispatch, so a production run never carries it.
admitted_sha: ${{ needs.firestore_readiness.outputs.admitted_sha }}
environment: ${{ github.event.inputs.environment }}
Expand Down
11 changes: 8 additions & 3 deletions backend/deploy/ops/prod.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -67,8 +67,11 @@ cloud_run:
HTTP_SYNC_JOBS_RUN_TIMEOUT: '1500'
LANGCHAIN_TRACING_V2: 'false'
MEMORY_MODE: 'off'
max_instances: 80
min_instances: 1
# 2026-10-24 costcut (live 22:48Z): cap 80 -> 50; warm floor 1 -> 5 so the sync
# pump keeps conversation-finalization headroom (min 5 x concurrency 20, the
# shape deploy-backend-stack sizes its queue against).
max_instances: 50
min_instances: 5
backend-sync-backfill:
env:
SYNC_TASKS_QUEUE: sync-backfill-fresh
Expand Down Expand Up @@ -107,7 +110,9 @@ cloud_run:
LANGCHAIN_TRACING_V2: 'false'
MEMORY_MODE: 'off'
TYPESENSE_HOST_PORT: '443'
min_instances: 1
# 2026-10-24: 3, matching the deploy flag (prod && 3); the old 1 fought every
# train and idled the projection pump between config-train applies.
min_instances: 3

cloud_tasks:
batch-quarantine-recovery:
Expand Down
6 changes: 3 additions & 3 deletions backend/deploy/staging/deploy.sh
Original file line number Diff line number Diff line change
Expand Up @@ -56,13 +56,13 @@ fi
PY="${REPO_ROOT}/backend/.venv/bin/python"
DEPLOY_LOG="${OMI_STAGING_DEPLOY_LOG:-$HOME/workspace/omi/data/run-2026-10-19-legacy-retire/lane02/deploys.log}"

# Prod-shaped runtime per service (prod live Cloud Run, read 2026-10-19 on ro-prod; sync and
# backfill memory follow the 2026-10-24 prod right-size to 4Gi). Replica counts are staging-sized
# Prod-shaped runtime per service (prod live Cloud Run, read 2026-10-19 on ro-prod; sync memory
# follows the 2026-10-24 costcut to 6Gi, backfill the right-size to 4Gi). Replica counts are staging-sized
# by the overlay; these mirror concurrency, CPU, memory and timeout.
shape_flags() {
case "$1" in
backend) echo "--concurrency=20 --cpu=2 --memory=8Gi --timeout=3600 --ingress=all" ;;
backend-sync) echo "--concurrency=20 --cpu=2 --memory=4Gi --timeout=3600 --ingress=all" ;;
backend-sync) echo "--concurrency=20 --cpu=2 --memory=6Gi --timeout=3600 --ingress=all" ;;
backend-sync-backfill) echo "--concurrency=6 --cpu=2 --memory=4Gi --timeout=1500 --ingress=all" ;;
backend-integration) echo "--concurrency=40 --cpu=2 --memory=2Gi --ingress=all" ;;
*) return 1 ;;
Expand Down
15 changes: 8 additions & 7 deletions backend/docs/runbooks/sync-ingest-archive.md
Original file line number Diff line number Diff line change
Expand Up @@ -86,13 +86,14 @@ invoker SA and its endpoint-specific OIDC audience. Configure
named task before the cohort ramp. Do not enable a worker without runtime-image,
identity, index-readiness and route qualification.

First deploy this SHA at the prod default memory (4Gi since the 2026-10-24
right-sizing; development keeps 8Gi) through GitHub Actions. For the
separate memory trial, the lead creates a tagged **no-traffic** 6Gi revision
from that already-built same-SHA image, routes at most 5% to it and waits ≥12h.
Only after qualification/full promotion set GitHub environment variable
`BACKEND_SYNC_MEMORY=6Gi` for future deployments; the backend-stack action validates
`4Gi|6Gi|8Gi`. Since the 2026-10-24 right-sizing, the manual workflow's fallback is `4Gi`
First deploy this SHA at the prod default memory (6Gi since the 2026-10-24
costcut promotion; development keeps 8Gi) through GitHub Actions. For a
separate memory trial, the lead creates a tagged **no-traffic** revision at
the trial size from that already-built same-SHA image, routes at most 5% to it
and waits ≥12h. Only after qualification/full promotion change the prod
fallback here (or set GitHub environment variable `BACKEND_SYNC_MEMORY`)
for future deployments; the backend-stack action validates
`4Gi|6Gi|8Gi`. Since the 2026-10-24 costcut, the manual workflow's fallback is `6Gi`
for prod and `8Gi` for development, and the dev-only deploys (auto-dev) keep the action's
`8Gi` default. The deployment remains through the existing release admission and
GitHub Actions workflow. Rollback: redeploy the admitted image at the prior size.
Expand Down
90 changes: 90 additions & 0 deletions backend/tests/unit/test_cloud_run_rightsizing_pins.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,90 @@
"""Contract pins for the 2026-10-24 prod Cloud Run right-sizing (the lead's costcut).

The T4 deploy (image 646e54d7, run 40138604325) re-asserted backend-sync at
4Gi / max 80: #23495 had mirrored the 05:54Z right-sizing, not the evening
costcut. These pins hold every deploy source -- the workflow fallback, the
deploy-action flags, the ops-train values and the staging shape -- to the
costcut values, so the next train can no longer silently revert them.

Live reference (ro-prod, 2026-10-24 22:50Z): backend 8Gi/cpu 2/concurrency 20/
min 3/max 100; backend-sync 6Gi/cpu 2/min 5/max 50; backend-sync-backfill
4Gi/cpu 2/concurrency 6/min 0 (cap redis_guard-bounded, live 16);
backend-integration 2Gi/cpu 2/concurrency 40/min 3/max 50.
"""

from __future__ import annotations

import re
from pathlib import Path

from scripts.ops import drift_check as dc

RIGHTSIZED_SERVICES = ('backend', 'backend-sync', 'backend-sync-backfill', 'backend-integration')
REPO_ROOT = Path(__file__).resolve().parents[3]


def test_workflow_backend_sync_memory_fallback_is_the_costcut_value():
# vars.BACKEND_SYNC_MEMORY is unset in both GitHub environments, so the fallback
# literal is what a prod deploy passes. Pinned as text because the drift model
# only sees the expression through _WORKFLOW_ENV_SCOPED: an edit that breaks the
# expression shape would otherwise pass silently with stale model values.
text = (REPO_ROOT / '.github' / 'workflows' / 'gcp_backend.yml').read_text(encoding='utf-8')
assert re.search(
r"backend_sync_memory: \$\{\{ vars\.BACKEND_SYNC_MEMORY \|\| "
r"\(github\.event\.inputs\.environment == 'prod' && '6Gi' \|\| '8Gi'\) \}\}",
text,
)


def test_prod_deploy_flags_match_the_live_right_sizing():
model = dc.source_model('prod')

backend = model['backend'].scale
assert (backend['memory'], backend['cpu'], backend['concurrency']) == ('8Gi', '2', 20)
assert (backend['revision_min'], backend['revision_max']) == (3, 100)

sync = model['backend-sync'].scale
assert (sync['memory'], sync['cpu']) == ('6Gi', '2')
# max-instances is live-resolved by design (Resolve live sync maxScale); the cap
# itself is pinned in the ops file. The deploy never touches sync min.
assert sync['revision_min'] is None and sync['revision_max'] is None

backfill = model['backend-sync-backfill'].scale
assert (backfill['memory'], backfill['cpu'], backfill['concurrency'], backfill['revision_min']) == (
'4Gi',
'2',
6,
0,
)

integration = model['backend-integration'].scale
assert (integration['memory'], integration['cpu'], integration['concurrency']) == ('2Gi', '2', 40)
assert (integration['revision_min'], integration['revision_max']) == (3, 50)


def test_ops_train_values_match_the_live_right_sizing():
model = dc.source_model('prod')

sync = model['backend-sync'].ops_scale
assert (sync['service_min'], sync['service_max']) == (5, 50)

# redis_guard owns the backfill cap between its bounds (live 16).
backfill = model['backend-sync-backfill'].ops_scale
assert backfill['service_min'] is None
assert backfill['service_max']['owner'] == 'redis_guard'
assert backfill['service_max']['floor'] <= 16 <= backfill['service_max']['ceiling']

# Equal to the deploy flag (prod && 3): the ops file must never fight the deploy.
integration = model['backend-integration'].ops_scale
assert (integration['service_min'], integration['service_max']) == (3, None)

backend = model['backend'].ops_scale
assert (backend['service_min'], backend['service_max']) == (None, None)


def test_staging_shape_keeps_parity_with_prod():
prod = dc.source_model('prod')
staging = dc.source_model('staging')
for name in RIGHTSIZED_SERVICES:
findings = dc._scale_vs_prod(name, staging[name].scale, prod[name].scale)
assert not findings, f'{name}: {[f.line() for f in findings]}'
4 changes: 2 additions & 2 deletions backend/tests/unit/test_drift_check.py
Original file line number Diff line number Diff line change
Expand Up @@ -458,8 +458,8 @@ def test_prod_source_model_reads_deploy_flags_and_the_ops_file():
assert integration.scale['revision_min'] == 3 and integration.scale['concurrency'] == 40
assert integration.ops_env['SUMMARY_DEBT_REPAIR_ENABLED'] == 'true'
# The workflow's env-scoped backend_sync_memory fallback reaches the source model, so a
# future regression of the prod 4Gi right-size is drift, not silence.
assert model['backend-sync'].scale['memory'] == '4Gi'
# future regression of the prod 6Gi costcut is drift, not silence.
assert model['backend-sync'].scale['memory'] == '6Gi'
assert model['backend-sync-backfill'].scale['memory'] == '4Gi'
assert model['backend-sync-backfill'].ops_scale['service_max'] == {
'owner': 'redis_guard',
Expand Down
16 changes: 12 additions & 4 deletions backend/tests/unit/test_render_backend_runtime_env.py
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
"""Renderer for backend Cloud Run runtime env."""

Check warning on line 1 in backend/tests/unit/test_render_backend_runtime_env.py

View workflow job for this annotation

GitHub Actions / Hygiene

Large changed file

backend/tests/unit/test_render_backend_runtime_env.py is 1406 lines; consider splitting files over 800 lines.

import json
import os
Expand Down Expand Up @@ -592,13 +592,16 @@
# here only; do not copy onto backend.
# backend-sync DOES carry --cpu=2 (#21668): its 6-8Gi memory requires >=2
# CPU on Cloud Run, and 1-CPU deploys of 8Gi are rejected outright.
# 2026-10-24 costcut mirror: backend now pins its OWN live shape (prod-scoped
# 8Gi/2 CPU/20/3/100) instead of inheriting it revision-to-revision; --cpu=2
# is legitimate there for the same #21668 pairing, but every
# integration-specific flag below still must not leak onto it.
integration_flags = _deploy_backend_stack_step_flags('deploy-backend-integration')
backend_flags = _deploy_backend_stack_step_flags('deploy-backend')
sync_flags = _deploy_backend_stack_step_flags('deploy-backend-sync')
# 2026-10-24 right-sizing: prod integration stays warm; the dev project idles at 0.
integration_min = "--min-instances=${{ inputs.runtime_env == 'prod' && '3' || '0' }}"
for flag in (
'--cpu=2',
'--memory=2Gi',
'--concurrency=40',
integration_min,
Expand All @@ -609,11 +612,16 @@
assert flag not in backend_flags, flag
# backend-sync owns its own scale-out cap (see test_sync_two_lane.py);
# its value may legitimately equal integration's, so only the other
# integration-specific flags must stay off it. --cpu=2 is likewise
# legitimate on sync (#21668: 8Gi needs >=2 CPU).
if not (flag.startswith('--max-instances=') or flag == '--cpu=2'):
# integration-specific flags must stay off it.
if not flag.startswith('--max-instances='):
assert flag not in sync_flags, flag
assert '--cpu=2' in integration_flags
assert '--cpu=2' in sync_flags
# Backend's own prod shape (dev branch empty: dev keeps inheriting).
assert (
"${{ inputs.runtime_env == 'prod' && '--cpu=2 --memory=8Gi --concurrency=20"
" --min-instances=3 --max-instances=100' || '' }}"
) in backend_flags


VERTEX_PT_CONTRACT = 'Company-paid Gemini text and vision generation may use only the active model'
Expand Down
Loading