diff --git a/.github/actions/deploy-backend-stack/action.yml b/.github/actions/deploy-backend-stack/action.yml index bf8abc67fb8..8d212959b4c 100644 --- a/.github/actions/deploy-backend-stack/action.yml +++ b/.github/actions/deploy-backend-stack/action.yml @@ -5,7 +5,7 @@ description: >- inputs: backend_sync_memory: - description: 'backend-sync memory: 4Gi prod right-size (2026-10-24), 6Gi/8Gi C023 trial values; dev callers keep 8Gi' + description: 'backend-sync memory: 6Gi prod right-size (2026-10-24 costcut), 4Gi/8Gi fallback trial values; dev callers keep 8Gi' required: false default: '8Gi' firebase_probe_signer_service_account: @@ -690,6 +690,8 @@ runs: project_id: ${{ inputs.project_id }} image: ${{ steps.service-images.outputs.backend }} no_traffic: true + # 2026-10-24 costcut mirror: prod backend keeps its live shape in source (it was + # revision-inherited only, so a fresh service would deploy defaults). Dev inherits. flags: >- --revision-suffix=${{ steps.image-tag.outputs.revision_suffix }}-base ${{ inputs.transcription_candidate_tag != '' && format('--tag={0}', inputs.transcription_candidate_tag) || '' }} @@ -697,6 +699,7 @@ runs: --remove-env-vars=MEMORY_ENABLED_USERS,MEMORY_CANONICAL_PROMOTION_CRON_ENABLED,MEMORY_CANONICAL_PROMOTION_CRON_INTERVAL_HOURS,MEMORY_CANONICAL_PROMOTION_FAST_TRACK_ENABLED,BASIC_TIER_MINUTES_LIMIT_PER_MONTH,BASIC_TIER_WORDS_TRANSCRIBED_LIMIT_PER_MONTH,BASIC_TIER_INSIGHTS_GAINED_LIMIT_PER_MONTH,BASIC_TIER_MEMORIES_CREATED_LIMIT_PER_MONTH,CONVERSATION_RELEVANCE_JEV_UID_ALLOWLIST,CONVERSATION_RELEVANCE_JEV_ENABLED,CONVERSATION_RELEVANCE_JEV_SHADOW_PERCENT,CONVERSATION_RELEVANCE_JEV_SHADOW_DAILY_CAP,CONVERSATION_RELEVANCE_KEEP_ALL_PERCENT,CONVERSATION_RELEVANCE_JEV_RESCUE,MEMORY_OWNER_JEV_FLIP_ENABLED,MEMORY_OWNER_JEV_FLIP_PERCENT,MEMORY_OWNER_JEV_SHADOW_PERCENT,MEMORY_OWNER_JEV_SHADOW_DAILY_CAP,CONVERSATION_SMART_MERGE_UID_ALLOWLIST,CONVERSATION_SMART_MERGE_WALLCLOCK_GAP_MODE,DESKTOP_PENDANT_CALL_LINK_MODE,DESKTOP_PENDANT_CALL_LINK_PERCENT,MEETING_NOTES_EPISODE_SELECTION,MEETING_NOTES_EPISODE_JEV_THRESHOLD,MEETING_NOTES_EPISODE_SELECTION_EFFORT,MEETING_NOTES_EPISODE_SELECTION_TIMEOUT_SECONDS,CONVERSATION_CALENDAR_CONTEXT_READ_ENABLED,CONVERSATION_OCR_CONTEXT_ENABLED,MEETING_NOTES_SCREEN_FRAMES_CONTEXT_ENABLED,SCREEN_FRAME_EGRESS_ENABLED,MEMORY_BELIEF_MODEL_ENABLED,FREE_TIER_MEMORY_SUPPRESSION,FREE_TIER_MEMORY_SUPPRESSION_COHORT,KNOWLEDGE_LEDGER_DRAIN_ENABLED,KNOWLEDGE_LEDGER_DRAIN_UID_ALLOWLIST,MEMORY_DAILY_MEMORY_SWEEP_ENABLED,MEMORY_DAILY_MEMORY_SWEEP_KILL_SWITCH,MEMORY_DAILY_MEMORY_SWEEP_MODEL_ENABLED,MEMORY_DAILY_MEMORY_SWEEP_MODEL_NAME,MEMORY_DAILY_MEMORY_SWEEP_MAX_MODEL_CANDIDATES,MEMORY_DAILY_MEMORY_SWEEP_MAX_MODEL_COST_USD,MEMORY_DAILY_MEMORY_SWEEP_COHORT_ENABLED,MEMORY_DAILY_MEMORY_SWEEP_COHORT_FLAG,MEMORY_DAILY_MEMORY_SWEEP_COHORT_NAME,MEMORY_DAILY_MEMORY_SWEEP_COHORT_TIMEOUT_SECONDS,MEMORY_DAILY_MEMORY_SWEEP_TIMEZONE_RECONCILIATION_ENABLED,MEMORY_DAILY_MEMORY_SWEEP_STAGGER_SECONDS,DEEPGRAM_SELF_HOSTED_ENABLED,DEEPGRAM_SELF_HOSTED_URL${{ steps.runtime-env.outputs.backend_remove_env_extra }} ${{ steps.runtime-env.outputs.cloud_run_flags }} ${{ steps.runtime-env.outputs.backend_identity_flags }} + ${{ inputs.runtime_env == 'prod' && '--cpu=2 --memory=8Gi --concurrency=20 --min-instances=3 --max-instances=100' || '' }} env_vars: |- ${{ steps.runtime-env.outputs.backend_env_vars }} OMI_BACKEND_RELEASE_SHA=${{ inputs.admitted_sha }} diff --git a/.github/workflows/gcp_backend.yml b/.github/workflows/gcp_backend.yml index 03ce6721073..52cd9b5e80f 100644 --- a/.github/workflows/gcp_backend.yml +++ b/.github/workflows/gcp_backend.yml @@ -711,8 +711,8 @@ jobs: LADDER_V1_ARCHITECT_YEAR_PRICE_ID: ${{ vars.LADDER_V1_ARCHITECT_YEAR_PRICE_ID }} with: deploy_profile: manual - # 2026-10-24 right-sizing: prod backend-sync runs 4Gi (live 05:54Z); dev keeps 8Gi. - backend_sync_memory: ${{ vars.BACKEND_SYNC_MEMORY || (github.event.inputs.environment == 'prod' && '4Gi' || '8Gi') }} + # 2026-10-24 costcut (live 22:45Z): prod backend-sync runs 6Gi; dev keeps 8Gi. + backend_sync_memory: ${{ vars.BACKEND_SYNC_MEMORY || (github.event.inputs.environment == 'prod' && '6Gi' || '8Gi') }} # Read only for a development dispatch, so a production run never carries it. admitted_sha: ${{ needs.firestore_readiness.outputs.admitted_sha }} environment: ${{ github.event.inputs.environment }} diff --git a/backend/deploy/ops/prod.yaml b/backend/deploy/ops/prod.yaml index 6e04367046f..c349bb730e7 100644 --- a/backend/deploy/ops/prod.yaml +++ b/backend/deploy/ops/prod.yaml @@ -67,8 +67,11 @@ cloud_run: HTTP_SYNC_JOBS_RUN_TIMEOUT: '1500' LANGCHAIN_TRACING_V2: 'false' MEMORY_MODE: 'off' - max_instances: 80 - min_instances: 1 + # 2026-10-24 costcut (live 22:48Z): cap 80 -> 50; warm floor 1 -> 5 so the sync + # pump keeps conversation-finalization headroom (min 5 x concurrency 20, the + # shape deploy-backend-stack sizes its queue against). + max_instances: 50 + min_instances: 5 backend-sync-backfill: env: SYNC_TASKS_QUEUE: sync-backfill-fresh @@ -107,7 +110,9 @@ cloud_run: LANGCHAIN_TRACING_V2: 'false' MEMORY_MODE: 'off' TYPESENSE_HOST_PORT: '443' - min_instances: 1 + # 2026-10-24: 3, matching the deploy flag (prod && 3); the old 1 fought every + # train and idled the projection pump between config-train applies. + min_instances: 3 cloud_tasks: batch-quarantine-recovery: diff --git a/backend/deploy/staging/deploy.sh b/backend/deploy/staging/deploy.sh index 301a724d81f..fc7900e7743 100755 --- a/backend/deploy/staging/deploy.sh +++ b/backend/deploy/staging/deploy.sh @@ -56,13 +56,13 @@ fi PY="${REPO_ROOT}/backend/.venv/bin/python" DEPLOY_LOG="${OMI_STAGING_DEPLOY_LOG:-$HOME/workspace/omi/data/run-2026-10-19-legacy-retire/lane02/deploys.log}" -# Prod-shaped runtime per service (prod live Cloud Run, read 2026-10-19 on ro-prod; sync and -# backfill memory follow the 2026-10-24 prod right-size to 4Gi). Replica counts are staging-sized +# Prod-shaped runtime per service (prod live Cloud Run, read 2026-10-19 on ro-prod; sync memory +# follows the 2026-10-24 costcut to 6Gi, backfill the right-size to 4Gi). Replica counts are staging-sized # by the overlay; these mirror concurrency, CPU, memory and timeout. shape_flags() { case "$1" in backend) echo "--concurrency=20 --cpu=2 --memory=8Gi --timeout=3600 --ingress=all" ;; - backend-sync) echo "--concurrency=20 --cpu=2 --memory=4Gi --timeout=3600 --ingress=all" ;; + backend-sync) echo "--concurrency=20 --cpu=2 --memory=6Gi --timeout=3600 --ingress=all" ;; backend-sync-backfill) echo "--concurrency=6 --cpu=2 --memory=4Gi --timeout=1500 --ingress=all" ;; backend-integration) echo "--concurrency=40 --cpu=2 --memory=2Gi --ingress=all" ;; *) return 1 ;; diff --git a/backend/docs/runbooks/sync-ingest-archive.md b/backend/docs/runbooks/sync-ingest-archive.md index c763a932fea..bfe06de26a9 100644 --- a/backend/docs/runbooks/sync-ingest-archive.md +++ b/backend/docs/runbooks/sync-ingest-archive.md @@ -86,13 +86,14 @@ invoker SA and its endpoint-specific OIDC audience. Configure named task before the cohort ramp. Do not enable a worker without runtime-image, identity, index-readiness and route qualification. -First deploy this SHA at the prod default memory (4Gi since the 2026-10-24 -right-sizing; development keeps 8Gi) through GitHub Actions. For the -separate memory trial, the lead creates a tagged **no-traffic** 6Gi revision -from that already-built same-SHA image, routes at most 5% to it and waits ≥12h. -Only after qualification/full promotion set GitHub environment variable -`BACKEND_SYNC_MEMORY=6Gi` for future deployments; the backend-stack action validates -`4Gi|6Gi|8Gi`. Since the 2026-10-24 right-sizing, the manual workflow's fallback is `4Gi` +First deploy this SHA at the prod default memory (6Gi since the 2026-10-24 +costcut promotion; development keeps 8Gi) through GitHub Actions. For a +separate memory trial, the lead creates a tagged **no-traffic** revision at +the trial size from that already-built same-SHA image, routes at most 5% to it +and waits ≥12h. Only after qualification/full promotion change the prod +fallback here (or set GitHub environment variable `BACKEND_SYNC_MEMORY`) +for future deployments; the backend-stack action validates +`4Gi|6Gi|8Gi`. Since the 2026-10-24 costcut, the manual workflow's fallback is `6Gi` for prod and `8Gi` for development, and the dev-only deploys (auto-dev) keep the action's `8Gi` default. The deployment remains through the existing release admission and GitHub Actions workflow. Rollback: redeploy the admitted image at the prior size. diff --git a/backend/tests/unit/test_cloud_run_rightsizing_pins.py b/backend/tests/unit/test_cloud_run_rightsizing_pins.py new file mode 100644 index 00000000000..00695be2ff8 --- /dev/null +++ b/backend/tests/unit/test_cloud_run_rightsizing_pins.py @@ -0,0 +1,90 @@ +"""Contract pins for the 2026-10-24 prod Cloud Run right-sizing (the lead's costcut). + +The T4 deploy (image 646e54d7, run 40138604325) re-asserted backend-sync at +4Gi / max 80: #23495 had mirrored the 05:54Z right-sizing, not the evening +costcut. These pins hold every deploy source -- the workflow fallback, the +deploy-action flags, the ops-train values and the staging shape -- to the +costcut values, so the next train can no longer silently revert them. + +Live reference (ro-prod, 2026-10-24 22:50Z): backend 8Gi/cpu 2/concurrency 20/ +min 3/max 100; backend-sync 6Gi/cpu 2/min 5/max 50; backend-sync-backfill +4Gi/cpu 2/concurrency 6/min 0 (cap redis_guard-bounded, live 16); +backend-integration 2Gi/cpu 2/concurrency 40/min 3/max 50. +""" + +from __future__ import annotations + +import re +from pathlib import Path + +from scripts.ops import drift_check as dc + +RIGHTSIZED_SERVICES = ('backend', 'backend-sync', 'backend-sync-backfill', 'backend-integration') +REPO_ROOT = Path(__file__).resolve().parents[3] + + +def test_workflow_backend_sync_memory_fallback_is_the_costcut_value(): + # vars.BACKEND_SYNC_MEMORY is unset in both GitHub environments, so the fallback + # literal is what a prod deploy passes. Pinned as text because the drift model + # only sees the expression through _WORKFLOW_ENV_SCOPED: an edit that breaks the + # expression shape would otherwise pass silently with stale model values. + text = (REPO_ROOT / '.github' / 'workflows' / 'gcp_backend.yml').read_text(encoding='utf-8') + assert re.search( + r"backend_sync_memory: \$\{\{ vars\.BACKEND_SYNC_MEMORY \|\| " + r"\(github\.event\.inputs\.environment == 'prod' && '6Gi' \|\| '8Gi'\) \}\}", + text, + ) + + +def test_prod_deploy_flags_match_the_live_right_sizing(): + model = dc.source_model('prod') + + backend = model['backend'].scale + assert (backend['memory'], backend['cpu'], backend['concurrency']) == ('8Gi', '2', 20) + assert (backend['revision_min'], backend['revision_max']) == (3, 100) + + sync = model['backend-sync'].scale + assert (sync['memory'], sync['cpu']) == ('6Gi', '2') + # max-instances is live-resolved by design (Resolve live sync maxScale); the cap + # itself is pinned in the ops file. The deploy never touches sync min. + assert sync['revision_min'] is None and sync['revision_max'] is None + + backfill = model['backend-sync-backfill'].scale + assert (backfill['memory'], backfill['cpu'], backfill['concurrency'], backfill['revision_min']) == ( + '4Gi', + '2', + 6, + 0, + ) + + integration = model['backend-integration'].scale + assert (integration['memory'], integration['cpu'], integration['concurrency']) == ('2Gi', '2', 40) + assert (integration['revision_min'], integration['revision_max']) == (3, 50) + + +def test_ops_train_values_match_the_live_right_sizing(): + model = dc.source_model('prod') + + sync = model['backend-sync'].ops_scale + assert (sync['service_min'], sync['service_max']) == (5, 50) + + # redis_guard owns the backfill cap between its bounds (live 16). + backfill = model['backend-sync-backfill'].ops_scale + assert backfill['service_min'] is None + assert backfill['service_max']['owner'] == 'redis_guard' + assert backfill['service_max']['floor'] <= 16 <= backfill['service_max']['ceiling'] + + # Equal to the deploy flag (prod && 3): the ops file must never fight the deploy. + integration = model['backend-integration'].ops_scale + assert (integration['service_min'], integration['service_max']) == (3, None) + + backend = model['backend'].ops_scale + assert (backend['service_min'], backend['service_max']) == (None, None) + + +def test_staging_shape_keeps_parity_with_prod(): + prod = dc.source_model('prod') + staging = dc.source_model('staging') + for name in RIGHTSIZED_SERVICES: + findings = dc._scale_vs_prod(name, staging[name].scale, prod[name].scale) + assert not findings, f'{name}: {[f.line() for f in findings]}' diff --git a/backend/tests/unit/test_drift_check.py b/backend/tests/unit/test_drift_check.py index 65ad80557d1..fc05861c672 100644 --- a/backend/tests/unit/test_drift_check.py +++ b/backend/tests/unit/test_drift_check.py @@ -458,8 +458,8 @@ def test_prod_source_model_reads_deploy_flags_and_the_ops_file(): assert integration.scale['revision_min'] == 3 and integration.scale['concurrency'] == 40 assert integration.ops_env['SUMMARY_DEBT_REPAIR_ENABLED'] == 'true' # The workflow's env-scoped backend_sync_memory fallback reaches the source model, so a - # future regression of the prod 4Gi right-size is drift, not silence. - assert model['backend-sync'].scale['memory'] == '4Gi' + # future regression of the prod 6Gi costcut is drift, not silence. + assert model['backend-sync'].scale['memory'] == '6Gi' assert model['backend-sync-backfill'].scale['memory'] == '4Gi' assert model['backend-sync-backfill'].ops_scale['service_max'] == { 'owner': 'redis_guard', diff --git a/backend/tests/unit/test_render_backend_runtime_env.py b/backend/tests/unit/test_render_backend_runtime_env.py index 0459b55cae6..7c362509156 100644 --- a/backend/tests/unit/test_render_backend_runtime_env.py +++ b/backend/tests/unit/test_render_backend_runtime_env.py @@ -592,13 +592,16 @@ def test_backend_integration_deploy_pins_mcp_serving_capacity(): # here only; do not copy onto backend. # backend-sync DOES carry --cpu=2 (#21668): its 6-8Gi memory requires >=2 # CPU on Cloud Run, and 1-CPU deploys of 8Gi are rejected outright. + # 2026-10-24 costcut mirror: backend now pins its OWN live shape (prod-scoped + # 8Gi/2 CPU/20/3/100) instead of inheriting it revision-to-revision; --cpu=2 + # is legitimate there for the same #21668 pairing, but every + # integration-specific flag below still must not leak onto it. integration_flags = _deploy_backend_stack_step_flags('deploy-backend-integration') backend_flags = _deploy_backend_stack_step_flags('deploy-backend') sync_flags = _deploy_backend_stack_step_flags('deploy-backend-sync') # 2026-10-24 right-sizing: prod integration stays warm; the dev project idles at 0. integration_min = "--min-instances=${{ inputs.runtime_env == 'prod' && '3' || '0' }}" for flag in ( - '--cpu=2', '--memory=2Gi', '--concurrency=40', integration_min, @@ -609,11 +612,16 @@ def test_backend_integration_deploy_pins_mcp_serving_capacity(): assert flag not in backend_flags, flag # backend-sync owns its own scale-out cap (see test_sync_two_lane.py); # its value may legitimately equal integration's, so only the other - # integration-specific flags must stay off it. --cpu=2 is likewise - # legitimate on sync (#21668: 8Gi needs >=2 CPU). - if not (flag.startswith('--max-instances=') or flag == '--cpu=2'): + # integration-specific flags must stay off it. + if not flag.startswith('--max-instances='): assert flag not in sync_flags, flag + assert '--cpu=2' in integration_flags assert '--cpu=2' in sync_flags + # Backend's own prod shape (dev branch empty: dev keeps inheriting). + assert ( + "${{ inputs.runtime_env == 'prod' && '--cpu=2 --memory=8Gi --concurrency=20" + " --min-instances=3 --max-instances=100' || '' }}" + ) in backend_flags VERTEX_PT_CONTRACT = 'Company-paid Gemini text and vision generation may use only the active model'