Skip to content

feat(relocation) Use a shared bucket for relocations #116108

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
markstory wants to merge 18 commits into
base: master
Choose a base branch
from
Open

feat(relocation) Use a shared bucket for relocations #116108

Show file tree
Hide file tree
Changes from 11 commits
Commits
Show all changes
18 commits
Select commit Hold shift + click to select a range
c24ebe1
Update relocation creation to write to the shared bucket.
markstory May 22, 2026
0362992
Changed my mind
markstory May 22, 2026
dd2dc81
Update retry endpoint to use direct bucket storage
markstory May 22, 2026
be24349
Make encrypted_bytes optional
markstory May 22, 2026
17e6149
Rework file paths again
markstory May 22, 2026
177a6e1
Remove logic to shuffle files around
markstory May 22, 2026
021d723
Remove storage bucket operations from transfer tests
markstory May 22, 2026
f3cae56
Fix more failing tests
markstory May 22, 2026
ec388cf
Get EndToEndTest passing
markstory May 22, 2026
2975977
Get UploadingStartTest passing
markstory May 22, 2026
0ececbc
Fix up tests
markstory May 22, 2026
42b31cf
Fix mistake
markstory May 25, 2026
6764a8d
Fix hardcoded value
markstory May 25, 2026
f2ad33b
Fix retry paths
markstory May 25, 2026
93dc700
Fix mypy
markstory May 25, 2026
ba3a35b
Address feedback from robots
markstory May 25, 2026
8234909
Fix more failing tests
markstory May 25, 2026
144e5e3
Merge branch 'master' into feat-relocation-no-blob-move
markstory May 25, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
22 changes: 17 additions & 5 deletions src/sentry/relocation/api/endpoints/index.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
from datetime import timedelta
from functools import reduce
from string import Template
from uuid import uuid4

from django.db import router
from django.db.models import Q
Expand All @@ -22,12 +23,13 @@
from sentry.api.serializers import serialize
from sentry.auth.elevated_mode import has_elevated_mode
from sentry.models.files.file import File
from sentry.models.files.utils import get_relocation_storage
from sentry.options import get
from sentry.relocation.api.endpoints import ERR_FEATURE_DISABLED
from sentry.relocation.api.serializers.relocation import RelocationSerializer
from sentry.relocation.models.relocation import Relocation, RelocationFile
from sentry.relocation.tasks.process import uploading_start
from sentry.relocation.utils import RELOCATION_BLOB_SIZE, RELOCATION_FILE_TYPE
from sentry.relocation.utils import RELOCATION_FILE_TYPE, relocation_raw_data_path
from sentry.search.utils import tokenize_query
from sentry.signals import relocation_link_promo_code
from sentry.users.models.user import MAX_USERNAME_LENGTH, User
Expand All @@ -53,7 +55,9 @@
RELOCATION_FILE_SIZE_MEDIUM = 100 * 1024**2


def get_relocation_size_category(size) -> str:
def get_relocation_size_category(size: int | None) -> str:
if size is None:
return "medium"
if size < RELOCATION_FILE_SIZE_SMALL:
return "small"
elif size < RELOCATION_FILE_SIZE_MEDIUM:
Expand All @@ -62,6 +66,7 @@ def get_relocation_size_category(size) -> str:


def should_throttle_relocation(relocation_bucket_size: str) -> bool:
# TODO(cells) Instead of doing this by size we could use the number of inflight relocations.
recent_relocation_files = RelocationFile.objects.filter(
date_added__gte=(timezone.now() - timedelta(days=1))
)
Expand Down Expand Up @@ -273,14 +278,20 @@ def post(self, request: Request) -> Response:
if err is not None:
return err

file = File.objects.create(name="raw-relocation-data.tar", type=RELOCATION_FILE_TYPE)
file.putfile(fileobj, blob_size=RELOCATION_BLOB_SIZE, logger=logger)
relocation_uuid = uuid4()
path = relocation_raw_data_path(relocation_uuid)
relocation_storage = get_relocation_storage()
relocation_storage.save(path, fileobj)

# TODO(cells) Make RelocationFile.file nullable
file = File.objects.create(name="stub", type=RELOCATION_FILE_TYPE, size=len(fileobj))

with atomic_transaction(
using=(router.db_for_write(Relocation), router.db_for_write(RelocationFile))
):
provenance = Relocation.Provenance.SELF_HOSTED
relocation: Relocation = Relocation.objects.create(
uuid=relocation_uuid,
creator_id=request.user.id,
owner_id=owner.id,
want_org_slugs=org_slugs,
Expand All @@ -290,8 +301,9 @@ def post(self, request: Request) -> Response:
)
RelocationFile.objects.create(
relocation=relocation,
file=file,
kind=RelocationFile.Kind.RAW_USER_DATA.value,
bucket_path=path,
file=file,
)
relocation_link_promo_code.send_robust(
relocation_uuid=relocation.uuid, promo_code=promo_code, sender=self.__class__
Expand Down
12 changes: 5 additions & 7 deletions src/sentry/relocation/api/endpoints/retry.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,6 @@
from sentry.api.exceptions import ResourceDoesNotExist
from sentry.api.permissions import SentryIsAuthenticated
from sentry.api.serializers import serialize
from sentry.models.files.file import File
from sentry.relocation.api.endpoints.index import (
get_autopause_value,
validate_new_relocation_request,
Expand Down Expand Up @@ -75,6 +74,7 @@ def post(self, request: Request, relocation_uuid: str) -> Response:
status=status.HTTP_400_BAD_REQUEST,
)

# TODO(cells) Update this when file becomes nullable/removed.
relocation_file = (
RelocationFile.objects.filter(relocation=relocation).select_related("file").first()
)
Expand All @@ -85,10 +85,7 @@ def post(self, request: Request, relocation_uuid: str) -> Response:
)

# We can re-use the same `File` instance in the database, avoiding duplicating data.
try:
file = File.objects.get(id=relocation_file.file_id)
fileobj = file.getfile()
except (File.DoesNotExist, FileNotFoundError):
if not relocation_file.file:
return Response(
{"detail": ERR_FILE_NO_LONGER_EXISTS},
status=status.HTTP_400_BAD_REQUEST,
Expand All @@ -102,7 +99,7 @@ def post(self, request: Request, relocation_uuid: str) -> Response:
)

err = validate_new_relocation_request(
request, owner.username, relocation.want_org_slugs, fileobj.size
request, owner.username, relocation.want_org_slugs, relocation_file.file.size
) or validate_relocation_uniqueness(owner)
if err is not None:
return err
Expand All @@ -128,8 +125,9 @@ def post(self, request: Request, relocation_uuid: str) -> Response:
)
RelocationFile.objects.create(
relocation=new_relocation,
file=file,
Comment thread
cursor[bot] marked this conversation as resolved.
kind=RelocationFile.Kind.RAW_USER_DATA.value,
bucket_path=relocation_file.bucket_path,
file=relocation_file.file,
Copy link
Copy Markdown
Contributor

@cursor cursor Bot May 25, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Retry copies null bucket path

Medium Severity

Failed-relocation retry reuses relocation_file.bucket_path on the new RelocationFile without checking it is set. Failures created before shared-bucket uploads can have a null path while data still lives on the legacy File record, so the retried run cannot open raw data from storage.

Additional Locations (1)
Fix in Cursor Fix in Web

Reviewed by Cursor Bugbot for commit 93dc700. Configure here.

)

uploading_start.delay(str(new_relocation.uuid), None, None)
Expand Down
42 changes: 15 additions & 27 deletions src/sentry/relocation/services/relocation_export/impl.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,15 +6,14 @@
import base64
import logging
from datetime import UTC, datetime
from io import BytesIO
from uuid import UUID

from django.db import router
from django.db.utils import IntegrityError
from django.utils import timezone
from sentry_sdk import capture_exception

from sentry.models.files.file import File
from sentry.models.files.utils import get_relocation_storage
from sentry.relocation.models.relocation import Relocation, RelocationFile
from sentry.relocation.models.relocationtransfer import (
RETRY_BACKOFF,
Expand All @@ -27,7 +26,10 @@
)
from sentry.relocation.tasks.process import fulfill_cross_region_export_request, uploading_complete
from sentry.relocation.tasks.transfer import process_relocation_transfer_control
from sentry.relocation.utils import RELOCATION_BLOB_SIZE, RELOCATION_FILE_TYPE
from sentry.relocation.utils import (
RELOCATION_FILE_TYPE,
relocation_raw_data_path,
)
from sentry.utils.db import atomic_transaction

logger = logging.getLogger("sentry.relocation")
Expand Down Expand Up @@ -77,8 +79,7 @@ def reply_with_export(
requesting_region_name: str,
replying_region_name: str,
org_slug: str,
encrypted_bytes: list[int],
# TODO(azaslavsky): finish transfer from `encrypted_contents` -> `encrypted_bytes`.
encrypted_bytes: list[int] | None = None,
encrypted_contents: bytes | None = None,
) -> None:
with atomic_transaction(
Expand All @@ -93,23 +94,22 @@ def reply_with_export(
"requesting_region_name": requesting_region_name,
"replying_region_name": replying_region_name,
"org_slug": org_slug,
# TODO(azaslavsky): finish transfer from `encrypted_contents` -> `encrypted_bytes`.
"encrypted_bytes_size": len(encrypted_bytes or []),
}
logger.info("SaaS -> SaaS reply received in triggering region", extra=logger_data)
uuid = UUID(relocation_uuid)

try:
relocation: Relocation = Relocation.objects.get(uuid=relocation_uuid)
relocation: Relocation = Relocation.objects.get(uuid=uuid)
except Relocation.DoesNotExist as e:
logger.exception("Could not locate Relocation model by UUID: %s", relocation_uuid)
capture_exception(e)
return

# TODO(azaslavsky): finish transfer from `encrypted_contents` -> `encrypted_bytes`.
fp = BytesIO(bytes(encrypted_bytes or []))
file = File.objects.create(name="raw-relocation-data.tar", type=RELOCATION_FILE_TYPE)
file.putfile(fp, blob_size=RELOCATION_BLOB_SIZE, logger=logger)
logger.info("SaaS -> SaaS relocation underlying File created", extra=logger_data)
# Now that we have confirmation that the export file
# was stored in the shared bucket, create a RelocationFile record
# so that the import process can begin.
# TODO(cells) Remove this once RelocationFile.file is optional.
file = File.objects.create(name="stub", type=RELOCATION_FILE_TYPE, size=10000)
Comment thread
sentry[bot] marked this conversation as resolved.
Outdated

# This write ensures that the entire chain triggered by `uploading_start` remains
# idempotent, since only one (relocation_uuid, relocation_file_kind) pairing can exist
Comment on lines +114 to 120
Copy link
Copy Markdown
Contributor

@sentry sentry Bot May 25, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Bug: Catching an IntegrityError leaves the transaction in an aborted state, leading to a TransactionManagementError and an infinite retry loop that repeatedly schedules a Celery task.
Severity: CRITICAL

Suggested Fix

When an IntegrityError is caught, the transaction should be explicitly rolled back to a savepoint created before the failing create() call. This will allow the transaction to be committed successfully, preventing the TransactionManagementError and the subsequent infinite retry loop. The pass statement should be replaced with logic to handle the transaction state correctly.

Prompt for AI Agent 
Review the code at the location below. A potential bug has been identified by an AI
agent. Verify if this is a real issue. If it is, propose a fix; if not, explain why it's
not valid.

Location: src/sentry/relocation/services/relocation_export/impl.py#L114-L120

Potential issue: When handling an `IntegrityError` from
`RelocationFile.objects.create()` for idempotency, the exception is caught, but the
underlying database transaction is left in an aborted state. The code then proceeds to
schedule an `uploading_complete` task via Celery. When the `atomic_transaction` block
exits, Django raises a `TransactionManagementError` because the transaction was aborted.
The calling function catches this error but fails to delete the `transfer` record,
causing an infinite retry loop where the `uploading_complete` task is repeatedly
scheduled on each attempt.

Expand All @@ -120,11 +120,11 @@ def reply_with_export(
relocation=relocation,
file=file,
kind=RelocationFile.Kind.RAW_USER_DATA.value,
bucket_path=relocation_raw_data_path(uuid),
)
except IntegrityError:
# We already have the file, we can proceed.
pass

Comment thread
cursor[bot] marked this conversation as resolved.
logger.info("SaaS -> SaaS relocation RelocationFile saved", extra=logger_data)

uploading_complete.apply_async(args=[relocation.uuid])
Expand Down Expand Up @@ -169,29 +169,17 @@ def reply_with_export(
requesting_region_name: str,
replying_region_name: str,
org_slug: str,
encrypted_bytes: list[int],
# TODO(azaslavsky): finish transfer from `encrypted_contents` -> `encrypted_bytes`.
encrypted_bytes: list[int] | None = None,
encrypted_contents: bytes | None = None,
) -> None:
logger_data = {
"uuid": relocation_uuid,
"requesting_region_name": requesting_region_name,
"replying_region_name": replying_region_name,
"org_slug": org_slug,
# TODO(azaslavsky): finish transfer from `encrypted_contents` -> `encrypted_bytes`.
"encrypted_bytes_size": len(encrypted_bytes or []),
}
logger.info("SaaS -> SaaS reply received on proxy", extra=logger_data)

# Save the payload into the control silo's "relocation" GCS bucket. This bucket is only used
# for temporary storage of `encrypted_bytes` being shuffled between cells like this.
path = f"runs/{relocation_uuid}/saas_to_saas_export/{org_slug}.tar"
relocation_storage = get_relocation_storage()
# TODO(azaslavsky): finish transfer from `encrypted_contents` -> `encrypted_bytes`.
fp = BytesIO(bytes(encrypted_bytes or []))
relocation_storage.save(path, fp)
logger.info("SaaS -> SaaS export contents retrieved", extra=logger_data)

# Save transfer record so we can push state to the requesting cell
transfer = ControlRelocationTransfer.objects.create(
relocation_uuid=relocation_uuid,
Expand Down
8 changes: 4 additions & 4 deletions src/sentry/relocation/services/relocation_export/service.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -73,8 +73,8 @@ def reply_with_export(
requesting_region_name: str,
replying_region_name: str,
org_slug: str,
encrypted_bytes: list[int],
# TODO(azaslavsky): finish transfer from `encrypted_contents` -> `encrypted_bytes`.
# TODO(cells) bytes/contents are deprecated
encrypted_bytes: list[int] | None = None,
encrypted_contents: bytes | None = None,
) -> None:
"""
Expand Down Expand Up @@ -132,8 +132,8 @@ def reply_with_export(
requesting_region_name: str,
replying_region_name: str,
org_slug: str,
encrypted_bytes: list[int],
# TODO(azaslavsky): finish transfer from `encrypted_contents` -> `encrypted_bytes`.
# TODO(cells) bytes/contents are deprecated
encrypted_bytes: list[int] | None = None,
encrypted_contents: bytes | None = None,
) -> None:
"""
Expand Down
44 changes: 21 additions & 23 deletions src/sentry/relocation/tasks/process.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,6 @@
)
from sentry.backup.helpers import ImportFlags
from sentry.backup.imports import import_in_organization_scope
from sentry.models.files.file import File
from sentry.models.files.utils import get_relocation_storage
from sentry.models.importchunk import ControlImportChunkReplica, RegionImportChunk
from sentry.models.organization import Organization, OrganizationStatus
Expand All @@ -72,6 +71,7 @@
create_cloudbuild_yaml,
fail_relocation,
get_relocations_bucket_name,
relocation_raw_data_path,
retry_task_or_fail_relocation,
send_relocation_update_email,
start_relocation_task,
Expand Down Expand Up @@ -363,7 +363,6 @@

log_gcp_credentials_details(logger)
uuid = UUID(uuid_str)
path = f"runs/{uuid}/saas_to_saas_export/{org_slug}.tar"
relocation_storage = get_relocation_storage()
fp = BytesIO()
logger.info(
Expand Down Expand Up @@ -392,8 +391,11 @@
extra=logger_data,
)

# Rewind the file pointer and write to storage.
fp.seek(0)
path = relocation_raw_data_path(uuid)
relocation_storage.save(path, fp)

logger_data["encrypted_contents_size"] = fp.tell()
logger.info(
"fulfill_cross_region_export_request: saved",
Expand Down Expand Up @@ -491,10 +493,11 @@
if relocation is None:
return

# Pull down the `RelocationFile` associated with this `Relocation`. Fallibility is expected
# here: we're pushing a potentially very large file with many blobs to a cloud store, so it is
# possible (likely, even) that not all of the blobs are yet available. If this segment fails,
# we'll just allow the Exception to bubble up and retry the task if possible.
# Ensure that the `RelocationFile` associated with this `Relocation` exists.
# Fallibility is expected here: we're pushing a potentially very large file with many blobs
# to a cloud store, so it is possible (likely, even) that not all of the blobs are yet
# available. If this segment fails, we'll just allow the Exception to bubble up and retry
# the task if possible.
with retry_task_or_fail_relocation(
relocation,
OrderedTask.UPLOADING_COMPLETE,
Expand All @@ -509,9 +512,11 @@
.select_related("file")
.get()
)
fp = raw_relocation_file.file.getfile()
with fp:
preprocessing_scan.apply_async(args=[uuid])
logger.info(
"uploading_complete.relocation_file_present",
extra={"uuid": uuid, "id": raw_relocation_file.id},
)
preprocessing_scan.apply_async(args=[uuid])
Comment thread
cursor[bot] marked this conversation as resolved.


@instrumented_task(
Expand Down Expand Up @@ -563,7 +568,8 @@
.select_related("file")
.get()
)
fp = raw_relocation_file.file.getfile()
relocation_storage = get_relocation_storage()
fp = relocation_storage.open(raw_relocation_file.bucket_path)
Comment thread
sentry[bot] marked this conversation as resolved.

with fp:
try:
Expand Down Expand Up @@ -746,32 +752,22 @@
kms_config_bytes = json.dumps(get_default_crypto_key_version()).encode("utf-8")
relocation_storage.save(f"runs/{uuid}/in/kms-config.json", BytesIO(kms_config_bytes))

# Now, upload the relocation data proper.
kind = RelocationFile.Kind.RAW_USER_DATA
# Ensure that the RelocationFile exists.
raw_relocation_file = (
RelocationFile.objects.filter(
relocation=relocation,
kind=kind.value,
kind=RelocationFile.Kind.RAW_USER_DATA.value,
)
.select_related("file")
.prefetch_related("file__blobs")
.first()
)
if raw_relocation_file is None:
raise FileNotFoundError("User-supplied relocation data not found.")

file: File = raw_relocation_file.file
path = f"runs/{uuid}/in/{kind.to_filename('tar')}"

# Copy all of the files from Django's abstract filestore into an isolated,
# backend-specific filestore for relocation operations only.
fp = file.getfile()
fp.seek(0)
relocation_storage.save(path, fp)

preprocessing_baseline_config.apply_async(args=[uuid])


Check failure on line 770 in src/sentry/relocation/tasks/process.py

View check run for this annotation

@sentry/warden / warden: sentry-backend-bugs

Retried relocations fail: raw user data is never written to the new UUID's storage path 

`RelocationRetryEndpoint.post` creates a new `Relocation` with a fresh UUID and a new `RelocationFile` whose `bucket_path` is copied verbatim from the old relocation (`runs/{old_uuid}/in/raw-relocation-data.tar`). After this PR removes the file-copy that previously happened in `preprocessing_transfer`, nothing ever writes the raw tarball to `runs/{new_uuid}/in/raw-relocation-data.tar`. When `preprocessing_complete` (`process.py` lines 905–908) iterates `RELOCATION_FILES_TO_BE_VALIDATED` and checks `relocation_storage.exists(f"runs/{uuid}/in/{filename}")` using the new UUID, the check fails and raises `FileNotFoundError("Could not locate `raw-relocation-data.tar` in relocation bucket.")`, failing every retried relocation.
@instrumented_task(
name="sentry.relocation.preprocessing_baseline_config",
namespace=relocation_tasks,
Expand Down Expand Up @@ -1400,7 +1396,9 @@
.select_related("file")
.get()
)
relocation_data_fp = raw_relocation_file.file.getfile()
relocation_storage = get_relocation_storage()
relocation_data_fp = relocation_storage.open(raw_relocation_file.bucket_path)

log_gcp_credentials_details(logger)
kms_config_fp = BytesIO(json.dumps(get_default_crypto_key_version()).encode("utf-8"))

Expand Down
Loading
Loading