From 1e7973f58ce80f159aec82776f5ebb583139ed79 Mon Sep 17 00:00:00 2001 From: Rafael Sierra Date: Wed, 9 Sep 2026 16:52:19 +0200 Subject: [PATCH 1/3] feat(s3): add object annotations support PutObjectAnnotation, GetObjectAnnotation, ListObjectAnnotations, DeleteObjectAnnotation via the ?annotation subresource, with per-version storage, x-amz-annotation-directive on CopyObject, s3:ObjectAnnotation:* notifications, and x-amz-object-if-match. --- README.md | 2 +- .../com/floci/test/S3AnnotationsTest.java | 224 +++++++ docs/services/s3.md | 35 + .../cloudtrail/CloudTrailService.java | 3 +- .../floci/services/s3/S3Controller.java | 235 +++++++ .../floci/services/s3/S3Service.java | 597 +++++++++++++++++- .../services/s3/model/CopyObjectOptions.java | 5 + .../services/s3/model/ObjectAnnotation.java | 102 +++ .../floci/services/s3/model/S3Object.java | 2 +- .../s3/S3AnnotationsIntegrationTest.java | 360 +++++++++++ .../services/s3/S3ServiceAnnotationsTest.java | 548 ++++++++++++++++ 11 files changed, 2097 insertions(+), 16 deletions(-) create mode 100644 compatibility-tests/sdk-test-java/src/test/java/com/floci/test/S3AnnotationsTest.java create mode 100644 src/main/java/io/github/hectorvent/floci/services/s3/model/ObjectAnnotation.java create mode 100644 src/test/java/io/github/hectorvent/floci/services/s3/S3AnnotationsIntegrationTest.java create mode 100644 src/test/java/io/github/hectorvent/floci/services/s3/S3ServiceAnnotationsTest.java diff --git a/README.md b/README.md index 295b76e725..ff7c8a38ff 100644 --- a/README.md +++ b/README.md @@ -252,7 +252,7 @@ For operation-level compatibility, see the [Services Overview](https://floci.io/ | SSM | In-process + EC2 containers | Parameter Store (version history, labels, SecureString, tagging); Run Command (SendCommand, GetCommandInvocation, direct EC2 container execution, agent polling) | | SQS | In-process | Standard and FIFO queues, DLQ, visibility timeout, batch operations, tagging | | SNS | In-process | Topics, subscriptions, SQS, Lambda and HTTP delivery, tagging | -| S3 | In-process | Versioning, multipart upload, pre-signed URLs, Object Lock, event notifications | +| S3 | In-process | Versioning, multipart upload, pre-signed URLs, Object Lock, object annotations, event notifications | | S3 Vectors | In-process | Vector buckets, indexes, put / get / list / delete vectors, cosine similarity queries | | DynamoDB | In-process | GSI, LSI, Query, Scan, TTL, transactions, batch operations; Streams with shard iterators and Lambda event source mapping | | Lambda | Real Docker | Runtime environment, execution model, warm container pool, aliases, Function URLs | diff --git a/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/S3AnnotationsTest.java b/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/S3AnnotationsTest.java new file mode 100644 index 0000000000..12de1c5176 --- /dev/null +++ b/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/S3AnnotationsTest.java @@ -0,0 +1,224 @@ +package com.floci.test; + +import org.junit.jupiter.api.*; + +import software.amazon.awssdk.core.ResponseBytes; +import software.amazon.awssdk.core.sync.RequestBody; +import software.amazon.awssdk.services.s3.S3Client; +import software.amazon.awssdk.services.s3.model.AnnotationDirective; +import software.amazon.awssdk.services.s3.model.AnnotationEntry; +import software.amazon.awssdk.services.s3.model.BucketVersioningStatus; +import software.amazon.awssdk.services.s3.model.ChecksumMode; +import software.amazon.awssdk.services.s3.model.CreateBucketRequest; +import software.amazon.awssdk.services.s3.model.GetObjectAnnotationResponse; +import software.amazon.awssdk.services.s3.model.ListObjectAnnotationsRequest; +import software.amazon.awssdk.services.s3.model.ListObjectAnnotationsResponse; +import software.amazon.awssdk.services.s3.model.NoSuchAnnotationException; +import software.amazon.awssdk.services.s3.model.PutBucketVersioningRequest; +import software.amazon.awssdk.services.s3.model.PutObjectAnnotationRequest; +import software.amazon.awssdk.services.s3.model.VersioningConfiguration; + +import static org.assertj.core.api.Assertions.*; + +@DisplayName("S3 Object Annotations") +@TestMethodOrder(MethodOrderer.OrderAnnotation.class) +class S3AnnotationsTest { + + private static S3Client s3; + private static final String BUCKET = "sdk-annotations-bucket"; + private static final String KEY = "docs/annotated.txt"; + + @BeforeAll + static void setup() { + s3 = TestFixtures.s3Client(); + s3.createBucket(CreateBucketRequest.builder().bucket(BUCKET).build()); + s3.putObject(r -> r.bucket(BUCKET).key(KEY), RequestBody.fromString("annotated object body")); + } + + @AfterAll + static void teardown() { + for (String key : new String[]{"docs/annotated.txt", "docs/annotated-copy.txt", "docs/annotated-copied.txt"}) { + try { + s3.deleteObject(r -> r.bucket(BUCKET).key(key)); + } catch (Exception ignored) { + } + } + try { + s3.deleteBucket(r -> r.bucket(BUCKET)); + } catch (Exception ignored) { + } + } + + @Test + @Order(1) + void putObjectAnnotation() { + var response = s3.putObjectAnnotation(PutObjectAnnotationRequest.builder() + .bucket(BUCKET) + .key(KEY) + .annotationName("classification") + .build(), + RequestBody.fromString("{\"label\": \"report\"}")); + + assertThat(response.key()).isEqualTo(KEY); + assertThat(response.annotationName()).isEqualTo("classification"); + assertThat(response.eTag()).isNotBlank(); + // CRC64NVME is the AWS default algorithm for annotations without an explicit one. + // The SDK computes a CRC32 payload checksum client-side by default and expects it echoed. + assertThat(response.checksumCRC32()).isNotBlank(); + assertThat(response.checksumTypeAsString()).isEqualTo("FULL_OBJECT"); + } + + @Test + @Order(2) + void getObjectAnnotation() { + ResponseBytes response = s3.getObjectAnnotationAsBytes( + r -> r.bucket(BUCKET).key(KEY).annotationName("classification").build()); + + assertThat(response.asUtf8String()).isEqualTo("{\"label\": \"report\"}"); + assertThat(response.response().eTag()).isNotBlank(); + assertThat(response.response().lastModified()).isNotNull(); + assertThat(response.response().contentLength()).isEqualTo(19L); + } + + @Test + @Order(3) + void getObjectAnnotationWithChecksumMode() { + ResponseBytes response = s3.getObjectAnnotationAsBytes( + r -> r.bucket(BUCKET).key(KEY) + .annotationName("classification") + .checksumMode(ChecksumMode.ENABLED) + .build()); + + assertThat(response.response().checksumCRC32()).isNotBlank(); + assertThat(response.response().checksumTypeAsString()).isEqualTo("FULL_OBJECT"); + } + + @Test + @Order(4) + void listObjectAnnotations() { + s3.putObjectAnnotation(PutObjectAnnotationRequest.builder() + .bucket(BUCKET).key(KEY).annotationName("summary") + .build(), + RequestBody.fromString("summary text")); + + ListObjectAnnotationsResponse response = s3.listObjectAnnotations( + ListObjectAnnotationsRequest.builder().bucket(BUCKET).key(KEY).build()); + + assertThat(response.bucket()).isEqualTo(BUCKET); + assertThat(response.key()).isEqualTo(KEY); + assertThat(response.annotationCount()).isEqualTo(2); + assertThat(response.annotations()) + .extracting(AnnotationEntry::annotationName) + .containsExactly("classification", "summary"); + assertThat(response.annotations()) + .filteredOn(a -> "summary".equals(a.annotationName())) + .allSatisfy(a -> assertThat(a.size()).isEqualTo(12L)); + assertThat(response.nextContinuationToken()).isNull(); + assertThat(response.maxAnnotationResults()).isEqualTo(1000); + } + + @Test + @Order(5) + void putObjectAnnotationWithChecksum() { + var response = s3.putObjectAnnotation(PutObjectAnnotationRequest.builder() + .bucket(BUCKET).key(KEY).annotationName("hashed") + .checksumAlgorithm(software.amazon.awssdk.services.s3.model.ChecksumAlgorithm.SHA256) + .build(), + RequestBody.fromString("checksummed payload")); + + assertThat(response.checksumSHA256()).isNotBlank(); + assertThat(response.eTag()).isNotBlank(); + } + + @Test + @Order(6) + void getMissingAnnotationThrows() { + assertThatThrownBy(() -> s3.getObjectAnnotationAsBytes(r -> r.bucket(BUCKET).key(KEY) + .annotationName("missing").build())) + .isInstanceOf(NoSuchAnnotationException.class); + } + + @Test + @Order(7) + void deleteObjectAnnotationIsIdempotent() { + s3.deleteObjectAnnotation(r -> r.bucket(BUCKET).key(KEY).annotationName("hashed")); + + assertThatThrownBy(() -> s3.getObjectAnnotationAsBytes(r -> r.bucket(BUCKET).key(KEY) + .annotationName("hashed").build())) + .isInstanceOf(NoSuchAnnotationException.class); + + // Deleting a nonexistent annotation is not an error. + assertThatCode(() -> s3.deleteObjectAnnotation(r -> r.bucket(BUCKET).key(KEY) + .annotationName("hashed"))).doesNotThrowAnyException(); + } + + @Test + @Order(8) + void copyObjectExcludeDirectiveSkipsAnnotations() { + s3.copyObject(r -> r.sourceBucket(BUCKET).sourceKey(KEY) + .destinationBucket(BUCKET).destinationKey("docs/annotated-copy.txt") + .annotationDirective(AnnotationDirective.EXCLUDE)); + + assertThatThrownBy(() -> s3.getObjectAnnotationAsBytes(r -> r.bucket(BUCKET) + .key("docs/annotated-copy.txt").annotationName("classification").build())) + .isInstanceOf(NoSuchAnnotationException.class); + } + + // ========== Versioned bucket behavior ========== + + @Test + @Order(10) + void enableVersioning() { + s3.putBucketVersioning(r -> r.bucket(BUCKET) + .versioningConfiguration(VersioningConfiguration.builder() + .status(BucketVersioningStatus.ENABLED) + .build())); + } + + @Test + @Order(11) + void annotationAttachesToSpecificVersion() { + s3.putObject(r -> r.bucket(BUCKET).key(KEY), RequestBody.fromString("version one")); + var v1 = s3.putObjectAnnotation(PutObjectAnnotationRequest.builder() + .bucket(BUCKET).key(KEY).annotationName("v1-note") + .build(), + RequestBody.fromString("on version one")); + assertThat(v1.objectVersionId()).isNotBlank(); + + s3.putObject(r -> r.bucket(BUCKET).key(KEY), RequestBody.fromString("version two")); + + // The new version has no annotations; the old one's stay reachable by versionId. + assertThatThrownBy(() -> s3.getObjectAnnotationAsBytes(r -> r.bucket(BUCKET).key(KEY) + .annotationName("v1-note").build())) + .isInstanceOf(NoSuchAnnotationException.class); + + ResponseBytes fromV1 = s3.getObjectAnnotationAsBytes( + r -> r.bucket(BUCKET).key(KEY) + .annotationName("v1-note") + .versionId(v1.objectVersionId()) + .build()); + assertThat(fromV1.asUtf8String()).isEqualTo("on version one"); + assertThat(fromV1.response().objectVersionId()).isEqualTo(v1.objectVersionId()); + } + + @Test + @Order(12) + void copyObjectCopiesAnnotationsByDefault() { + // The latest version (the copy source) must carry an annotation for the copy to take. + s3.putObjectAnnotation(PutObjectAnnotationRequest.builder() + .bucket(BUCKET).key(KEY).annotationName("latest-note") + .build(), + RequestBody.fromString("copied annotation")); + s3.copyObject(r -> r.sourceBucket(BUCKET).sourceKey(KEY) + .destinationBucket(BUCKET).destinationKey("docs/annotated-copied.txt")); + + ResponseBytes copied = s3.getObjectAnnotationAsBytes( + r -> r.bucket(BUCKET) + .key("docs/annotated-copied.txt").annotationName("latest-note") + .versionId(s3.headObject(b -> b.bucket(BUCKET).key("docs/annotated-copied.txt")) + .versionId()) + .build()); + assertThat(copied.asUtf8String()).isEqualTo("copied annotation"); + s3.deleteObject(r -> r.bucket(BUCKET).key("docs/annotated-copied.txt")); + } +} \ No newline at end of file diff --git a/docs/services/s3.md b/docs/services/s3.md index ce5eb24a41..41f21c4a7d 100644 --- a/docs/services/s3.md +++ b/docs/services/s3.md @@ -13,6 +13,7 @@ | **Multipart** | CreateMultipartUpload, UploadPart, CompleteMultipartUpload, AbortMultipartUpload, ListMultipartUploads | | **Versioning** | PutBucketVersioning, GetBucketVersioning | | **Tagging** | PutBucketTagging, GetBucketTagging, PutObjectTagging, GetObjectTagging, DeleteObjectTagging | +| **Annotations** | PutObjectAnnotation, GetObjectAnnotation, ListObjectAnnotations, DeleteObjectAnnotation | | **Policy** | PutBucketPolicy, GetBucketPolicy, DeleteBucketPolicy | | **CORS** | PutBucketCors, GetBucketCors, DeleteBucketCors | | **Lifecycle** | PutBucketLifecycle, GetBucketLifecycle, DeleteBucketLifecycle | @@ -38,6 +39,40 @@ Browser and presigned POST uploads emit `s3:ObjectCreated:Post`, matching AWS S3. They do not emit `s3:ObjectCreated:Put`; use `s3:ObjectCreated:*` to subscribe to objects created by either method. +Annotation changes emit `s3:ObjectAnnotation:Put` and `s3:ObjectAnnotation:Delete`. + +## Object Annotations + +Annotations are named UTF-8 text payloads (up to 1 MiB each, 1,000 per object version) attached to a +specific object version through the four `?annotation` operations. Notes on the emulation: + +- Annotation names allow letters (any language), digits, `_`, `.`, and `-`; names longer than 512 + bytes, empty or whitespace-only names, names with other characters, and names starting with `aws` + or `s3` (case-insensitive) are rejected. +- Payloads must be valid UTF-8 text between 1 byte and 1 MiB; anything else is rejected with 400, + and non-UTF-8 payloads return 415 `UnsupportedMediaType`. +- `x-amz-object-if-match` is validated against the parent object's ETag on put and delete. +- Versioning semantics match AWS: annotations attach to one object version, new versions do not + inherit them, overwriting a non-versioned object or deleting it drops its annotations, a delete + marker preserves the underlying version's annotations, and deleting a specific version deletes + its annotations. Annotation deletion is permanent. +- `CopyObject` copies annotations by default; the `x-amz-object-annotation-directive` header + (as the AWS SDK sends it; `x-amz-annotation-directive` is also accepted) set to `EXCLUDE` + skips them. +- Checksums are per-annotation and independent of the object checksum. The default algorithm is + CRC64NVME. Supported: CRC32, CRC32C, CRC64NVME, SHA1, SHA256. SHA512, XXHASH64, XXHASH3, XXHASH128, + and MD5 are rejected as unsupported. +- Annotations on SSE-C encrypted objects are rejected, as on AWS, and are not copied onto SSE-C + copy destinations. +- Annotation operations serialize against object writes on the same bucket. On Object + Lock-protected versions, annotation put and delete follow the same rules as object delete: + governance retention requires `x-amz-bypass-governance-retention` (put never takes the bypass), + compliance and legal hold always block. +- The literal `versionId=null` (as reported by ListObjectVersions for pre-versioning objects) + addresses the pre-versioning entry. +- Annotations are stored per AWS account. With `globalBucketNamespace` enabled, object reads + resolve cross-account but annotation reads, writes, and listings stay in the caller's account. +- S3 Metadata annotation tables and annotation replication are not implemented. ## Website Hosting diff --git a/src/main/java/io/github/hectorvent/floci/services/cloudtrail/CloudTrailService.java b/src/main/java/io/github/hectorvent/floci/services/cloudtrail/CloudTrailService.java index 4600fb0931..406ae506d7 100644 --- a/src/main/java/io/github/hectorvent/floci/services/cloudtrail/CloudTrailService.java +++ b/src/main/java/io/github/hectorvent/floci/services/cloudtrail/CloudTrailService.java @@ -452,7 +452,8 @@ static boolean isReadOnlyEvent(String eventName) { if (eventName == null) return true; return switch (eventName) { case "GetObject", "HeadObject", "ListObjects", "ListObjectsV2", - "GetObjectAcl", "GetObjectTagging", "ListMultipartUploads" -> true; + "GetObjectAcl", "GetObjectTagging", "ListMultipartUploads", + "GetObjectAnnotation", "ListObjectAnnotations" -> true; default -> false; }; } diff --git a/src/main/java/io/github/hectorvent/floci/services/s3/S3Controller.java b/src/main/java/io/github/hectorvent/floci/services/s3/S3Controller.java index 55ad3066cd..60ddae803e 100644 --- a/src/main/java/io/github/hectorvent/floci/services/s3/S3Controller.java +++ b/src/main/java/io/github/hectorvent/floci/services/s3/S3Controller.java @@ -18,6 +18,7 @@ import io.github.hectorvent.floci.services.s3.model.ChecksumType; import io.github.hectorvent.floci.services.s3.model.GetObjectAttributesParts; import io.github.hectorvent.floci.services.s3.model.GetObjectAttributesResult; +import io.github.hectorvent.floci.services.s3.model.ObjectAnnotation; import io.github.hectorvent.floci.services.s3.model.LambdaNotification; import io.github.hectorvent.floci.services.s3.model.MultipartUpload; import io.github.hectorvent.floci.services.s3.model.FilterRule; @@ -762,6 +763,11 @@ public Response putObject(@PathParam("bucket") String bucket, return Response.ok().build(); } + if (hasQueryParam(uriInfo, "annotation")) { + s3Service.authorizeObjectWrite(bucket, key, "s3:PutObjectAnnotation", authorization); + return handlePutObjectAnnotation(bucket, key, body, uriInfo, httpHeaders); + } + if (uploadId != null && partNumber != null) { s3Service.authorizeObjectWrite(bucket, key, "s3:PutObject", authorization); if (copySource != null && !copySource.isEmpty()) { @@ -893,6 +899,17 @@ public Response getObject(@PathParam("bucket") String bucket, s3Service.authorizeObjectRead(bucket, key, versionId, "s3:GetObjectTagging", authorization); return handleGetObjectTagging(bucket, key); } + if (hasQueryParam(uriInfo, "annotation")) { + String annotationNameParam = uriInfo.getQueryParameters().getFirst("annotationName"); + // An empty annotationName is not a Get request for the empty name: it falls + // through to ListObjectAnnotations, matching the blank-name put semantics. + if (annotationNameParam != null && !annotationNameParam.isEmpty()) { + s3Service.authorizeObjectRead(bucket, key, versionId, "s3:GetObjectAnnotation", authorization); + return handleGetObjectAnnotation(bucket, key, uriInfo, httpHeaders); + } + s3Service.authorizeObjectRead(bucket, key, versionId, "s3:ListObjectAnnotations", authorization); + return handleListObjectAnnotations(bucket, key, uriInfo); + } if (hasQueryParam(uriInfo, "retention")) { s3Service.authorizeObjectRead(bucket, key, versionId, "s3:GetObjectRetention", authorization); return handleGetObjectRetention(bucket, key, versionId); @@ -1114,6 +1131,19 @@ public Response headObject(@PathParam("bucket") String bucket, return headOnlyResponse(websiteResponse); } } + // HEAD honors the annotation subresource so a HEAD probe reports the same status and + // annotation metadata a GET would, instead of falling through to the object's headers. + if (hasQueryParam(uriInfo, "annotation")) { + String annotationNameParam = uriInfo.getQueryParameters().getFirst("annotationName"); + // An empty annotationName is not a Get request for the empty name: it falls + // through to ListObjectAnnotations, matching the blank-name put semantics. + if (annotationNameParam != null && !annotationNameParam.isEmpty()) { + s3Service.authorizeObjectRead(bucket, key, versionId, "s3:GetObjectAnnotation", authorization); + return headOnlyResponse(handleGetObjectAnnotation(bucket, key, uriInfo, httpHeaders)); + } + s3Service.authorizeObjectRead(bucket, key, versionId, "s3:ListObjectAnnotations", authorization); + return headOnlyResponse(handleListObjectAnnotations(bucket, key, uriInfo)); + } s3Service.authorizeGetObject(bucket, key, versionId, authorization); S3Object obj = s3Service.headObject(bucket, key, versionId); @@ -1248,6 +1278,15 @@ public Response deleteObject(@PathParam("bucket") String bucket, s3Service.deleteObjectTagging(bucket, key); return Response.noContent().build(); } + if (hasQueryParam(uriInfo, "annotation")) { + boolean bypass = "true".equalsIgnoreCase( + httpHeaders.getHeaderString("x-amz-bypass-governance-retention")); + s3Service.authorizeObjectWrite(bucket, key, "s3:DeleteObjectAnnotation", authorization); + if (bypass) { + s3Service.authorizeObjectWrite(bucket, key, "s3:BypassGovernanceRetention", authorization); + } + return handleDeleteObjectAnnotation(bucket, key, uriInfo, httpHeaders, bypass); + } if (uploadId != null) { s3Service.authorizeObjectWrite(bucket, key, "s3:AbortMultipartUpload", authorization); s3Service.abortMultipartUpload(bucket, key, uploadId); @@ -2085,6 +2124,188 @@ private String buildTaggingXml(Map tags) { return xml.build(); } + // --- Object Annotations --- + + private Response handlePutObjectAnnotation(String bucket, String key, byte[] body, + UriInfo uriInfo, HttpHeaders httpHeaders) { + try { + // Strip aws-chunked framing when present, like the PutObject path this handler + // mirrors: a streaming-signed request otherwise persists its framing bytes as + // the annotation payload. + byte[] payload = decodeAwsChunked(body != null ? body : new byte[0], + httpHeaders.getHeaderString("Content-Encoding"), + httpHeaders.getHeaderString("x-amz-content-sha256")); + String annotationName = uriInfo.getQueryParameters().getFirst("annotationName"); + String versionId = uriInfo.getQueryParameters().getFirst("versionId"); + String algorithmHeader = getChecksumAlgorithm(httpHeaders); + ChecksumAlgorithm algorithm = ChecksumAlgorithm.fromWireValue(algorithmHeader); + validateChecksumHeaders(httpHeaders, payload, algorithmHeader); + validateContentMd5(httpHeaders, payload); + ObjectAnnotation annotation = s3Service.putObjectAnnotation(bucket, key, annotationName, + versionId, payload, httpHeaders.getHeaderString("x-amz-object-if-match"), algorithm); + Response.ResponseBuilder response = Response.ok(putObjectAnnotationXml(annotation)) + .type(MediaType.APPLICATION_XML) + .header("ETag", annotation.getETag()); + if (annotation.getVersionId() != null) { + response.header("x-amz-object-version-id", annotation.getVersionId()); + } + appendAnnotationChecksumHeaders(response, annotation); + appendAnnotationSseHeader(response, annotation); + emitCloudTrailEvent("PutObjectAnnotation", bucket, key, payload.length, 0L, null, null); + return response.build(); + } catch (AwsException e) { + emitCloudTrailEvent("PutObjectAnnotation", bucket, key, 0L, 0L, e.getErrorCode(), e.getMessage()); + return xmlErrorResponse(e); + } + } + + private Response handleGetObjectAnnotation(String bucket, String key, + UriInfo uriInfo, HttpHeaders httpHeaders) { + try { + String annotationName = uriInfo.getQueryParameters().getFirst("annotationName"); + String versionId = uriInfo.getQueryParameters().getFirst("versionId"); + ObjectAnnotation annotation = s3Service.getObjectAnnotation(bucket, key, annotationName, versionId); + byte[] payload = s3Service.readObjectAnnotationPayload(annotation); + Response.ResponseBuilder response = Response.ok((Object) payload) + .type(MediaType.APPLICATION_OCTET_STREAM) + .header("Content-Length", payload.length) + .header("ETag", annotation.getETag()) + .header("Last-Modified", RFC_822.format(annotation.getLastModified())); + // No Accept-Ranges: range requests are not honored on annotation payloads. + if (annotation.getVersionId() != null) { + response.header("x-amz-object-version-id", annotation.getVersionId()); + } + if ("ENABLED".equalsIgnoreCase(httpHeaders.getHeaderString("x-amz-checksum-mode"))) { + appendAnnotationChecksumHeaders(response, annotation); + } + appendAnnotationSseHeader(response, annotation); + emitCloudTrailEvent("GetObjectAnnotation", bucket, key, 0L, payload.length, null, null); + return response.build(); + } catch (AwsException e) { + emitCloudTrailEvent("GetObjectAnnotation", bucket, key, 0L, 0L, e.getErrorCode(), e.getMessage()); + return xmlErrorResponse(e); + } + } + + private Response handleListObjectAnnotations(String bucket, String key, UriInfo uriInfo) { + try { + String versionId = uriInfo.getQueryParameters().getFirst("versionId"); + String prefix = uriInfo.getQueryParameters().getFirst("annotation-prefix"); + String token = uriInfo.getQueryParameters().getFirst("continuation-token"); + String maxRaw = uriInfo.getQueryParameters().getFirst("max-annotation-results"); + Integer max = null; + if (maxRaw != null && !maxRaw.isBlank()) { + try { + max = Integer.parseInt(maxRaw); + } catch (NumberFormatException e) { + throw new AwsException("InvalidArgument", + "max-annotation-results must be an integer.", 400); + } + } + S3Service.ListObjectAnnotationsResult result = + s3Service.listObjectAnnotations(bucket, key, prefix, max, token, versionId); + XmlBuilder xml = new XmlBuilder() + .raw("") + .start("ListObjectAnnotationsOutput", AwsNamespaces.S3) + .start("Annotations"); + for (ObjectAnnotation annotation : result.annotations()) { + xml.start("AnnotationEntry") + .elem("AnnotationName", annotation.getAnnotationName()) + .elem("ChecksumAlgorithm", annotation.getChecksumAlgorithm()) + .elem("ETag", annotation.getETag()) + .elem("LastModified", ISO_FORMAT.format(annotation.getLastModified())) + .elem("Size", annotation.getSize()) + .end("AnnotationEntry"); + } + xml.end("Annotations") + .elem("Bucket", bucket) + .elem("Key", key); + if (prefix != null) { + xml.elem("AnnotationPrefix", prefix); + } + xml.elem("MaxAnnotationResults", result.maxAnnotationResults()) + .elem("AnnotationCount", result.annotations().size()) + .elem("IsTruncated", result.isTruncated()); + if (token != null) { + xml.elem("ContinuationToken", token); + } + if (result.isTruncated()) { + xml.elem("NextContinuationToken", result.nextContinuationToken()); + } + xml.end("ListObjectAnnotationsOutput"); + Response.ResponseBuilder response = Response.ok(xml.build()).type(MediaType.APPLICATION_XML); + String versionIdOfPage = result.annotations().isEmpty() + ? versionId : result.annotations().get(0).getVersionId(); + if (versionIdOfPage != null) { + response.header("x-amz-object-version-id", versionIdOfPage); + } + emitCloudTrailEvent("ListObjectAnnotations", bucket, key, 0L, 0L, null, null); + return response.build(); + } catch (AwsException e) { + emitCloudTrailEvent("ListObjectAnnotations", bucket, key, 0L, 0L, e.getErrorCode(), e.getMessage()); + return xmlErrorResponse(e); + } + } + + private Response handleDeleteObjectAnnotation(String bucket, String key, + UriInfo uriInfo, HttpHeaders httpHeaders, + boolean bypassGovernance) { + try { + String annotationName = uriInfo.getQueryParameters().getFirst("annotationName"); + String versionId = uriInfo.getQueryParameters().getFirst("versionId"); + String parentVersionId = s3Service.deleteObjectAnnotation(bucket, key, annotationName, versionId, + httpHeaders.getHeaderString("x-amz-object-if-match"), bypassGovernance); + Response.ResponseBuilder response = Response.noContent(); + if (parentVersionId != null) { + response.header("x-amz-object-version-id", parentVersionId); + } + emitCloudTrailEvent("DeleteObjectAnnotation", bucket, key, 0L, 0L, null, null); + return response.build(); + } catch (AwsException e) { + emitCloudTrailEvent("DeleteObjectAnnotation", bucket, key, 0L, 0L, e.getErrorCode(), e.getMessage()); + return xmlErrorResponse(e); + } + } + + private String putObjectAnnotationXml(ObjectAnnotation annotation) { + return new XmlBuilder() + .raw("") + .start("PutObjectAnnotationOutput", AwsNamespaces.S3) + .elem("Key", annotation.getKey()) + .elem("AnnotationName", annotation.getAnnotationName()) + .end("PutObjectAnnotationOutput") + .build(); + } + + private void appendAnnotationChecksumHeaders(Response.ResponseBuilder response, ObjectAnnotation annotation) { + if (annotation.getChecksumAlgorithm() == null || annotation.getChecksumValue() == null) { + return; + } + response.header(ObjectAnnotation.checksumHeaderName(annotation.getChecksumAlgorithm()), annotation.getChecksumValue()) + .header("x-amz-checksum-type", "FULL_OBJECT"); + } + + private void appendAnnotationSseHeader(Response.ResponseBuilder response, ObjectAnnotation annotation) { + if (annotation.getServerSideEncryption() != null) { + response.header("x-amz-server-side-encryption", annotation.getServerSideEncryption()); + } + } + + private void validateContentMd5(HttpHeaders httpHeaders, byte[] body) { + String contentMd5 = httpHeaders.getHeaderString("Content-MD5"); + if (contentMd5 == null) { + return; + } + try { + byte[] digest = java.security.MessageDigest.getInstance("MD5").digest(body); + if (!contentMd5.equals(java.util.Base64.getEncoder().encodeToString(digest))) { + throw new AwsException("BadDigest", "The Content-MD5 you specified did not match the payload.", 400); + } + } catch (java.security.NoSuchAlgorithmException e) { + throw new IllegalStateException("MD5 algorithm is not available", e); + } + } + // --- Object Lock Configuration --- private Response handlePutObjectLockConfiguration(String bucket, byte[] body) { @@ -2314,6 +2535,19 @@ private Response handleCopyObject(String copySource, String destBucket, String d Map replacementTagging = "REPLACE".equalsIgnoreCase(taggingDirective) ? (taggingHeader != null ? parseInlineTaggingHeader(taggingHeader) : Map.of()) : null; + // The AWS SDK (and real S3) marshal the copy annotation directive as + // x-amz-object-annotation-directive; the user guide also names x-amz-annotation-directive, + // so both are accepted. + String annotationDirective = httpHeaders.getHeaderString("x-amz-object-annotation-directive"); + if (annotationDirective == null) { + annotationDirective = httpHeaders.getHeaderString("x-amz-annotation-directive"); + } + if (annotationDirective != null + && !"COPY".equalsIgnoreCase(annotationDirective) + && !"EXCLUDE".equalsIgnoreCase(annotationDirective)) { + throw new AwsException("InvalidRequest", + "x-amz-annotation-directive must be COPY or EXCLUDE.", 400); + } S3Object copy = s3Service.copyObject(sourceBucket, sourceObject.objectKey(), destBucket, destKey, sourceObject.versionId(), new CopyObjectOptions() @@ -2334,6 +2568,7 @@ private Response handleCopyObject(String copySource, String destBucket, String d .withCopySourceSseCustomerKey(httpHeaders.getHeaderString("x-amz-copy-source-server-side-encryption-customer-key")) .withCopySourceSseCustomerKeyMd5(httpHeaders.getHeaderString("x-amz-copy-source-server-side-encryption-customer-key-MD5")) .withChecksumAlgorithm(getChecksumAlgorithm(httpHeaders)) + .withAnnotationDirective(annotationDirective) .withAcl(cannedAcl) .withGrantRead(httpHeaders.getHeaderString("x-amz-grant-read")) .withGrantWrite(httpHeaders.getHeaderString("x-amz-grant-write")) diff --git a/src/main/java/io/github/hectorvent/floci/services/s3/S3Service.java b/src/main/java/io/github/hectorvent/floci/services/s3/S3Service.java index 43e546def9..7ca79bd998 100644 --- a/src/main/java/io/github/hectorvent/floci/services/s3/S3Service.java +++ b/src/main/java/io/github/hectorvent/floci/services/s3/S3Service.java @@ -33,7 +33,9 @@ import java.io.IOException; import java.io.InputStream; import java.io.UncheckedIOException; +import java.net.URLEncoder; import java.nio.charset.StandardCharsets; +import java.nio.file.NoSuchFileException; import java.nio.file.AtomicMoveNotSupportedException; import java.nio.file.Files; import java.nio.file.Path; @@ -77,9 +79,15 @@ static RequestAuthorization unsigned() { private final StorageBackend bucketStore; private final StorageBackend objectStore; + private final StorageBackend annotationStore; private final Path dataRoot; private final boolean inMemory; private final ConcurrentHashMap memoryDataStore = new ConcurrentHashMap<>(); + // Annotation payload bytes, keyed by physical key like memoryDataStore. Kept out of + // annotationStore for the same reason object bodies are kept out of objectStore: every + // backend serializes its whole map into a single document on each flush, so payloads + // (up to 1 MiB each, up to 1,000 per object version) must not be inline. + private final ConcurrentHashMap memoryAnnotationStore = new ConcurrentHashMap<>(); // Guards disk writes/deletes against a racing legacy migration for the same path (see // copyLegacyFileIfPresent()). Fixed-size stripes keep memory bounded, unlike a per-path // map that would need reference counting to ever shrink safely. @@ -135,6 +143,9 @@ public S3Service(StorageFactory storageFactory, EmulatorConfig config, storageFactory.create("s3", "s3-objects.json", new TypeReference>() { }), + storageFactory.create("s3", "s3-annotations.json", + new TypeReference>() { + }), storageFactory.create("s3", "s3-account-public-access-block.json", new TypeReference>() { }), @@ -155,7 +166,7 @@ public S3Service(StorageFactory storageFactory, EmulatorConfig config, S3Service(StorageBackend bucketStore, StorageBackend objectStore, Path dataRoot, boolean inMemory) { - this(bucketStore, objectStore, defaultAccountPublicAccessBlockStore(), + this(bucketStore, objectStore, defaultAnnotationStore(), defaultAccountPublicAccessBlockStore(), dataRoot, inMemory, null, null, null, null, null, null, null, null, "http://localhost:4566", new ObjectMapper(), false, null, false); } @@ -165,7 +176,7 @@ public S3Service(StorageFactory storageFactory, EmulatorConfig config, StorageBackend objectStore, AccountAwareStorageBackend accountPublicAccessBlockStore, Path dataRoot, boolean inMemory) { - this(bucketStore, objectStore, accountPublicAccessBlockStore, + this(bucketStore, objectStore, defaultAnnotationStore(), accountPublicAccessBlockStore, dataRoot, inMemory, null, null, null, null, null, null, null, null, "http://localhost:4566", new ObjectMapper(), false, null, false); } @@ -174,7 +185,7 @@ public S3Service(StorageFactory storageFactory, EmulatorConfig config, S3Service(StorageBackend bucketStore, StorageBackend objectStore, Path dataRoot, boolean inMemory, boolean globalBucketNamespace) { - this(bucketStore, objectStore, defaultAccountPublicAccessBlockStore(), + this(bucketStore, objectStore, defaultAnnotationStore(), defaultAccountPublicAccessBlockStore(), dataRoot, inMemory, null, null, null, null, null, null, null, null, "http://localhost:4566", new ObjectMapper(), false, null, globalBucketNamespace); } @@ -184,7 +195,7 @@ public S3Service(StorageFactory storageFactory, EmulatorConfig config, Path dataRoot, boolean inMemory, LambdaService lambdaService, RegionResolver regionResolver) { - this(bucketStore, objectStore, defaultAccountPublicAccessBlockStore(), + this(bucketStore, objectStore, defaultAnnotationStore(), defaultAccountPublicAccessBlockStore(), dataRoot, inMemory, null, null, lambdaService, null, null, null, null, regionResolver, "http://localhost:4566", new ObjectMapper(), false, null, false); } @@ -194,7 +205,7 @@ public S3Service(StorageFactory storageFactory, EmulatorConfig config, Path dataRoot, boolean inMemory, LambdaInvoker lambdaInvoker, RegionResolver regionResolver) { - this(bucketStore, objectStore, defaultAccountPublicAccessBlockStore(), + this(bucketStore, objectStore, defaultAnnotationStore(), defaultAccountPublicAccessBlockStore(), dataRoot, inMemory, null, null, null, null, lambdaInvoker, null, null, regionResolver, "http://localhost:4566", new ObjectMapper(), false, null, false); } @@ -204,8 +215,14 @@ private static AccountAwareStorageBackend defaultAccountPublicAccessBloc return AccountAwareStorageBackend.inMemory("000000000000"); } + /** In-memory annotation store for the package-private test constructors. */ + private static AccountAwareStorageBackend defaultAnnotationStore() { + return AccountAwareStorageBackend.inMemory("000000000000"); + } + private S3Service(StorageBackend bucketStore, StorageBackend objectStore, + AccountAwareStorageBackend annotationStore, AccountAwareStorageBackend accountPublicAccessBlockStore, Path dataRoot, boolean inMemory, SqsService sqsService, SnsService snsService, LambdaService lambdaService, @@ -217,6 +234,7 @@ private S3Service(StorageBackend bucketStore, boolean enforceAuth, IamService iamService, boolean globalBucketNamespace) { this.bucketStore = bucketStore; this.objectStore = objectStore; + this.annotationStore = annotationStore; this.accountPublicAccessBlockStore = accountPublicAccessBlockStore; this.dataRoot = dataRoot; this.inMemory = inMemory; @@ -244,6 +262,7 @@ private S3Service(StorageBackend bucketStore, public void clear() { memoryDataStore.clear(); + memoryAnnotationStore.clear(); memoryMultipartStore.clear(); multipartUploads.clear(); } @@ -300,11 +319,15 @@ private void deleteBucketLocked(String bucketName) { } bucketStore.delete(bucketName); + deleteAllAnnotationsForBucket(bucketName); if (inMemory) { String prefix = ownerId() + "/" + bucketName + "/"; memoryDataStore.keySet().removeIf(k -> k.startsWith(prefix)); + memoryAnnotationStore.keySet().removeIf(k -> k.startsWith(prefix)); } else { deleteDirectory(dataRoot.resolve(ACCOUNT_STORAGE_ROOT).resolve(ownerId()).resolve(bucketName)); + deleteDirectory(dataRoot.resolve(ACCOUNT_STORAGE_ROOT).resolve(ownerId()) + .resolve(ANNOTATION_STORAGE_ROOT).resolve(bucketName)); } } @@ -488,6 +511,10 @@ private S3Object storeObjectInternal(Bucket bucket, String bucketName, String ke // Check lock protection on the current latest before overwriting String latestKey = objectKey(bucketName, key); + // A pre-versioning object being replaced: its annotations were keyed at the plain + // object key and would be left unreachable by the new version. Cleanup is deferred + // until the replacement body is on disk, so a failed write does not drop them. + boolean[] dropPreVersioningAnnotations = {false}; objectStore.get(latestKey).ifPresent(prev -> { if (prev.isLatest() && !prev.isDeleteMarker() && bucket.isObjectLockEnabled()) { checkLockProtection(prev, false); @@ -495,6 +522,8 @@ private S3Object storeObjectInternal(Bucket bucket, String bucketName, String ke if (prev.getVersionId() != null) { prev.setLatest(false); objectStore.put(versionedKey(bucketName, key, prev.getVersionId()), prev); + } else { + dropPreVersioningAnnotations[0] = true; } }); @@ -514,6 +543,11 @@ private S3Object storeObjectInternal(Bucket bucket, String bucketName, String ke // GET can't observe corrupted "latest" bytes paired with the old metadata. writeVersionedFile(bucketName, key, versionId, data); writeFile(bucketName, key, data); + // Deferred pre-versioning annotation cleanup: only after the replacement body is on + // disk, so a failed write keeps the old body and its annotations together. + if (dropPreVersioningAnnotations[0]) { + deleteAllAnnotationsFor(annotationParentKey(bucketName, key, null)); + } // Release the cached payload before publishing: once objectStore.put makes this // instance visible to other threads, a concurrent getObject can hold a reference to // it (copyObject reads getData() without any lock) and race this null-out otherwise. @@ -543,6 +577,11 @@ private S3Object storeObjectInternal(Bucket bucket, String bucketName, String ke // Write the body before publishing metadata - see the comment in the versioned // branch above; the same ordering requirement applies here. writeFile(bucketName, key, data); + // An overwrite replaces the object's annotations (AWS drops them on overwrite). + // The cleanup runs only after the body write succeeds, so a failed PUT keeps the + // old body together with its annotations; and before the new metadata is published, + // so the replacement never appears annotated. + deleteAllAnnotationsFor(annotationParentKey(bucketName, key, null)); // Release the cached payload before publishing - see the comment in the versioned // branch above; the same race applies here. object.setData(null); @@ -1141,6 +1180,16 @@ public S3Object deleteObject(String bucketName, String key, String versionId, bo .orElseThrow(() -> new AwsException("NoSuchBucket", "The specified bucket does not exist.", 404)); + // The bucket monitor serializes this against PutObject's annotation cleanup and the + // annotation subresource writes, which hold the same monitor (storeObjectInternal + // already runs under it via storeObject). + synchronized (bucket) { + return deleteObjectLocked(bucket, bucketName, key, versionId, bypassGovernance); + } + } + + private S3Object deleteObjectLocked(Bucket bucket, String bucketName, String key, + String versionId, boolean bypassGovernance) { if (bucket.isVersioningEnabled() && versionId == null) { // Check lock on current latest before placing a delete marker objectStore.get(objectKey(bucketName, key)).ifPresent(prev -> { @@ -1161,6 +1210,11 @@ public S3Object deleteObject(String bucketName, String key, String versionId, bo if (prev.getVersionId() != null) { prev.setLatest(false); objectStore.put(versionedKey(bucketName, key, prev.getVersionId()), prev); + } else { + // The marker replaces a pre-versioning object: no versioned entry ever + // existed, so its annotations become unreachable and are removed here + // (they are permanent, as on AWS). + deleteAllAnnotationsFor(annotationParentKey(bucketName, key, null)); } }); @@ -1178,6 +1232,7 @@ public S3Object deleteObject(String bucketName, String key, String versionId, bo // Permanently delete a specific version (metadata + file data) objectStore.delete(versionedKey(bucketName, key, versionId)); deleteVersionedFile(bucketName, key, versionId); + deleteAllAnnotationsFor(annotationParentKey(bucketName, key, versionId)); LOG.debugv("Permanently deleted version: {0}/{1} v={2}", bucketName, key, versionId); // Promote the next most-recent version when the deleted one was the latest String latestKey = objectKey(bucketName, key); @@ -1219,6 +1274,7 @@ public S3Object deleteObject(String bucketName, String key, String versionId, bo // Non-versioned delete objectStore.delete(objectKey(bucketName, key)); deleteFile(bucketName, key); + deleteAllAnnotationsFor(annotationParentKey(bucketName, key, null)); LOG.debugv("Deleted object: {0}/{1}", bucketName, key); fireNotifications(bucketName, key, "ObjectRemoved:Delete", null); return null; @@ -1646,6 +1702,428 @@ public void deleteObjectTagging(String bucketName, String key) { LOG.debugv("Deleted tags from object: {0}/{1}", bucketName, key); } + // --- Object Annotations --- + + public static final int MAX_ANNOTATIONS_PER_VERSION = 1_000; + public static final int MAX_ANNOTATION_RESULTS = 1_000; + private static final int MAX_ANNOTATION_NAME_BYTES = 512; + private static final int MAX_ANNOTATION_PAYLOAD_BYTES = 1_048_576; + // '@' can never occur in a valid annotation name, so this separator cannot be produced by a + // name itself. Object keys CAN contain '@' and '#', which is why annotationParentKey + // URL-encodes the key before appending the separators. + private static final String ANNOTATION_SEPARATOR = "@ann@"; + private static final String ANNOTATION_DATA_SUFFIX = ".s3ann"; + private static final String ANNOTATION_STORAGE_ROOT = ".annotations"; + + /** maxAnnotationResults carries the effective limit (default applied) so callers echo the value the service enforced. */ + public record ListObjectAnnotationsResult(List annotations, boolean isTruncated, + String nextContinuationToken, int maxAnnotationResults) {} + + public ObjectAnnotation putObjectAnnotation(String bucketName, String key, String annotationName, + String versionId, byte[] payload, String ifMatch, + ChecksumAlgorithm checksumAlgorithm) { + Bucket bucket = requireBucket(bucketName); + S3Object[] notificationTarget = {null}; + ObjectAnnotation annotation; + synchronized (bucket) { + versionId = normalizeNullVersionId(versionId); + S3Object parent = resolveParentObject(bucketName, key, versionId); + // Symmetric with deleteObjectAnnotation: an annotation put must not add or replace + // an annotation on a retention-protected version, or the delete path's protection + // is circumvented by a re-put. + checkLockProtection(parent, false); + if (parent.getSseCustomerAlgorithm() != null) { + // AWS rejects annotations on SSE-C encrypted objects. + throw new AwsException("InvalidRequest", + "Server-side encryption with customer-provided keys is not supported for annotations.", 400); + } + if (ifMatch != null && !eTagMatches(ifMatch, parent.getETag())) { + throw new S3PreconditionFailedException("If-Match"); + } + validateAnnotationName(annotationName); + validateAnnotationPayload(payload); + + String parentKey = parentStoreKey(bucketName, key, versionId, parent); + String storeKey = annotationStoreKey(parentKey, annotationName); + // Account-scoped, like every annotation write: in globalBucketNamespace mode a + // cross-account probe would let the caller bypass the per-version limit by reading + // another account's entry, while the put itself lands in the caller's partition. + boolean isUpdate = annotationStore.get(storeKey).isPresent(); + // Check-then-act against concurrent annotation puts is safe: this method holds the + // bucket monitor, the same one storeObject's overwrite cleanup holds. + if (!isUpdate && countAnnotations(parentKey) >= MAX_ANNOTATIONS_PER_VERSION) { + throw new AwsException("AnnotationLimitExceeded", + "The maximum number of annotations for this object version has been reached.", 400); + } + + ChecksumAlgorithm algorithm = checksumAlgorithm != null ? checksumAlgorithm : ChecksumAlgorithm.CRC64NVME; + annotation = new ObjectAnnotation(bucketName, key, parent.getVersionId(), + annotationName, payload.length, S3Object.computeETag(payload), Instant.now(), + algorithm.name(), algorithm.compute(payload)); + annotation.setServerSideEncryption(parent.getServerSideEncryption()); + + // Write the payload before publishing metadata, mirroring storeObjectInternal's + // write-before-publish ordering. + writeAnnotationPayload(annotation, payload); + annotationStore.put(storeKey, annotation); + LOG.debugv("Put annotation {0} on object: {1}/{2}", annotationName, bucketName, key); + notificationTarget[0] = parent; + } + // Fired outside the bucket monitor (the storeObject callers' pattern): a slow SQS/SNS/ + // Lambda delivery must not block every other write and annotation op on the bucket. + fireNotifications(bucketName, key, "ObjectAnnotation:Put", notificationTarget[0]); + return annotation; + } + + public ObjectAnnotation getObjectAnnotation(String bucketName, String key, String annotationName, + String versionId) { + Bucket bucket = requireBucket(bucketName); + synchronized (bucket) { + versionId = normalizeNullVersionId(versionId); + S3Object parent = resolveParentObject(bucketName, key, versionId); + validateAnnotationName(annotationName); + String storeKey = annotationStoreKey(parentStoreKey(bucketName, key, versionId, parent), annotationName); + return annotationStore.get(storeKey) + .orElseThrow(() -> new AwsException("NoSuchAnnotation", + "The specified annotation does not exist.", 404)); + } + } + + public byte[] readObjectAnnotationPayload(ObjectAnnotation annotation) { + byte[] payload = readAnnotationPayload(annotation); + if (payload == null) { + throw new AwsException("NoSuchAnnotation", + "The specified annotation does not exist.", 404); + } + return payload; + } + + public ListObjectAnnotationsResult listObjectAnnotations(String bucketName, String key, + String annotationPrefix, + Integer maxAnnotationResults, + String continuationToken, String versionId) { + Bucket bucket = requireBucket(bucketName); + synchronized (bucket) { + versionId = normalizeNullVersionId(versionId); + S3Object parent = resolveParentObject(bucketName, key, versionId); + int limit = maxAnnotationResults != null ? maxAnnotationResults : MAX_ANNOTATION_RESULTS; + if (limit < 1 || limit > MAX_ANNOTATION_RESULTS) { + throw new AwsException("InvalidArgument", + "max-annotation-results must be between 1 and 1000.", 400); + } + validateAnnotationPrefix(annotationPrefix); + String startAfter = decodeAnnotationContinuationToken(continuationToken); + + String parentKey = parentStoreKey(bucketName, key, versionId, parent); + List matches = annotationStore.scan(k -> k.startsWith(parentKey + ANNOTATION_SEPARATOR)) + .stream() + .filter(a -> annotationPrefix == null || a.getAnnotationName().startsWith(annotationPrefix)) + .sorted(Comparator.comparing(ObjectAnnotation::getAnnotationName)) + .toList(); + if (startAfter != null) { + matches = matches.stream() + .filter(a -> a.getAnnotationName().compareTo(startAfter) > 0) + .toList(); + } + boolean truncated = matches.size() > limit; + List page = truncated ? new ArrayList<>(matches.subList(0, limit)) : matches; + String nextToken = truncated ? encodeAnnotationContinuationToken(page.get(page.size() - 1).getAnnotationName()) : null; + return new ListObjectAnnotationsResult(page, truncated, nextToken, limit); + } + } + + /** Returns the parent object's versionId (null in non-versioned buckets) for the response header. */ + public String deleteObjectAnnotation(String bucketName, String key, String annotationName, + String versionId, String ifMatch, boolean bypassGovernance) { + Bucket bucket = requireBucket(bucketName); + S3Object[] notificationTarget = {null}; + String parentVersionId; + synchronized (bucket) { + versionId = normalizeNullVersionId(versionId); + S3Object parent = resolveParentObject(bucketName, key, versionId); + // Deleting an annotation on a locked version follows DeleteObject's rules: governance + // retention needs x-amz-bypass-governance-retention, compliance and legal hold always block. + checkLockProtection(parent, bypassGovernance); + if (ifMatch != null && !eTagMatches(ifMatch, parent.getETag())) { + throw new S3PreconditionFailedException("If-Match"); + } + validateAnnotationName(annotationName); + String parentKey = parentStoreKey(bucketName, key, versionId, parent); + String storeKey = annotationStoreKey(parentKey, annotationName); + // Account-scoped existence check, like the objectStore delete path: a cross-account + // probe would report another account's annotation, which this delete must not remove. + ObjectAnnotation existing = annotationStore.get(storeKey).orElse(null); + if (existing == null) { + // Deleting a nonexistent annotation is not an error (idempotent), but the parent + // object still had to exist and pass its precondition check above. + return parent.getVersionId(); + } + annotationStore.delete(storeKey); + deleteAnnotationPayload(existing); + LOG.debugv("Deleted annotation {0} from object: {1}/{2}", annotationName, bucketName, key); + parentVersionId = parent.getVersionId(); + notificationTarget[0] = parent; + } + // Fired outside the bucket monitor, as in putObjectAnnotation. + fireNotifications(bucketName, key, "ObjectAnnotation:Delete", notificationTarget[0]); + return parentVersionId; + } + + /** + * ListObjectVersions reports pre-versioning objects with the literal VersionId {@code "null"}; + * a version-echoing client sends it back. Treat it as a request for the pre-versioning entry + * at the plain object key. + */ + private static String normalizeNullVersionId(String versionId) { + return "null".equals(versionId) ? null : versionId; + } + + /** Removes every annotation attached to one object version (metadata + payload). */ + private void deleteAllAnnotationsFor(String parentKey) { + for (ObjectAnnotation annotation : annotationStore.scan(k -> k.startsWith(parentKey + ANNOTATION_SEPARATOR))) { + annotationStore.delete(annotationStoreKey(parentKey, annotation.getAnnotationName())); + deleteAnnotationPayload(annotation); + } + } + + /** Removes every annotation in a bucket (metadata + payload); used by DeleteBucket. */ + private void deleteAllAnnotationsForBucket(String bucketName) { + for (ObjectAnnotation annotation : annotationStore.scan(k -> k.startsWith(bucketName + "/"))) { + String parentKey = parentKeyOf(annotation); + annotationStore.delete(annotationStoreKey(parentKey, annotation.getAnnotationName())); + deleteAnnotationPayload(annotation); + } + } + + private int countAnnotations(String parentKey) { + return (int) annotationStore.scan(k -> k.startsWith(parentKey + ANNOTATION_SEPARATOR)).size(); + } + + private String annotationStoreKey(String parentKey, String annotationName) { + return parentKey + ANNOTATION_SEPARATOR + annotationName; + } + + /** + * Resolves the annotation-store key of the object version an annotation request targets. + * An absent versionId means the current latest object: the latest entry's own versionId when + * the bucket is versioned, otherwise the plain object key. The delete-marker check lives in + * {@link #resolveParentObject}. + */ + private String parentStoreKey(String bucketName, String key, String versionId, S3Object parent) { + if (versionId != null) { + return annotationParentKey(bucketName, key, versionId); + } + return parent.getVersionId() != null + ? annotationParentKey(bucketName, key, parent.getVersionId()) + : annotationParentKey(bucketName, key, null); + } + + /** + * Builds the annotation identity for one object version. Unlike the objectStore key scheme, + * this must be injective: {@code '@'} and {@code "#v#"} mark the separators, and an S3 object + * key may contain any character, so the key is URL-encoded first. Encoded keys never contain + * '@', '#' or bare '%', so no other object's identity can forge these separators or extend + * another object's scan prefix. + */ + private String annotationParentKey(String bucketName, String key, String versionId) { + return bucketName + "/" + annotationIdentity(key, versionId); + } + + private String annotationIdentity(String key, String versionId) { + String encodedKey = URLEncoder.encode(key, StandardCharsets.UTF_8); + return versionId != null ? encodedKey + "#v#" + versionId : encodedKey; + } + + /** Resolves the object a subresource request targets; a delete-marker latest reads as absent. */ + private S3Object resolveParentObject(String bucketName, String key, String versionId) { + S3Object object = resolveObject(versionId != null + ? versionedKey(bucketName, key, versionId) + : objectKey(bucketName, key)) + .orElseThrow(() -> new AwsException("NoSuchKey", + "The specified key does not exist.", 404)); + if (object.isDeleteMarker()) { + throw new AwsException("NoSuchKey", "The specified key does not exist.", 404); + } + return object; + } + + private void validateAnnotationName(String annotationName) { + if (annotationName == null || annotationName.isBlank()) { + throw new AwsException("InvalidAnnotationName", + "The annotation name must not be empty or consist only of whitespace.", 400); + } + if (annotationName.getBytes(StandardCharsets.UTF_8).length > MAX_ANNOTATION_NAME_BYTES) { + throw new AwsException("AnnotationNameTooLong", + "The annotation name exceeds the maximum length of 512 bytes.", 400); + } + for (int i = 0; i < annotationName.length(); ) { + int codePoint = annotationName.codePointAt(i); + if (!isAllowedAnnotationNameCodePoint(codePoint)) { + throw new AwsException("InvalidAnnotationName", + "The annotation name contains invalid characters.", 400); + } + i += Character.charCount(codePoint); + } + String lowercased = annotationName.toLowerCase(Locale.ROOT); + if (lowercased.startsWith("aws") || lowercased.startsWith("s3")) { + throw new AwsException("InvalidAnnotationName", + "Annotation names must not start with 'aws' or 's3'.", 400); + } + } + + private static boolean isAllowedAnnotationNameCodePoint(int codePoint) { + return Character.isLetter(codePoint) || Character.isDigit(codePoint) + || codePoint == '_' || codePoint == '.' || codePoint == '-'; + } + + private void validateAnnotationPrefix(String annotationPrefix) { + if (annotationPrefix == null || annotationPrefix.isEmpty()) { + return; + } + for (int i = 0; i < annotationPrefix.length(); ) { + int codePoint = annotationPrefix.codePointAt(i); + if (!isAllowedAnnotationNameCodePoint(codePoint)) { + throw new AwsException("InvalidPrefix", + "The annotation prefix you provided is invalid.", 400); + } + i += Character.charCount(codePoint); + } + } + + private void validateAnnotationPayload(byte[] payload) { + if (payload == null || payload.length < 1) { + throw new AwsException("InvalidRequest", + "The annotation payload must be between 1 byte and 1 MiB in size.", 400); + } + if (payload.length > MAX_ANNOTATION_PAYLOAD_BYTES) { + throw new AwsException("InvalidRequest", + "The annotation payload exceeds the maximum size of 1 MiB.", 400); + } + if (!ObjectAnnotation.isValidUtf8(payload)) { + throw new AwsException("UnsupportedMediaType", + "The annotation payload is not valid UTF-8 encoded text.", 415); + } + } + + private String encodeAnnotationContinuationToken(String lastAnnotationName) { + return Base64.getUrlEncoder().withoutPadding() + .encodeToString(lastAnnotationName.getBytes(StandardCharsets.UTF_8)); + } + + private String decodeAnnotationContinuationToken(String token) { + if (token == null || token.isEmpty()) { + return null; + } + try { + return new String(Base64.getUrlDecoder().decode(token), StandardCharsets.UTF_8); + } catch (IllegalArgumentException e) { + throw new AwsException("InvalidArgument", "The continuation token you provided is invalid.", 400); + } + } + + // Annotation payload bytes live outside the annotation store, the same way object bodies + // live outside s3-objects.json: in memoryAnnotationStore in memory mode, .s3ann files on disk. + + private String physicalAnnotationKey(String parentKey, String annotationName) { + return ownerId() + "/" + annotationStoreKey(parentKey, annotationName); + } + + private Path resolveAnnotationPath(String bucketName, String key, String versionId, String annotationName) { + Path bucketDir = dataRoot.resolve(ACCOUNT_STORAGE_ROOT).resolve(ownerId()) + .resolve(ANNOTATION_STORAGE_ROOT).resolve(bucketName).normalize(); + // Both directory components are SHA-256 hex of our own injective identity (the object key + // is URL-encoded inside it), so the path is bounded in length, filesystem-safe, and + // collision-free across object keys that contain '#v#', '@', or path-like characters. + // Cleanup is metadata-driven, so the mapping never needs to be reversed. + // Every path component below bucketDir is SHA-256 hex, so no traversal is possible. + Path parentDir = bucketDir.resolve(sha256Hex(annotationIdentity(key, versionId))); + return parentDir.resolve(sha256Hex(annotationName) + ANNOTATION_DATA_SUFFIX); + } + + private static String sha256Hex(String value) { + try { + byte[] digest = MessageDigest.getInstance("SHA-256") + .digest(value.getBytes(StandardCharsets.UTF_8)); + StringBuilder sb = new StringBuilder(digest.length * 2); + for (byte b : digest) { + sb.append(String.format("%02x", b)); + } + return sb.toString(); + } catch (NoSuchAlgorithmException e) { + throw new IllegalStateException("SHA-256 algorithm is not available", e); + } + } + + private void writeAnnotationPayload(ObjectAnnotation annotation, byte[] payload) { + if (inMemory) { + memoryAnnotationStore.put(physicalAnnotationKey( + parentKeyOf(annotation), annotation.getAnnotationName()), payload); + return; + } + Path filePath = resolveAnnotationPath(annotation.getBucketName(), annotation.getKey(), + annotation.getVersionId(), annotation.getAnnotationName()); + ReentrantLock lock = diskFileLock(filePath); + lock.lock(); + try { + atomicWrite(filePath, payload); + } catch (IOException e) { + throw new UncheckedIOException("Failed to write S3 annotation payload file", e); + } finally { + lock.unlock(); + } + } + + /** Returns the payload bytes, or {@code null} when the payload is gone while its metadata survived. */ + private byte[] readAnnotationPayload(ObjectAnnotation annotation) { + if (inMemory) { + return memoryAnnotationStore.get(physicalAnnotationKey( + parentKeyOf(annotation), annotation.getAnnotationName())); + } + Path filePath = resolveAnnotationPath(annotation.getBucketName(), annotation.getKey(), + annotation.getVersionId(), annotation.getAnnotationName()); + // The same lock writeAnnotationPayload and deleteAnnotationPayload hold: without it a + // concurrent delete between the existence check and the read surfaces as an + // UncheckedIOException (HTTP 500) instead of the intended NoSuchAnnotation (404). + ReentrantLock lock = diskFileLock(filePath); + lock.lock(); + try { + if (!Files.exists(filePath)) { + return null; + } + return Files.readAllBytes(filePath); + } catch (NoSuchFileException e) { + return null; + } catch (IOException e) { + throw new UncheckedIOException("Failed to read S3 annotation payload file", e); + } finally { + lock.unlock(); + } + } + + private void deleteAnnotationPayload(ObjectAnnotation annotation) { + if (inMemory) { + memoryAnnotationStore.remove(physicalAnnotationKey( + parentKeyOf(annotation), annotation.getAnnotationName())); + return; + } + Path filePath = resolveAnnotationPath(annotation.getBucketName(), annotation.getKey(), + annotation.getVersionId(), annotation.getAnnotationName()); + ReentrantLock lock = diskFileLock(filePath); + lock.lock(); + try { + Files.deleteIfExists(filePath); + } catch (IOException e) { + LOG.errorv(e, "Failed to delete S3 annotation payload file for {0}/{1} / {2}", + annotation.getBucketName(), annotation.getKey(), annotation.getAnnotationName()); + } finally { + lock.unlock(); + } + } + + private String parentKeyOf(ObjectAnnotation annotation) { + return annotationParentKey(annotation.getBucketName(), annotation.getKey(), annotation.getVersionId()); + } + // --- Bucket Tagging --- public void putBucketTagging(String bucketName, Map tags) { @@ -3430,7 +3908,9 @@ private void fireNotifications(String bucketName, String key, String eventName, if (config.isEventBridgeEnabled() && eventBridgeService != null) { try { - String detailType = eventName.startsWith("ObjectCreated") ? "Object Created" : "Object Deleted"; + String detailType = eventName.startsWith("ObjectCreated") ? "Object Created" + : eventName.startsWith("ObjectAnnotation") ? "Object Annotation" + : "Object Deleted"; Map entry = new java.util.HashMap<>(); entry.put("Source", "aws.s3"); entry.put("DetailType", detailType); @@ -4095,8 +4575,69 @@ private S3Object copyS3Object(String sourceBucket, String sourceKey, effectiveChecksum = null; } - S3Object copy = storeObject(destBucket, destKey, source.getData(), effectiveContentType, metadata, - effectiveChecksum, null, + // Annotations travel with the copy by default (x-amz-annotation-directive COPY). They are + // snapshotted before storeObject: a self-copy (same bucket and key) or a pre-versioning + // overwrite deletes the shared annotation entries as part of the overwrite, so metadata + // and payload must be read beforehand. The snapshot holds payload bytes in memory, the + // same profile as the source body copy itself. + boolean copyAnnotations = !"EXCLUDE".equalsIgnoreCase(effectiveOptions.getAnnotationDirective()); + boolean selfCopy = sourceBucket.equals(destBucket); + Bucket sourceMonitor = requireBucket(sourceBucket); + List sourceAnnotations = List.of(); + if (copyAnnotations && !selfCopy) { + // Cross-bucket copy: the destination overwrite cannot touch the source's annotation + // entries, so a monitor-guarded snapshot is enough. Holding the source monitor across + // storeObject here would risk a deadlock with a concurrent reverse copy. + synchronized (sourceMonitor) { + sourceAnnotations = snapshotAnnotations(source); + } + } + + // A copy is written as one object, so it keeps the ETag storeObject computed (the MD5 of the + // whole content) instead of the source's, which for a multipart source ends in "-N". As on S3. + if (selfCopy) { + // The overwrite deletes the shared annotation entries, so snapshot, overwrite, and + // restore must be atomic against annotation writes: all three under the bucket + // monitor, which storeObject re-enters for the same bucket. + S3Object[] result = {null}; + synchronized (sourceMonitor) { + if (copyAnnotations) { + sourceAnnotations = snapshotAnnotations(source); + } + result[0] = storeObjectCopy(destBucket, destKey, source, metadata, effectiveChecksum, + effectiveContentType, effectiveStorageClass, effectiveContentEncoding, + effectiveContentDisposition, effectiveCacheControl, effectiveServerSideEncryption, + effectiveOptions, copyChecksumAlgorithm, effectiveTags); + if (copyAnnotations) { + restoreAnnotations(sourceAnnotations, destBucket, destKey, result[0]); + } + } + // Fired outside the bucket monitor, matching the cross-bucket path. + LOG.debugv("Copied object: {0}/{1} -> {2}/{3}", sourceBucket, sourceKey, destBucket, destKey); + fireNotifications(destBucket, destKey, "ObjectCreated:Copy", result[0]); + return result[0]; + } + S3Object copy = storeObjectCopy(destBucket, destKey, source, metadata, effectiveChecksum, + effectiveContentType, effectiveStorageClass, effectiveContentEncoding, + effectiveContentDisposition, effectiveCacheControl, effectiveServerSideEncryption, + effectiveOptions, copyChecksumAlgorithm, effectiveTags); + if (copyAnnotations) { + restoreAnnotations(sourceAnnotations, destBucket, destKey, copy); + } + LOG.debugv("Copied object: {0}/{1} -> {2}/{3}", sourceBucket, sourceKey, destBucket, destKey); + fireNotifications(destBucket, destKey, "ObjectCreated:Copy", copy); + return copy; + } + + private S3Object storeObjectCopy(String destBucket, String destKey, S3Object source, + Map metadata, S3Checksum effectiveChecksum, + String effectiveContentType, String effectiveStorageClass, + String effectiveContentEncoding, String effectiveContentDisposition, + String effectiveCacheControl, String effectiveServerSideEncryption, + CopyObjectOptions effectiveOptions, ChecksumAlgorithm copyChecksumAlgorithm, + Map effectiveTags) { + return storeObject(destBucket, destKey, source.getData(), effectiveContentType, + metadata, effectiveChecksum, null, new PutObjectOptions() .withStorageClass(effectiveStorageClass) .withContentEncoding(effectiveContentEncoding) @@ -4114,11 +4655,41 @@ private S3Object copyS3Object(String sourceBucket, String sourceKey, .withGrantWriteAcp(effectiveOptions.getGrantWriteAcp()) .withChecksumAlgorithm(copyChecksumAlgorithm != null ? copyChecksumAlgorithm.name() : null) .withTagging(effectiveTags)); - // A copy is written as one object, so it keeps the ETag storeObject computed (the MD5 of the - // whole content) instead of the source's, which for a multipart source ends in "-N". As on S3. - LOG.debugv("Copied object: {0}/{1} -> {2}/{3}", sourceBucket, sourceKey, destBucket, destKey); - fireNotifications(destBucket, destKey, "ObjectCreated:Copy", copy); - return copy; + } + + private record AnnotationSnapshot(ObjectAnnotation metadata, byte[] payload) {} + + private List snapshotAnnotations(S3Object source) { + String sourceParentKey = annotationParentKey(source.getBucketName(), source.getKey(), source.getVersionId()); + List snapshots = new ArrayList<>(); + for (ObjectAnnotation annotation : annotationStore.scan(k -> k.startsWith(sourceParentKey + ANNOTATION_SEPARATOR))) { + byte[] payload = readAnnotationPayload(annotation); + if (payload != null) { + snapshots.add(new AnnotationSnapshot(annotation, payload)); + } + } + return snapshots; + } + + private void restoreAnnotations(List snapshots, String destBucket, String destKey, + S3Object copy) { + if (copy.getSseCustomerAlgorithm() != null) { + // The destination copy is SSE-C encrypted: annotations cannot live on it, the same + // rule a direct PutObjectAnnotation enforces. + return; + } + String destParentKey = annotationParentKey(destBucket, destKey, copy.getVersionId()); + for (AnnotationSnapshot snapshot : snapshots) { + // The payload bytes are identical, so the source annotation's ETag and checksum are + // preserved; only the identity fields and lastModified are recomputed. + ObjectAnnotation copied = new ObjectAnnotation(destBucket, destKey, copy.getVersionId(), + snapshot.metadata().getAnnotationName(), snapshot.metadata().getSize(), + snapshot.metadata().getETag(), Instant.now(), + snapshot.metadata().getChecksumAlgorithm(), snapshot.metadata().getChecksumValue()); + copied.setServerSideEncryption(copy.getServerSideEncryption()); + writeAnnotationPayload(copied, snapshot.payload()); + annotationStore.put(annotationStoreKey(destParentKey, snapshot.metadata().getAnnotationName()), copied); + } } @Override diff --git a/src/main/java/io/github/hectorvent/floci/services/s3/model/CopyObjectOptions.java b/src/main/java/io/github/hectorvent/floci/services/s3/model/CopyObjectOptions.java index ca4dd314ff..a677a36d32 100644 --- a/src/main/java/io/github/hectorvent/floci/services/s3/model/CopyObjectOptions.java +++ b/src/main/java/io/github/hectorvent/floci/services/s3/model/CopyObjectOptions.java @@ -26,6 +26,8 @@ public class CopyObjectOptions { private String grantReadAcp; private String grantWriteAcp; private String checksumAlgorithm; + // Whether annotations travel with the copy: COPY (the default) or EXCLUDE. + private String annotationDirective; public String getMetadataDirective() { return metadataDirective; } public CopyObjectOptions withMetadataDirective(String metadataDirective) { this.metadataDirective = metadataDirective; return this; } @@ -95,4 +97,7 @@ public class CopyObjectOptions { public String getChecksumAlgorithm() { return checksumAlgorithm; } public CopyObjectOptions withChecksumAlgorithm(String checksumAlgorithm) { this.checksumAlgorithm = checksumAlgorithm; return this; } + + public String getAnnotationDirective() { return annotationDirective; } + public CopyObjectOptions withAnnotationDirective(String annotationDirective) { this.annotationDirective = annotationDirective; return this; } } diff --git a/src/main/java/io/github/hectorvent/floci/services/s3/model/ObjectAnnotation.java b/src/main/java/io/github/hectorvent/floci/services/s3/model/ObjectAnnotation.java new file mode 100644 index 0000000000..fb3b149c98 --- /dev/null +++ b/src/main/java/io/github/hectorvent/floci/services/s3/model/ObjectAnnotation.java @@ -0,0 +1,102 @@ +package io.github.hectorvent.floci.services.s3.model; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import io.quarkus.runtime.annotations.RegisterForReflection; + +import java.nio.charset.StandardCharsets; +import java.time.Instant; +import java.time.temporal.ChronoUnit; +import java.util.Locale; + +/** + * An S3 object annotation: a named payload (1 byte to 1 MiB of UTF-8 text) attached to a specific + * object version. Only the metadata lives here; the payload bytes are stored outside the + * annotation store, exactly the way {@code S3Object.data} is kept out of {@code s3-objects.json}. + */ +@RegisterForReflection +@JsonIgnoreProperties(ignoreUnknown = true) +public class ObjectAnnotation { + + private String bucketName; + private String key; + // Parent object's versionId; null for objects in non-versioned buckets. + private String versionId; + private String annotationName; + private long size; + private String eTag; + private Instant lastModified; + // Wire name (e.g. "CRC64NVME") and Base64 value, kept as plain strings rather than S3Checksum + // so the annotation algorithm set stays independent of the object checksum machinery. + private String checksumAlgorithm; + private String checksumValue; + // Reserved for annotation replication; always null until that feature exists. + private String replicationStatus; + // Inherited from the parent object at write time; annotations cannot use SSE-C. + private String serverSideEncryption; + + public ObjectAnnotation() { + } + + public ObjectAnnotation(String bucketName, String key, String versionId, String annotationName, + long size, String eTag, Instant lastModified, + String checksumAlgorithm, String checksumValue) { + this.bucketName = bucketName; + this.key = key; + this.versionId = versionId; + this.annotationName = annotationName; + this.size = size; + this.eTag = eTag; + this.lastModified = lastModified != null ? lastModified.truncatedTo(ChronoUnit.MILLIS) : null; + this.checksumAlgorithm = checksumAlgorithm; + this.checksumValue = checksumValue; + } + + public String getBucketName() { return bucketName; } + public void setBucketName(String bucketName) { this.bucketName = bucketName; } + + public String getKey() { return key; } + public void setKey(String key) { this.key = key; } + + public String getVersionId() { return versionId; } + public void setVersionId(String versionId) { this.versionId = versionId; } + + public String getAnnotationName() { return annotationName; } + public void setAnnotationName(String annotationName) { this.annotationName = annotationName; } + + public long getSize() { return size; } + public void setSize(long size) { this.size = size; } + + public String getETag() { return eTag; } + public void setETag(String eTag) { this.eTag = eTag; } + + public Instant getLastModified() { return lastModified; } + public void setLastModified(Instant lastModified) { this.lastModified = lastModified; } + + public String getChecksumAlgorithm() { return checksumAlgorithm; } + public void setChecksumAlgorithm(String checksumAlgorithm) { this.checksumAlgorithm = checksumAlgorithm; } + + public String getChecksumValue() { return checksumValue; } + public void setChecksumValue(String checksumValue) { this.checksumValue = checksumValue; } + + public String getReplicationStatus() { return replicationStatus; } + public void setReplicationStatus(String replicationStatus) { this.replicationStatus = replicationStatus; } + + public String getServerSideEncryption() { return serverSideEncryption; } + public void setServerSideEncryption(String serverSideEncryption) { this.serverSideEncryption = serverSideEncryption; } + + public static String checksumHeaderName(String algorithm) { + return "x-amz-checksum-" + algorithm.toLowerCase(Locale.ROOT); + } + + public static boolean isValidUtf8(byte[] data) { + try { + StandardCharsets.UTF_8.newDecoder() + .onMalformedInput(java.nio.charset.CodingErrorAction.REPORT) + .onUnmappableCharacter(java.nio.charset.CodingErrorAction.REPORT) + .decode(java.nio.ByteBuffer.wrap(data)); + return true; + } catch (Exception e) { + return false; + } + } +} \ No newline at end of file diff --git a/src/main/java/io/github/hectorvent/floci/services/s3/model/S3Object.java b/src/main/java/io/github/hectorvent/floci/services/s3/model/S3Object.java index d2f1e0fa21..280c5b027e 100644 --- a/src/main/java/io/github/hectorvent/floci/services/s3/model/S3Object.java +++ b/src/main/java/io/github/hectorvent/floci/services/s3/model/S3Object.java @@ -154,7 +154,7 @@ public S3Object(String bucketName, String key, byte[] data, String contentType) public String getDataGeneration() { return dataGeneration; } public void setDataGeneration(String dataGeneration) { this.dataGeneration = dataGeneration; } - private static String computeETag(byte[] data) { + public static String computeETag(byte[] data) { try { var md = java.security.MessageDigest.getInstance("MD5"); byte[] digest = md.digest(data); diff --git a/src/test/java/io/github/hectorvent/floci/services/s3/S3AnnotationsIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/s3/S3AnnotationsIntegrationTest.java new file mode 100644 index 0000000000..80a89f2ca9 --- /dev/null +++ b/src/test/java/io/github/hectorvent/floci/services/s3/S3AnnotationsIntegrationTest.java @@ -0,0 +1,360 @@ +package io.github.hectorvent.floci.services.s3; + +import io.quarkus.test.junit.QuarkusTest; +import org.junit.jupiter.api.MethodOrderer; +import org.junit.jupiter.api.Order; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestMethodOrder; + +import java.nio.charset.StandardCharsets; + +import static io.restassured.RestAssured.given; +import static org.hamcrest.Matchers.*; + +/** + * Wire-level integration tests for the S3 Object Annotations subresource (?annotation): + * PutObjectAnnotation, GetObjectAnnotation, ListObjectAnnotations, DeleteObjectAnnotation. + */ +@QuarkusTest +@TestMethodOrder(MethodOrderer.OrderAnnotation.class) +class S3AnnotationsIntegrationTest { + + private static final String BUCKET = "annotations-it-bucket"; + + @Test + @Order(1) + void createBucket() { + given() + .when() + .put("/" + BUCKET) + .then() + .statusCode(200); + } + + @Test + @Order(2) + void createTargetObject() { + given() + .contentType("text/plain") + .body("annotation target body") + .when() + .put("/" + BUCKET + "/docs/report.txt") + .then() + .statusCode(200) + .header("ETag", notNullValue()); + } + + @Test + @Order(3) + void putObjectAnnotationReturnsOutputXmlAndETag() { + given() + .body("{\"classification\": \"report\"}") + .when() + .put("/" + BUCKET + "/docs/report.txt?annotation&annotationName=labels") + .then() + .statusCode(200) + .header("ETag", notNullValue()) + .header("x-amz-checksum-crc64nvme", notNullValue()) + .header("x-amz-checksum-type", equalTo("FULL_OBJECT")) + .body(containsString("docs/report.txt")) + .body(containsString("labels")); + } + + @Test + @Order(4) + void getObjectAnnotationReturnsPayload() { + given() + .when() + .get("/" + BUCKET + "/docs/report.txt?annotation&annotationName=labels") + .then() + .statusCode(200) + .body(equalTo("{\"classification\": \"report\"}")) + .header("ETag", notNullValue()) + .header("Last-Modified", notNullValue()); + } + + @Test + @Order(5) + void getObjectAnnotationWithChecksumModeReturnsChecksumHeaders() { + given() + .header("x-amz-checksum-mode", "ENABLED") + .when() + .get("/" + BUCKET + "/docs/report.txt?annotation&annotationName=labels") + .then() + .statusCode(200) + .header("x-amz-checksum-crc64nvme", notNullValue()) + .header("x-amz-checksum-type", equalTo("FULL_OBJECT")); + } + + @Test + @Order(6) + void getObjectAnnotationWithoutChecksumModeOmitsChecksumHeaders() { + given() + .when() + .get("/" + BUCKET + "/docs/report.txt?annotation&annotationName=labels") + .then() + .statusCode(200) + .header("x-amz-checksum-crc64nvme", nullValue()); + } + + @Test + @Order(7) + void listObjectAnnotationsReturnsEntries() { + given() + .body("second") + .when() + .put("/" + BUCKET + "/docs/report.txt?annotation&annotationName=summary") + .then() + .statusCode(200); + given() + .when() + .get("/" + BUCKET + "/docs/report.txt?annotation") + .then() + .statusCode(200) + .body(containsString("" + BUCKET + "")) + .body(containsString("docs/report.txt")) + .body(containsString("")) + .body(containsString("labels")) + .body(containsString("summary")) + .body(containsString("6")) + .body(containsString("2")) + .body(containsString("1000")) + .body(containsString("false")) + .body(not(containsString(""))); + } + + @Test + @Order(7) + void headObjectAnnotationMatchesGetStatus() { + given() + .when() + .head("/" + BUCKET + "/docs/report.txt?annotation&annotationName=labels") + .then() + .statusCode(200) + .header("ETag", notNullValue()); + given() + .when() + .head("/" + BUCKET + "/docs/report.txt?annotation&annotationName=missing") + .then() + .statusCode(404); + } + + @Test + @Order(8) + void getMissingAnnotationReturnsNoSuchAnnotation() { + given() + .when() + .get("/" + BUCKET + "/docs/report.txt?annotation&annotationName=missing") + .then() + .statusCode(404) + .body(containsString("NoSuchAnnotation")); + } + + @Test + @Order(9) + void getAnnotationOnMissingObjectReturnsNoSuchKey() { + given() + .when() + .get("/" + BUCKET + "/no-such-key.txt?annotation&annotationName=labels") + .then() + .statusCode(404) + .body(containsString("NoSuchKey")); + } + + @Test + @Order(10) + void putAnnotationWithInvalidNameIsRejected() { + given() + .body("x") + .when() + .put("/" + BUCKET + "/docs/report.txt?annotation&annotationName=s3-reserved") + .then() + .statusCode(400) + .body(containsString("InvalidAnnotationName")); + } + + @Test + @Order(11) + void putAnnotationWithOversizedPayloadIsRejected() { + byte[] tooBig = new byte[1_048_577]; + given() + .body(tooBig) + .when() + .put("/" + BUCKET + "/docs/report.txt?annotation&annotationName=big") + .then() + .statusCode(400) + .body(containsString("InvalidRequest")); + } + + @Test + @Order(12) + void deleteObjectAnnotationReturns204() { + given() + .when() + .delete("/" + BUCKET + "/docs/report.txt?annotation&annotationName=summary") + .then() + .statusCode(204); + // Deleting a nonexistent annotation is idempotent. + given() + .when() + .delete("/" + BUCKET + "/docs/report.txt?annotation&annotationName=summary") + .then() + .statusCode(204); + given() + .when() + .get("/" + BUCKET + "/docs/report.txt?annotation&annotationName=summary") + .then() + .statusCode(404); + } + + @Test + @Order(13) + void emptyAnnotationNameDispatchesToList() { + given() + .when() + .get("/" + BUCKET + "/docs/report.txt?annotation&annotationName=") + .then() + .statusCode(200) + .body(containsString("Enabled") + .when() + .put("/" + BUCKET + "?versioning") + .then() + .statusCode(200); + } + + @Test + @Order(21) + void putVersionedObjects() { + given() + .contentType("text/plain") + .body("version one") + .when() + .put("/" + BUCKET + "/docs/versioned.txt") + .then() + .statusCode(200) + .header("x-amz-version-id", notNullValue()); + given() + .contentType("text/plain") + .body("version two") + .when() + .put("/" + BUCKET + "/docs/versioned.txt") + .then() + .statusCode(200) + .header("x-amz-version-id", notNullValue()); + } + + @Test + @Order(22) + void annotationOnLatestVersionHasNoInheritanceFromPrevious() { + given() + .body("latest annotation") + .when() + .put("/" + BUCKET + "/docs/versioned.txt?annotation&annotationName=state") + .then() + .statusCode(200) + .header("x-amz-object-version-id", notNullValue()); + } + + @Test + @Order(23) + void listOnLatestReturnsSingleAnnotation() { + given() + .when() + .get("/" + BUCKET + "/docs/versioned.txt?annotation") + .then() + .statusCode(200) + .body(containsString("state")) + .body(containsString("1")) + .header("x-amz-object-version-id", notNullValue()); + } + + @Test + @Order(24) + void deleteMarkerHidesAnnotationsOnLatest() { + given() + .when() + .delete("/" + BUCKET + "/docs/versioned.txt") + .then() + .statusCode(204) + .header("x-amz-delete-marker", equalTo("true")); + given() + .when() + .get("/" + BUCKET + "/docs/versioned.txt?annotation") + .then() + .statusCode(404) + .body(containsString("NoSuchKey")); + } + + @Test + @Order(25) + void putAnnotationWithChecksumAlgorithmIsStoredPerAnnotation() { + // Recreate the object after the delete marker. + String versionId = given() + .contentType("text/plain") + .body("version three") + .when() + .put("/" + BUCKET + "/docs/versioned.txt") + .then() + .statusCode(200) + .header("x-amz-version-id", notNullValue()) + .extract() + .header("x-amz-version-id"); + given() + .header("x-amz-sdk-checksum-algorithm", "SHA256") + .body("checksummed annotation") + .when() + .put("/" + BUCKET + "/docs/versioned.txt?annotation&annotationName=hashed") + .then() + .statusCode(200) + .header("x-amz-checksum-sha256", notNullValue()) + .header("x-amz-checksum-type", equalTo("FULL_OBJECT")) + .header("x-amz-object-version-id", equalTo(versionId)); + given() + .header("x-amz-checksum-mode", "ENABLED") + .when() + .get("/" + BUCKET + "/docs/versioned.txt?annotation&annotationName=hashed") + .then() + .statusCode(200) + .body(equalTo("checksummed annotation")) + .header("x-amz-checksum-sha256", notNullValue()); + // The stored annotation reports the SHA256 algorithm in the list output. + given() + .when() + .get("/" + BUCKET + "/docs/versioned.txt?annotation&annotation-prefix=hash") + .then() + .statusCode(200) + .body(containsString("SHA256")); + } + + @Test + @Order(26) + void unicodePayloadRoundTrips() { + given() + .queryParam("annotation") + .queryParam("annotationName", "résumé") + .body("données annotées".getBytes(StandardCharsets.UTF_8)) + .when() + .put("/" + BUCKET + "/docs/report.txt") + .then() + .statusCode(200); + given() + .queryParam("annotation") + .queryParam("annotationName", "résumé") + .when() + .get("/" + BUCKET + "/docs/report.txt") + .then() + .statusCode(200) + .body(equalTo("données annotées")); + } +} \ No newline at end of file diff --git a/src/test/java/io/github/hectorvent/floci/services/s3/S3ServiceAnnotationsTest.java b/src/test/java/io/github/hectorvent/floci/services/s3/S3ServiceAnnotationsTest.java new file mode 100644 index 0000000000..9ca42bef53 --- /dev/null +++ b/src/test/java/io/github/hectorvent/floci/services/s3/S3ServiceAnnotationsTest.java @@ -0,0 +1,548 @@ +package io.github.hectorvent.floci.services.s3; + +import io.github.hectorvent.floci.core.common.AwsException; +import io.github.hectorvent.floci.core.storage.InMemoryStorage; +import io.github.hectorvent.floci.services.s3.model.ChecksumAlgorithm; +import io.github.hectorvent.floci.services.s3.model.ObjectAnnotation; +import io.github.hectorvent.floci.services.s3.model.S3Object; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.*; + +class S3ServiceAnnotationsTest { + + private static final String BUCKET = "annotations-bucket"; + + @TempDir + Path tempDir; + + private S3Service s3Service; + + @BeforeEach + void setUp() { + s3Service = new S3Service(new InMemoryStorage<>(), new InMemoryStorage<>(), + tempDir.resolve("s3"), false); + s3Service.createBucket(BUCKET, "us-east-1"); + s3Service.putObject(BUCKET, "docs/readme.txt", + "object body".getBytes(StandardCharsets.UTF_8), "text/plain", null, null); + } + + private ObjectAnnotation putAnnotation(String name, String payload) { + return s3Service.putObjectAnnotation(BUCKET, "docs/readme.txt", name, null, + payload.getBytes(StandardCharsets.UTF_8), null, null); + } + + private String readPayload(String name) { + ObjectAnnotation annotation = s3Service.getObjectAnnotation(BUCKET, "docs/readme.txt", name, null); + return new String(s3Service.readObjectAnnotationPayload(annotation), StandardCharsets.UTF_8); + } + + // ========== Validation ========== + + @Test + void putAndGetRoundTripPreservesPayloadAndMetadata() { + ObjectAnnotation annotation = putAnnotation("classification", "{\"label\": \"doc\"}"); + assertEquals("classification", annotation.getAnnotationName()); + assertEquals(16, annotation.getSize()); + assertEquals("{\"label\": \"doc\"}", readPayload("classification")); + assertEquals("CRC64NVME", annotation.getChecksumAlgorithm()); + assertNotNull(annotation.getChecksumValue()); + assertTrue(annotation.getETag().startsWith("\"") && annotation.getETag().endsWith("\"")); + assertNotNull(annotation.getLastModified()); + } + + @Test + void blankNameIsRejected() { + assertThrowsAws("InvalidAnnotationName", () -> putAnnotation(" ", "x")); + assertThrowsAws("InvalidAnnotationName", () -> + s3Service.putObjectAnnotation(BUCKET, "docs/readme.txt", null, null, + new byte[]{1}, null, null)); + } + + @Test + void nameLongerThan512BytesIsRejected() { + assertThrowsAws("AnnotationNameTooLong", () -> putAnnotation("a".repeat(513), "x")); + // Exactly 512 bytes is accepted. + putAnnotation("a".repeat(512), "x"); + } + + @Test + void invalidNameCharactersAreRejected() { + assertThrowsAws("InvalidAnnotationName", () -> putAnnotation("bad/name", "x")); + assertThrowsAws("InvalidAnnotationName", () -> putAnnotation("bad name", "x")); + assertThrowsAws("InvalidAnnotationName", () -> putAnnotation("bad@name", "x")); + } + + @Test + void reservedPrefixesAreRejectedCaseInsensitively() { + assertThrowsAws("InvalidAnnotationName", () -> putAnnotation("aws-thing", "x")); + assertThrowsAws("InvalidAnnotationName", () -> putAnnotation("AWS-thing", "x")); + assertThrowsAws("InvalidAnnotationName", () -> putAnnotation("s3-result", "x")); + assertThrowsAws("InvalidAnnotationName", () -> putAnnotation("S3-result", "x")); + } + + @Test + void unicodeNameIsAccepted() { + putAnnotation("分类", "x"); + assertEquals("x", readPayload("分类")); + } + + @Test + void emptyPayloadIsRejected() { + assertThrowsAws("InvalidRequest", () -> + s3Service.putObjectAnnotation(BUCKET, "docs/readme.txt", "name", null, + new byte[0], null, null)); + assertThrowsAws("InvalidRequest", () -> + s3Service.putObjectAnnotation(BUCKET, "docs/readme.txt", "name", null, + null, null, null)); + } + + @Test + void oversizedPayloadIsRejected() { + byte[] tooBig = new byte[1_048_577]; + assertThrowsAws("InvalidRequest", () -> + s3Service.putObjectAnnotation(BUCKET, "docs/readme.txt", "name", null, tooBig, null, null)); + // Exactly 1 MiB is accepted. + byte[] maxPayload = new byte[1_048_576]; + s3Service.putObjectAnnotation(BUCKET, "docs/readme.txt", "name", null, maxPayload, null, null); + } + + @Test + void nonUtf8PayloadIsRejectedWithUnsupportedMediaType() { + byte[] invalidUtf8 = new byte[]{(byte) 0xff, (byte) 0xfe, (byte) 0xfd}; + assertThrowsAws("UnsupportedMediaType", () -> + s3Service.putObjectAnnotation(BUCKET, "docs/readme.txt", "name", null, invalidUtf8, null, null)); + } + + @Test + void thousandAnnotationLimitIsEnforcedButUpdateDoesNotCount() { + for (int i = 0; i < 1000; i++) { + putAnnotation("ann-" + i, "v"); + } + assertThrowsAws("AnnotationLimitExceeded", () -> putAnnotation("ann-new", "v")); + // Updating an existing name is an update, not a new entry. + putAnnotation("ann-500", "updated"); + assertEquals("updated", readPayload("ann-500")); + } + + @Test + void missingBucketOrObjectYieldsAwsErrors() { + assertThrowsAws("NoSuchBucket", () -> + s3Service.putObjectAnnotation("no-such-bucket", "k", "name", null, + new byte[]{1}, null, null)); + assertThrowsAws("NoSuchKey", () -> + s3Service.getObjectAnnotation(BUCKET, "missing-key", "name", null)); + assertThrowsAws("NoSuchAnnotation", () -> + s3Service.getObjectAnnotation(BUCKET, "docs/readme.txt", "missing-ann", null)); + } + + @Test + void objectIfMatchValidatesParentETag() { + S3Object object = s3Service.getObject(BUCKET, "docs/readme.txt", null); + putAnnotation("guarded", "v1"); + // Matching the parent ETag succeeds (idempotent update). + s3Service.putObjectAnnotation(BUCKET, "docs/readme.txt", "guarded", + null, "v2".getBytes(StandardCharsets.UTF_8), object.getETag(), null); + assertEquals("v2", readPayload("guarded")); + // A wrong ETag is a 412. + assertThrows(S3PreconditionFailedException.class, () -> + s3Service.putObjectAnnotation(BUCKET, "docs/readme.txt", "guarded", + null, "v3".getBytes(StandardCharsets.UTF_8), "\"beef\"", null)); + assertThrows(S3PreconditionFailedException.class, () -> + s3Service.deleteObjectAnnotation(BUCKET, "docs/readme.txt", "guarded", null, "\"beef\"", false)); + } + + @Test + void annotationPutDoesNotChangeParentETagOrLastModified() { + S3Object before = s3Service.getObject(BUCKET, "docs/readme.txt", null); + putAnnotation("meta-neutral", "payload"); + S3Object after = s3Service.getObject(BUCKET, "docs/readme.txt", null); + assertEquals(before.getETag(), after.getETag()); + assertEquals(before.getLastModified(), after.getLastModified()); + } + + // ========== Delete semantics ========== + + @Test + void deleteAnnotationIsIdempotentForMissingAnnotation() { + assertDoesNotThrow(() -> + s3Service.deleteObjectAnnotation(BUCKET, "docs/readme.txt", "never-existed", null, null, false)); + } + + @Test + void deleteAnnotationRemovesMetadataAndPayload() { + putAnnotation("doomed", "payload"); + s3Service.deleteObjectAnnotation(BUCKET, "docs/readme.txt", "doomed", null, null, false); + assertThrowsAws("NoSuchAnnotation", () -> readPayload("doomed")); + } + + // ========== Versioning ========== + + @Test + void newVersionDoesNotInheritAnnotationsButOldVersionKeepsItsOwn() { + s3Service.putBucketVersioning(BUCKET, "Enabled"); + s3Service.putObject(BUCKET, "docs/readme.txt", + "body v1".getBytes(StandardCharsets.UTF_8), "text/plain", null, null); + putAnnotation("v1-only", "first"); + String versionId = s3Service.getObject(BUCKET, "docs/readme.txt", null).getVersionId(); + s3Service.putObject(BUCKET, "docs/readme.txt", + "body v2".getBytes(StandardCharsets.UTF_8), "text/plain", null, null); + + // The new version has no annotations. + assertThrowsAws("NoSuchAnnotation", () -> + s3Service.getObjectAnnotation(BUCKET, "docs/readme.txt", "v1-only", null)); + // The old version's annotations remain reachable by versionId. + ObjectAnnotation annotation = s3Service.getObjectAnnotation(BUCKET, "docs/readme.txt", + "v1-only", versionId); + assertEquals("first", new String( + s3Service.readObjectAnnotationPayload(annotation), StandardCharsets.UTF_8)); + assertEquals(versionId, annotation.getVersionId()); + } + + @Test + void deleteMarkerPreservesAnnotations() { + s3Service.putBucketVersioning(BUCKET, "Enabled"); + s3Service.putObject(BUCKET, "docs/readme.txt", + "body v1".getBytes(StandardCharsets.UTF_8), "text/plain", null, null); + putAnnotation("under-marker", "kept"); + String versionId = s3Service.getObject(BUCKET, "docs/readme.txt", null).getVersionId(); + s3Service.deleteObject(BUCKET, "docs/readme.txt"); + assertThrowsAws("NoSuchKey", () -> + s3Service.getObjectAnnotation(BUCKET, "docs/readme.txt", "under-marker", null)); + ObjectAnnotation annotation = s3Service.getObjectAnnotation(BUCKET, "docs/readme.txt", + "under-marker", versionId); + assertEquals("kept", new String( + s3Service.readObjectAnnotationPayload(annotation), StandardCharsets.UTF_8)); + } + + @Test + void deletingASpecificVersionDeletesItsAnnotations() { + s3Service.putBucketVersioning(BUCKET, "Enabled"); + s3Service.putObject(BUCKET, "docs/readme.txt", + "body v1".getBytes(StandardCharsets.UTF_8), "text/plain", null, null); + putAnnotation("version-bound", "payload"); + String versionId = s3Service.getObject(BUCKET, "docs/readme.txt", null).getVersionId(); + s3Service.deleteObject(BUCKET, "docs/readme.txt", versionId); + assertThrowsAws("NoSuchKey", () -> + s3Service.getObjectAnnotation(BUCKET, "docs/readme.txt", "version-bound", versionId)); + } + + @Test + void nonVersionedOverwriteDropsAnnotations() { + putAnnotation("before-overwrite", "payload"); + s3Service.putObject(BUCKET, "docs/readme.txt", + "new body".getBytes(StandardCharsets.UTF_8), "text/plain", null, null); + assertThrowsAws("NoSuchAnnotation", () -> + s3Service.getObjectAnnotation(BUCKET, "docs/readme.txt", "before-overwrite", null)); + } + + @Test + void nonVersionedDeleteDropsAnnotations() { + putAnnotation("before-delete", "payload"); + s3Service.deleteObject(BUCKET, "docs/readme.txt"); + assertThrowsAws("NoSuchKey", () -> + s3Service.getObjectAnnotation(BUCKET, "docs/readme.txt", "before-delete", null)); + } + + // ========== List ========== + + @Test + void listReturnsSortedAnnotationsWithCount() { + putAnnotation("b-second", "1"); + putAnnotation("a-first", "2"); + S3Service.ListObjectAnnotationsResult result = + s3Service.listObjectAnnotations(BUCKET, "docs/readme.txt", null, null, null, null); + assertEquals(2, result.annotations().size()); + assertEquals("a-first", result.annotations().get(0).getAnnotationName()); + assertEquals("b-second", result.annotations().get(1).getAnnotationName()); + assertFalse(result.isTruncated()); + assertNull(result.nextContinuationToken()); + } + + @Test + void listPaginatesWithContinuationToken() { + for (int i = 0; i < 5; i++) { + putAnnotation("ann-" + i, String.valueOf(i)); + } + S3Service.ListObjectAnnotationsResult page1 = + s3Service.listObjectAnnotations(BUCKET, "docs/readme.txt", null, 2, null, null); + assertEquals(2, page1.annotations().size()); + assertTrue(page1.isTruncated()); + assertEquals("ann-0", page1.annotations().get(0).getAnnotationName()); + assertEquals("ann-1", page1.annotations().get(1).getAnnotationName()); + assertNotNull(page1.nextContinuationToken()); + + S3Service.ListObjectAnnotationsResult page2 = + s3Service.listObjectAnnotations(BUCKET, "docs/readme.txt", null, 2, + page1.nextContinuationToken(), null); + assertEquals(2, page2.annotations().size()); + assertEquals("ann-2", page2.annotations().get(0).getAnnotationName()); + assertEquals("ann-3", page2.annotations().get(1).getAnnotationName()); + + S3Service.ListObjectAnnotationsResult page3 = + s3Service.listObjectAnnotations(BUCKET, "docs/readme.txt", null, 2, + page2.nextContinuationToken(), null); + assertEquals(1, page3.annotations().size()); + assertFalse(page3.isTruncated()); + } + + @Test + void listFiltersByPrefix() { + putAnnotation("label-a", "1"); + putAnnotation("label-b", "2"); + putAnnotation("other", "3"); + S3Service.ListObjectAnnotationsResult result = + s3Service.listObjectAnnotations(BUCKET, "docs/readme.txt", "label-", null, null, null); + assertEquals(2, result.annotations().size()); + assertEquals("label-a", result.annotations().get(0).getAnnotationName()); + assertEquals("label-b", result.annotations().get(1).getAnnotationName()); + } + + @Test + void listRejectsInvalidPrefixLimitAndToken() { + assertThrowsAws("InvalidPrefix", () -> + s3Service.listObjectAnnotations(BUCKET, "docs/readme.txt", "bad!", null, null, null)); + assertThrowsAws("InvalidArgument", () -> + s3Service.listObjectAnnotations(BUCKET, "docs/readme.txt", null, 0, null, null)); + assertThrowsAws("InvalidArgument", () -> + s3Service.listObjectAnnotations(BUCKET, "docs/readme.txt", null, 1001, null, null)); + assertThrowsAws("InvalidArgument", () -> + s3Service.listObjectAnnotations(BUCKET, "docs/readme.txt", null, null, "not-base64!", null)); + } + + // ========== Checksums ========== + + @Test + void annotationChecksumAlgorithmDefaultsToCrc64NvmeAndHonorsRequested() { + ObjectAnnotation defaultChecksum = putAnnotation("defaulted", "payload"); + assertEquals("CRC64NVME", defaultChecksum.getChecksumAlgorithm()); + assertEquals(ChecksumAlgorithm.CRC64NVME.compute("payload".getBytes(StandardCharsets.UTF_8)), + defaultChecksum.getChecksumValue()); + + ObjectAnnotation sha256 = s3Service.putObjectAnnotation(BUCKET, "docs/readme.txt", "sha256-ed", + null, "payload".getBytes(StandardCharsets.UTF_8), null, ChecksumAlgorithm.SHA256); + assertEquals("SHA256", sha256.getChecksumAlgorithm()); + assertEquals(ChecksumAlgorithm.SHA256.compute("payload".getBytes(StandardCharsets.UTF_8)), + sha256.getChecksumValue()); + } + + @Test + void sha512IsRejectedForAnnotationsAndObjects() { + assertThrowsAws("InvalidRequest", () -> + s3Service.putObjectAnnotation(BUCKET, "docs/readme.txt", "sha512-ed", null, + "payload".getBytes(StandardCharsets.UTF_8), null, + ChecksumAlgorithm.fromWireValue("SHA512"))); + assertThrowsAws("InvalidRequest", () -> ChecksumAlgorithm.fromWireValue("SHA512")); + } + + // ========== CopyObject ========== + + @Test + void copyObjectCopiesAnnotationsByDefault() { + putAnnotation("copied", "payload"); + s3Service.copyObject(BUCKET, "docs/readme.txt", BUCKET, "docs/copy.txt", + new io.github.hectorvent.floci.services.s3.model.CopyObjectOptions()); + ObjectAnnotation annotation = s3Service.getObjectAnnotation(BUCKET, "docs/copy.txt", "copied", null); + assertEquals("payload", new String( + s3Service.readObjectAnnotationPayload(annotation), StandardCharsets.UTF_8)); + assertEquals("payload", new String(s3Service.readObjectAnnotationPayload( + s3Service.getObjectAnnotation(BUCKET, "docs/readme.txt", "copied", null)), + StandardCharsets.UTF_8)); + } + + @Test + void copyObjectWithExcludeDirectiveSkipsAnnotations() { + putAnnotation("excluded", "payload"); + s3Service.copyObject(BUCKET, "docs/readme.txt", BUCKET, "docs/copy2.txt", + new io.github.hectorvent.floci.services.s3.model.CopyObjectOptions() + .withAnnotationDirective("EXCLUDE")); + assertThrowsAws("NoSuchAnnotation", () -> + s3Service.getObjectAnnotation(BUCKET, "docs/copy2.txt", "excluded", null)); + } + + // ========== Review regressions ========== + + @Test + void annotationKeysDoNotCollideAcrossObjectKeysContainingSeparators() { + s3Service.putObject(BUCKET, "weird@ann@key", "body".getBytes(StandardCharsets.UTF_8), null, null, null); + s3Service.putObjectAnnotation(BUCKET, "weird@ann@key", "note", null, + "other object".getBytes(StandardCharsets.UTF_8), null, null); + putAnnotation("plain", "this object"); + putAnnotation("also-plain", "also here"); + + // Object "docs/readme.txt" must not see the other object's annotations. + S3Service.ListObjectAnnotationsResult result = + s3Service.listObjectAnnotations(BUCKET, "docs/readme.txt", null, null, null, null); + assertEquals(2, result.annotations().size()); + assertEquals("also-plain", result.annotations().get(0).getAnnotationName()); + assertEquals("plain", result.annotations().get(1).getAnnotationName()); + + // The other object's annotations are independent and unaffected. + ObjectAnnotation annotation = s3Service.getObjectAnnotation(BUCKET, "weird@ann@key", "note", null); + assertEquals("other object", new String( + s3Service.readObjectAnnotationPayload(annotation), StandardCharsets.UTF_8)); + + // Deleting one object's annotations leaves the other object's intact. + s3Service.deleteObjectAnnotation(BUCKET, "docs/readme.txt", "plain", null, null, false); + assertEquals("other object", new String(s3Service.readObjectAnnotationPayload( + s3Service.getObjectAnnotation(BUCKET, "weird@ann@key", "note", null)), + StandardCharsets.UTF_8)); + } + + @Test + void annotationKeysDoNotCollideAcrossKeysContainingVMarker() { + s3Service.putBucketVersioning(BUCKET, "Enabled"); + s3Service.putObject(BUCKET, "docs/readme.txt", "v1".getBytes(StandardCharsets.UTF_8), null, null, null); + String versionId = s3Service.getObject(BUCKET, "docs/readme.txt", null).getVersionId(); + putAnnotation("note", "on version"); + + // A different object whose key embeds the same "#v#" text must not see the annotation. + s3Service.putObject(BUCKET, "docs/readme.txt#v#" + versionId, + "tricky".getBytes(StandardCharsets.UTF_8), null, null, null); + S3Service.ListObjectAnnotationsResult result = s3Service.listObjectAnnotations( + BUCKET, "docs/readme.txt#" + "v#" + versionId, null, null, null, null); + assertEquals(0, result.annotations().size()); + assertEquals(1, s3Service.listObjectAnnotations( + BUCKET, "docs/readme.txt", null, null, null, versionId).annotations().size()); + } + + @Test + void selfCopyPreservesAnnotations() { + putAnnotation("kept-through-copy", "payload"); + s3Service.copyObject(BUCKET, "docs/readme.txt", BUCKET, "docs/readme.txt", + new io.github.hectorvent.floci.services.s3.model.CopyObjectOptions()); + assertEquals("payload", readPayload("kept-through-copy")); + ObjectAnnotation annotation = s3Service.getObjectAnnotation(BUCKET, "docs/readme.txt", + "kept-through-copy", null); + assertEquals("payload", new String( + s3Service.readObjectAnnotationPayload(annotation), StandardCharsets.UTF_8)); + } + + @Test + void deleteMarkerOnPreVersioningObjectRemovesItsAnnotations() throws Exception { + putAnnotation("pre-versioning", "payload"); + s3Service.putBucketVersioning(BUCKET, "Enabled"); + s3Service.deleteObject(BUCKET, "docs/readme.txt"); + + // The annotations are gone, including the payload files on disk. + try (var stream = Files.walk(tempDir.resolve("s3").resolve(".accounts") + .resolve("000000000000").resolve(".annotations"))) { + List files = stream.filter(Files::isRegularFile).toList(); + assertTrue(files.isEmpty(), "annotation payload files should be removed, found: " + files); + } + } + + @Test + void failedOverwriteDoesNotDropAnnotations() { + putAnnotation("survivor", "payload"); + // A key that escapes the bucket directory fails the body write (InvalidKey): the + // annotation cleanup runs only after the write succeeds, so the annotation survives. + assertThrowsAws("InvalidKey", () -> s3Service.putObject(BUCKET, "../docs/readme.txt", + "body".getBytes(StandardCharsets.UTF_8), null, null, null)); + assertEquals("payload", readPayload("survivor")); + } + + @Test + void governanceRetentionBlocksAnnotationDeleteWithoutBypass() { + s3Service.putObject(BUCKET, "locked.txt", "body".getBytes(StandardCharsets.UTF_8), + null, null, new io.github.hectorvent.floci.services.s3.model.PutObjectOptions() + .withObjectLockMode("GOVERNANCE") + .withRetainUntilDate(java.time.Instant.now().plusSeconds(3600))); + // Put is symmetric with delete: a retention-protected version cannot receive annotations. + assertThrowsAws("AccessDenied", () -> + s3Service.putObjectAnnotation(BUCKET, "locked.txt", "locked-ann", null, + "payload".getBytes(StandardCharsets.UTF_8), null, null)); + s3Service.putObject(BUCKET, "governed-plain.txt", "body".getBytes(StandardCharsets.UTF_8), + null, null, null); + s3Service.putObjectAnnotation(BUCKET, "governed-plain.txt", "ann", null, + "payload".getBytes(StandardCharsets.UTF_8), null, null); + s3Service.putObject(BUCKET, "governed-plain.txt", "body v2".getBytes(StandardCharsets.UTF_8), + null, null, new io.github.hectorvent.floci.services.s3.model.PutObjectOptions() + .withObjectLockMode("GOVERNANCE") + .withRetainUntilDate(java.time.Instant.now().plusSeconds(3600))); + // Replacing the annotation via a re-put is blocked too: the protection cannot be + // circumvented by a put. + assertThrowsAws("AccessDenied", () -> + s3Service.putObjectAnnotation(BUCKET, "governed-plain.txt", "ann", null, + "changed".getBytes(StandardCharsets.UTF_8), null, null)); + assertThrowsAws("AccessDenied", () -> + s3Service.deleteObjectAnnotation(BUCKET, "governed-plain.txt", "ann", null, null, false)); + // With the bypass flag the delete succeeds. + assertDoesNotThrow(() -> + s3Service.deleteObjectAnnotation(BUCKET, "governed-plain.txt", "ann", null, null, true)); + } + + @Test + void complianceRetentionBlocksAnnotationPutAndDelete() { + s3Service.putObject(BUCKET, "compliance.txt", "body".getBytes(StandardCharsets.UTF_8), + null, null, new io.github.hectorvent.floci.services.s3.model.PutObjectOptions() + .withObjectLockMode("COMPLIANCE") + .withRetainUntilDate(java.time.Instant.now().plusSeconds(3600))); + // Put is blocked outright on a COMPLIANCE-protected version (no bypass exists for put). + assertThrowsAws("AccessDenied", () -> + s3Service.putObjectAnnotation(BUCKET, "compliance.txt", "compliance-ann", null, + "payload".getBytes(StandardCharsets.UTF_8), null, null)); + // Delete is blocked even with the governance bypass. + assertThrowsAws("AccessDenied", () -> + s3Service.deleteObjectAnnotation(BUCKET, "compliance.txt", "compliance-ann", null, null, true)); + } + + @Test + void literalNullVersionIdAddressesPreVersioningObject() { + putAnnotation("pre-versioning", "payload"); + // ListObjectVersions reports pre-versioning objects with VersionId "null"; echoing it + // back must address the same annotation as omitting versionId. + ObjectAnnotation annotation = s3Service.getObjectAnnotation(BUCKET, "docs/readme.txt", + "pre-versioning", "null"); + assertEquals("payload", new String( + s3Service.readObjectAnnotationPayload(annotation), StandardCharsets.UTF_8)); + s3Service.deleteObjectAnnotation(BUCKET, "docs/readme.txt", "pre-versioning", "null", null, false); + assertThrowsAws("NoSuchAnnotation", () -> + s3Service.getObjectAnnotation(BUCKET, "docs/readme.txt", "pre-versioning", null)); + } + + // ========== Persistence ========== + + @Test + void payloadsAreStoredOnDiskNotInMetadataStore() throws Exception { + putAnnotation("disk-backed", "on disk"); + ObjectAnnotation annotation = s3Service.getObjectAnnotation(BUCKET, "docs/readme.txt", + "disk-backed", null); + assertEquals("on disk", readPayload("disk-backed")); + // The .s3ann file exists under the account-scoped .annotations root. + try (var stream = Files.walk(tempDir.resolve("s3").resolve(".accounts") + .resolve("000000000000").resolve(".annotations"))) { + List files = stream.filter(Files::isRegularFile).toList(); + assertEquals(1, files.size()); + assertTrue(files.get(0).getFileName().toString().endsWith(".s3ann")); + assertEquals("on disk", Files.readString(files.get(0), StandardCharsets.UTF_8)); + } + assertEquals(annotation.getSize(), "on disk".getBytes(StandardCharsets.UTF_8).length); + } + + @Test + void clearRemovesInMemoryAnnotationPayloads() { + S3Service inMemoryService = new S3Service(new InMemoryStorage<>(), new InMemoryStorage<>(), + tempDir, true); + inMemoryService.createBucket(BUCKET, "us-east-1"); + inMemoryService.putObject(BUCKET, "k", "body".getBytes(StandardCharsets.UTF_8), null, null, null); + inMemoryService.putObjectAnnotation(BUCKET, "k", "ann", null, + "payload".getBytes(StandardCharsets.UTF_8), null, null); + inMemoryService.clear(); + ObjectAnnotation annotation = inMemoryService.getObjectAnnotation(BUCKET, "k", "ann", null); + // Metadata survives clear() (it lives in the store); the payload is gone. + assertThrows(AwsException.class, () -> inMemoryService.readObjectAnnotationPayload(annotation)); + } + + private static void assertThrowsAws(String errorCode, Runnable action) { + AwsException exception = assertThrows(AwsException.class, action::run); + assertEquals(errorCode, exception.getErrorCode()); + } +} \ No newline at end of file From b9da9ad7a58182f886758e53c461bd268f9fe1e8 Mon Sep 17 00:00:00 2001 From: Rafael Sierra Date: Thu, 10 Sep 2026 16:40:33 +0200 Subject: [PATCH 2/3] fix(s3): serialize annotation restore with copy publication and sweep payloads on reset --- .../floci/services/s3/S3Service.java | 91 +++++++++++++++---- .../services/s3/S3ServiceAnnotationsTest.java | 41 +++++++++ 2 files changed, 114 insertions(+), 18 deletions(-) diff --git a/src/main/java/io/github/hectorvent/floci/services/s3/S3Service.java b/src/main/java/io/github/hectorvent/floci/services/s3/S3Service.java index 7ca79bd998..ba2701a2ec 100644 --- a/src/main/java/io/github/hectorvent/floci/services/s3/S3Service.java +++ b/src/main/java/io/github/hectorvent/floci/services/s3/S3Service.java @@ -265,6 +265,31 @@ public void clear() { memoryAnnotationStore.clear(); memoryMultipartStore.clear(); multipartUploads.clear(); + if (!inMemory) { + // The reset above erases the annotation metadata through storageFactory.clearAll(), + // so detached .s3ann payload files become unreachable: sweep the annotation payload + // root for every account partition (reset runs outside request context, so the + // default account alone is not enough). Mirrors the metadata erase; the pre-existing + // .s3data behavior is unchanged. + deleteAnnotationPayloadRoots(); + } + } + + private void deleteAnnotationPayloadRoots() { + Path accountsRoot = dataRoot.resolve(ACCOUNT_STORAGE_ROOT); + if (!Files.isDirectory(accountsRoot)) { + return; + } + try (var accounts = Files.list(accountsRoot)) { + for (Path account : accounts.toList()) { + Path annotationsRoot = account.resolve(ANNOTATION_STORAGE_ROOT); + if (Files.isDirectory(annotationsRoot)) { + deleteDirectory(annotationsRoot); + } + } + } catch (IOException e) { + LOG.errorv(e, "Failed to reset annotation payload files under {0}", accountsRoot); + } } public Bucket createBucket(String bucketName, String region) { @@ -4617,16 +4642,26 @@ private S3Object copyS3Object(String sourceBucket, String sourceKey, fireNotifications(destBucket, destKey, "ObjectCreated:Copy", result[0]); return result[0]; } - S3Object copy = storeObjectCopy(destBucket, destKey, source, metadata, effectiveChecksum, - effectiveContentType, effectiveStorageClass, effectiveContentEncoding, - effectiveContentDisposition, effectiveCacheControl, effectiveServerSideEncryption, - effectiveOptions, copyChecksumAlgorithm, effectiveTags); - if (copyAnnotations) { - restoreAnnotations(sourceAnnotations, destBucket, destKey, copy); + // Publish and restore under the DESTINATION bucket monitor: an overwrite of the + // destination key is serialized against the restore, so the copied annotations can + // never attach to a newer, unrelated object that lands in between (the annotations' + // plain-key identity is shared by every non-versioned object at this key). The source + // monitor above was already released, so the two locks are never held together and a + // concurrent reverse copy cannot deadlock. + S3Object[] result = {null}; + synchronized (requireBucket(destBucket)) { + result[0] = storeObjectCopy(destBucket, destKey, source, metadata, effectiveChecksum, + effectiveContentType, effectiveStorageClass, effectiveContentEncoding, + effectiveContentDisposition, effectiveCacheControl, effectiveServerSideEncryption, + effectiveOptions, copyChecksumAlgorithm, effectiveTags); + if (copyAnnotations) { + restoreAnnotations(sourceAnnotations, destBucket, destKey, result[0]); + } } + // Fired outside the bucket monitor, matching the self-copy path. LOG.debugv("Copied object: {0}/{1} -> {2}/{3}", sourceBucket, sourceKey, destBucket, destKey); - fireNotifications(destBucket, destKey, "ObjectCreated:Copy", copy); - return copy; + fireNotifications(destBucket, destKey, "ObjectCreated:Copy", result[0]); + return result[0]; } private S3Object storeObjectCopy(String destBucket, String destKey, S3Object source, @@ -4679,16 +4714,36 @@ private void restoreAnnotations(List snapshots, String destB return; } String destParentKey = annotationParentKey(destBucket, destKey, copy.getVersionId()); - for (AnnotationSnapshot snapshot : snapshots) { - // The payload bytes are identical, so the source annotation's ETag and checksum are - // preserved; only the identity fields and lastModified are recomputed. - ObjectAnnotation copied = new ObjectAnnotation(destBucket, destKey, copy.getVersionId(), - snapshot.metadata().getAnnotationName(), snapshot.metadata().getSize(), - snapshot.metadata().getETag(), Instant.now(), - snapshot.metadata().getChecksumAlgorithm(), snapshot.metadata().getChecksumValue()); - copied.setServerSideEncryption(copy.getServerSideEncryption()); - writeAnnotationPayload(copied, snapshot.payload()); - annotationStore.put(annotationStoreKey(destParentKey, snapshot.metadata().getAnnotationName()), copied); + // The destination object is already published, so a mid-restore failure must not leave a + // partial annotation set behind (a failed copy whose retry would find partial state). + // Every restored annotation is tracked before its writes; on failure the written + // annotations are rolled back best-effort, leaving the destination with none of the + // copied annotations rather than a partial set. + List restored = new ArrayList<>(); + try { + for (AnnotationSnapshot snapshot : snapshots) { + // The payload bytes are identical, so the source annotation's ETag and checksum + // are preserved; only the identity fields and lastModified are recomputed. + ObjectAnnotation copied = new ObjectAnnotation(destBucket, destKey, copy.getVersionId(), + snapshot.metadata().getAnnotationName(), snapshot.metadata().getSize(), + snapshot.metadata().getETag(), Instant.now(), + snapshot.metadata().getChecksumAlgorithm(), snapshot.metadata().getChecksumValue()); + copied.setServerSideEncryption(copy.getServerSideEncryption()); + restored.add(copied); + writeAnnotationPayload(copied, snapshot.payload()); + annotationStore.put(annotationStoreKey(destParentKey, copied.getAnnotationName()), copied); + } + } catch (RuntimeException e) { + for (ObjectAnnotation restoredAnnotation : restored) { + try { + annotationStore.delete(annotationStoreKey(destParentKey, restoredAnnotation.getAnnotationName())); + deleteAnnotationPayload(restoredAnnotation); + } catch (RuntimeException rollbackError) { + LOG.warnv(rollbackError, "Failed to roll back annotation {0} on copy destination {1}/{2}", + restoredAnnotation.getAnnotationName(), destBucket, destKey); + } + } + throw e; } } diff --git a/src/test/java/io/github/hectorvent/floci/services/s3/S3ServiceAnnotationsTest.java b/src/test/java/io/github/hectorvent/floci/services/s3/S3ServiceAnnotationsTest.java index 9ca42bef53..23a16f7300 100644 --- a/src/test/java/io/github/hectorvent/floci/services/s3/S3ServiceAnnotationsTest.java +++ b/src/test/java/io/github/hectorvent/floci/services/s3/S3ServiceAnnotationsTest.java @@ -367,6 +367,47 @@ void copyObjectWithExcludeDirectiveSkipsAnnotations() { s3Service.getObjectAnnotation(BUCKET, "docs/copy2.txt", "excluded", null)); } + @Test + void crossBucketCopyCarriesAnnotationsByDefaultAndHonorsExclude() { + putAnnotation("cross-bucket", "payload"); + s3Service.createBucket("cross-bucket-dest", "us-east-1"); + s3Service.copyObject(BUCKET, "docs/readme.txt", "cross-bucket-dest", "docs/copy.txt", + new io.github.hectorvent.floci.services.s3.model.CopyObjectOptions()); + ObjectAnnotation annotation = s3Service.getObjectAnnotation("cross-bucket-dest", + "docs/copy.txt", "cross-bucket", null); + assertEquals("payload", new String( + s3Service.readObjectAnnotationPayload(annotation), StandardCharsets.UTF_8)); + + s3Service.copyObject(BUCKET, "docs/readme.txt", "cross-bucket-dest", "docs/copy-excluded.txt", + new io.github.hectorvent.floci.services.s3.model.CopyObjectOptions() + .withAnnotationDirective("EXCLUDE")); + assertThrowsAws("NoSuchAnnotation", () -> + s3Service.getObjectAnnotation("cross-bucket-dest", "docs/copy-excluded.txt", "cross-bucket", null)); + } + + // ========== Reset ========== + + @Test + void resetRemovesAnnotationPayloadFiles() throws Exception { + putAnnotation("reset-me", "payload"); + s3Service.clear(); + // The metadata was erased and the .s3ann payload files must not leak: the .annotations + // root is empty for every account partition. + Path accountsRoot = tempDir.resolve("s3").resolve(".accounts"); + if (Files.isDirectory(accountsRoot)) { + try (var accounts = Files.list(accountsRoot)) { + for (Path account : accounts.toList()) { + Path annotationsRoot = account.resolve(".annotations"); + assertFalse(Files.exists(annotationsRoot), + "annotation payloads survived reset: " + annotationsRoot); + } + } + } + assertThrowsAws("NoSuchAnnotation", () -> + s3Service.readObjectAnnotationPayload( + s3Service.getObjectAnnotation(BUCKET, "docs/readme.txt", "reset-me", null))); + } + // ========== Review regressions ========== @Test From 816f887b90fa0ab60ce073906561938ca8da76da Mon Sep 17 00:00:00 2001 From: Rafael Sierra Date: Thu, 10 Sep 2026 17:31:56 +0200 Subject: [PATCH 3/3] fix(s3): lock cross-account copy buckets via resolveBucket and log compat test cleanup --- .../java/com/floci/test/S3AnnotationsTest.java | 12 ++++++++++-- .../hectorvent/floci/services/s3/S3Service.java | 14 +++++++++++--- 2 files changed, 21 insertions(+), 5 deletions(-) diff --git a/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/S3AnnotationsTest.java b/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/S3AnnotationsTest.java index 12de1c5176..2e58f0448c 100644 --- a/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/S3AnnotationsTest.java +++ b/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/S3AnnotationsTest.java @@ -2,6 +2,9 @@ import org.junit.jupiter.api.*; +import java.util.logging.Level; +import java.util.logging.Logger; + import software.amazon.awssdk.core.ResponseBytes; import software.amazon.awssdk.core.sync.RequestBody; import software.amazon.awssdk.services.s3.S3Client; @@ -24,6 +27,7 @@ @TestMethodOrder(MethodOrderer.OrderAnnotation.class) class S3AnnotationsTest { + private static final Logger LOG = Logger.getLogger(S3AnnotationsTest.class.getName()); private static S3Client s3; private static final String BUCKET = "sdk-annotations-bucket"; private static final String KEY = "docs/annotated.txt"; @@ -37,15 +41,19 @@ static void setup() { @AfterAll static void teardown() { + // Cleanup failures are logged with context instead of swallowed: a leaked object or a + // bucket that failed to delete would otherwise hide from later test runs. for (String key : new String[]{"docs/annotated.txt", "docs/annotated-copy.txt", "docs/annotated-copied.txt"}) { try { s3.deleteObject(r -> r.bucket(BUCKET).key(key)); - } catch (Exception ignored) { + } catch (Exception e) { + LOG.log(Level.WARNING, "Failed to clean up object " + BUCKET + "/" + key, e); } } try { s3.deleteBucket(r -> r.bucket(BUCKET)); - } catch (Exception ignored) { + } catch (Exception e) { + LOG.log(Level.WARNING, "Failed to clean up bucket " + BUCKET, e); } } diff --git a/src/main/java/io/github/hectorvent/floci/services/s3/S3Service.java b/src/main/java/io/github/hectorvent/floci/services/s3/S3Service.java index ba2701a2ec..2b20ecbe62 100644 --- a/src/main/java/io/github/hectorvent/floci/services/s3/S3Service.java +++ b/src/main/java/io/github/hectorvent/floci/services/s3/S3Service.java @@ -4607,7 +4607,11 @@ private S3Object copyS3Object(String sourceBucket, String sourceKey, // same profile as the source body copy itself. boolean copyAnnotations = !"EXCLUDE".equalsIgnoreCase(effectiveOptions.getAnnotationDirective()); boolean selfCopy = sourceBucket.equals(destBucket); - Bucket sourceMonitor = requireBucket(sourceBucket); + // resolveBucket (not requireBucket): with globalBucketNamespace the bucket can belong to + // another account, and this must be the same instance the write path (storeObject) locks. + Bucket sourceMonitor = resolveBucket(sourceBucket) + .orElseThrow(() -> new AwsException("NoSuchBucket", + "The specified bucket does not exist.", 404)); List sourceAnnotations = List.of(); if (copyAnnotations && !selfCopy) { // Cross-bucket copy: the destination overwrite cannot touch the source's annotation @@ -4647,9 +4651,13 @@ private S3Object copyS3Object(String sourceBucket, String sourceKey, // never attach to a newer, unrelated object that lands in between (the annotations' // plain-key identity is shared by every non-versioned object at this key). The source // monitor above was already released, so the two locks are never held together and a - // concurrent reverse copy cannot deadlock. + // concurrent reverse copy cannot deadlock. resolveBucket (not requireBucket) keeps + // cross-account destinations working with globalBucketNamespace, and returns the same + // instance storeObject locks, so this monitor re-enters the write path's own. S3Object[] result = {null}; - synchronized (requireBucket(destBucket)) { + synchronized (resolveBucket(destBucket) + .orElseThrow(() -> new AwsException("NoSuchBucket", + "The specified bucket does not exist.", 404))) { result[0] = storeObjectCopy(destBucket, destKey, source, metadata, effectiveChecksum, effectiveContentType, effectiveStorageClass, effectiveContentEncoding, effectiveContentDisposition, effectiveCacheControl, effectiveServerSideEncryption,