diff --git a/docs/services/gcs.md b/docs/services/gcs.md index ba0527c4..d5abdabb 100644 --- a/docs/services/gcs.md +++ b/docs/services/gcs.md @@ -275,7 +275,9 @@ The embedded DNS server resolves `*.localhost.floci.io` to floci-gcp's container **Cloud Storage gRPC v2:** - Buckets: `CreateBucket`, `GetBucket`, `ListBuckets`, `UpdateBucket`, `DeleteBucket` -- Objects: `ComposeObject`, `GetObject`, `ListObjects`, `UpdateObject`, `DeleteObject` +- Objects: `ComposeObject`, `GetObject`, `ListObjects`, `UpdateObject`, `DeleteObject`; + object system metadata (`cacheControl`, `contentDisposition`, `contentEncoding`, + `contentLanguage`, `customTime`, `storageClass`) round-trips with the REST path - Data path: `ReadObject`, `WriteObject`, `BidiWriteObject` - Resumable writes: `StartResumableWrite`, `QueryWriteStatus` @@ -323,10 +325,13 @@ handles, or redirection. Unsupported RPCs return gRPC `UNIMPLEMENTED`. - `CopyObject` - `MoveObject` - `HeadObject` -- `PatchObject` (update metadata: `contentType`, `contentDisposition`, `contentEncoding`, `contentLanguage`, `customTime`, custom metadata) +- `PatchObject` (update metadata: `contentType`, `contentDisposition`, `contentEncoding`, `contentLanguage`, `cacheControl`, `customTime`, custom metadata) - System metadata at upload time (`contentEncoding`, `contentDisposition`, `contentLanguage`, - `customTime`, `storageClass`) as query parameters or in the JSON metadata part of a - multipart/resumable upload + `cacheControl`, `customTime`, `storageClass`) in the JSON metadata part of a + multipart/resumable upload; on the upload URL only `contentEncoding` is honoured, as on GCS +- `customTime` follows the GCS rules: rendered in UTC, never removed once set (a `null` + patch or an unset gRPC `custom_time` under the mask is a no-op), and a decrease is + rejected with `400` / `INVALID_ARGUMENT` - `ComposeObject` (concatenate 1 to 32 source objects; 0 or more than 32 is a 400) - `RewriteObject` (multi-call when the source and destination span storage classes or bucket locations: a `maxBytesRewrittenPerCall`, which must be a multiple of 1 MiB, below the object diff --git a/src/main/java/io/floci/gcp/services/gcs/GcsCustomTime.java b/src/main/java/io/floci/gcp/services/gcs/GcsCustomTime.java new file mode 100644 index 00000000..c2dad18b --- /dev/null +++ b/src/main/java/io/floci/gcp/services/gcs/GcsCustomTime.java @@ -0,0 +1,93 @@ +package io.floci.gcp.services.gcs; + +import com.google.protobuf.Timestamp; +import io.floci.gcp.core.common.GcpException; + +import java.math.BigInteger; +import java.time.Instant; +import java.time.ZoneOffset; +import java.time.format.DateTimeFormatter; +import java.time.format.DateTimeParseException; +import java.util.function.Function; + +/** + * customTime rules shared by the JSON and gRPC paths. GCS stores the value as a protobuf + * Timestamp, renders it in UTC with 0, 3, 6 or 9 fraction digits, never removes it once set, + * and refuses to move it backwards. + */ +final class GcsCustomTime { + + private static final String PARSE_ERROR = """ + Parse Error: Invalid value for type.googleapis.com/google.protobuf.Timestamp field: \ + 'Field 'customTime', Illegal timestamp format; timestamps must end with 'Z' or have \ + a valid timezone offset.'."""; + + private static final DateTimeFormatter ERROR_SECONDS = DateTimeFormatter + .ofPattern("uuuu-MM-dd'T'HH:mm:ss").withZone(ZoneOffset.UTC); + + // GCS holds the value as int64 nanoseconds. A seconds field it cannot multiply is + // "too large"; a Timestamp that breaks the protobuf rules (nanos outside 0..999999999, + // or before 0001-01-01) is an internal error; anything else saturates at the int64 limits. + private static final long MAX_SECONDS = Long.MAX_VALUE / 1_000_000_000L; + private static final long MIN_SECONDS = -62_135_596_800L; + private static final String TOO_LARGE = "Invalid timestamp - too large to convert to nanoseconds."; + private static final String INTERNAL = "We encountered an internal error. Please try again."; + private static final BigInteger NANOS_PER_SECOND = BigInteger.valueOf(1_000_000_000L); + private static final BigInteger INT64_MIN = BigInteger.valueOf(Long.MIN_VALUE); + private static final BigInteger INT64_MAX = BigInteger.valueOf(Long.MAX_VALUE); + + private GcsCustomTime() { + } + + static String normalize(String value) { + try { + return Instant.parse(value).toString(); + } catch (DateTimeParseException e) { + throw GcpException.invalidArgument(PARSE_ERROR); + } + } + + /** WriteObject reports a too-large custom_time as INVALID_ARGUMENT. */ + static String fromWrite(Timestamp value) { + return fromProto(value, GcpException::invalidArgument); + } + + /** UpdateObject reports the same failure as INTERNAL. */ + static String fromUpdate(Timestamp value) { + return fromProto(value, GcpException::internal); + } + + private static String fromProto(Timestamp value, Function tooLarge) { + if (value.getSeconds() > MAX_SECONDS) { + throw tooLarge.apply(TOO_LARGE); + } + if (value.getNanos() < 0 || value.getNanos() > 999_999_999 || value.getSeconds() < MIN_SECONDS) { + throw GcpException.internal(INTERNAL); + } + var nanos = BigInteger.valueOf(value.getSeconds()).multiply(NANOS_PER_SECOND) + .add(BigInteger.valueOf(value.getNanos())) + .max(INT64_MIN).min(INT64_MAX); + return Instant.EPOCH.plusNanos(nanos.longValueExact()).toString(); + } + + static void requireNotDecreased(String previous, String next) { + if (previous == null) { + return; + } + var before = Instant.parse(previous); + var after = Instant.parse(next); + if (after.isBefore(before)) { + throw GcpException.invalidArgument("Custom time cannot be decreased. Previously: " + + errorFormat(before) + ". Attempting to set: " + errorFormat(after) + "."); + } + } + + // The message renders the fraction without trailing zeros and with an explicit +00:00. + private static String errorFormat(Instant instant) { + var text = new StringBuilder(ERROR_SECONDS.format(instant)); + if (instant.getNano() != 0) { + text.append('.').append(String.format("%09d", instant.getNano()).replaceFirst("0+$", "")); + } + return text.append("+00:00").toString(); + } +} diff --git a/src/main/java/io/floci/gcp/services/gcs/GcsGrpcMapper.java b/src/main/java/io/floci/gcp/services/gcs/GcsGrpcMapper.java index 6957d7ee..74e14ee5 100644 --- a/src/main/java/io/floci/gcp/services/gcs/GcsGrpcMapper.java +++ b/src/main/java/io/floci/gcp/services/gcs/GcsGrpcMapper.java @@ -127,12 +127,14 @@ static com.google.storage.v2.Object toProto(GcsObjectMeta stored) { .setContentDisposition(orEmpty(stored.getContentDisposition())) .setContentEncoding(orEmpty(stored.getContentEncoding())) .setContentLanguage(orEmpty(stored.getContentLanguage())) + .setCacheControl(orEmpty(stored.getCacheControl())) .setTemporaryHold(Boolean.TRUE.equals(stored.getTemporaryHold())) .setEventBasedHold(Boolean.TRUE.equals(stored.getEventBasedHold())); timestamp(stored.getTimeCreated()).ifPresent(value::setCreateTime); timestamp(stored.getUpdated()).ifPresent(value::setUpdateTime); timestamp(stored.getTimeDeleted()).ifPresent(value::setDeleteTime); timestamp(stored.getRetentionExpirationTime()).ifPresent(value::setRetentionExpireTime); + timestamp(stored.getCustomTime()).ifPresent(value::setCustomTime); if (stored.getMetadata() != null) { value.putAllMetadata(stored.getMetadata()); } @@ -156,6 +158,8 @@ static GcsObjectMeta fromProto(com.google.storage.v2.Object value) { meta.setContentDisposition(blankToNull(value.getContentDisposition())); meta.setContentEncoding(blankToNull(value.getContentEncoding())); meta.setContentLanguage(blankToNull(value.getContentLanguage())); + meta.setCacheControl(blankToNull(value.getCacheControl())); + meta.setCustomTime(value.hasCustomTime() ? GcsCustomTime.fromWrite(value.getCustomTime()) : null); meta.setTemporaryHold(value.getTemporaryHold()); meta.setEventBasedHold(value.hasEventBasedHold() ? value.getEventBasedHold() : null); if (value.getMetadataCount() > 0) { @@ -169,7 +173,8 @@ static Map objectUpdateFields(com.google.storage.v2.Ob Map patch = new LinkedHashMap<>(); java.util.Set selected = paths.contains("*") ? java.util.Set.of("content_type", "content_disposition", "content_encoding", - "content_language", "metadata", "temporary_hold", "event_based_hold") + "content_language", "cache_control", "custom_time", "metadata", + "temporary_hold", "event_based_hold") : new java.util.LinkedHashSet<>(paths); for (String path : selected) { switch (path) { @@ -177,6 +182,13 @@ static Map objectUpdateFields(com.google.storage.v2.Ob case "content_disposition" -> patch.put("contentDisposition", value.getContentDisposition()); case "content_encoding" -> patch.put("contentEncoding", value.getContentEncoding()); case "content_language" -> patch.put("contentLanguage", value.getContentLanguage()); + case "cache_control" -> patch.put("cacheControl", blankToNull(value.getCacheControl())); + // GCS never removes a custom time. An unset custom_time under the mask is a no-op. + case "custom_time" -> { + if (value.hasCustomTime()) { + patch.put("customTime", GcsCustomTime.fromUpdate(value.getCustomTime())); + } + } case "metadata" -> patch.put("metadata", new LinkedHashMap<>(value.getMetadataMap())); case "temporary_hold" -> patch.put("temporaryHold", value.getTemporaryHold()); case "event_based_hold" -> patch.put("eventBasedHold", value.getEventBasedHold()); diff --git a/src/main/java/io/floci/gcp/services/gcs/GcsService.java b/src/main/java/io/floci/gcp/services/gcs/GcsService.java index 44fabdc4..68575c10 100644 --- a/src/main/java/io/floci/gcp/services/gcs/GcsService.java +++ b/src/main/java/io/floci/gcp/services/gcs/GcsService.java @@ -753,6 +753,12 @@ private GcsObjectMeta patchObjectLocked(String bucket, String objectName, Map GcpException.notFound("Object not found: " + objectName)); + // GCS never removes a custom time, so a null here leaves the field alone. + String customTime = null; + if (patch.get("customTime") instanceof String requested) { + customTime = GcsCustomTime.normalize(requested); + GcsCustomTime.requireNotDecreased(meta.getCustomTime(), customTime); + } if (patch.containsKey("contentType")) { meta.setContentType((String) patch.get("contentType")); } @@ -779,8 +785,8 @@ private GcsObjectMeta patchObjectLocked(String bucket, String objectName, Map metadata) { GcsObjectMeta merged = base; for (String field : SYSTEM_METADATA_FIELDS) { - if (!(metadata.get(field) instanceof String value) || value.isBlank()) { + if (!(metadata.get(field) instanceof String value)) { + continue; + } + // GCS rejects an empty customTime instead of ignoring it like the other fields. + if (value.isBlank() && !field.equals("customTime")) { continue; } if (merged == null) { @@ -182,7 +181,7 @@ private static void assignSystemMetadata(GcsObjectMeta meta, String field, Strin case "contentDisposition" -> meta.setContentDisposition(value); case "contentLanguage" -> meta.setContentLanguage(value); case "cacheControl" -> meta.setCacheControl(value); - case "customTime" -> meta.setCustomTime(value); + case "customTime" -> meta.setCustomTime(GcsCustomTime.normalize(value)); case "storageClass" -> meta.setStorageClass(value); default -> { /* unreachable: every SYSTEM_METADATA_FIELDS entry is handled above */ } } diff --git a/src/test/java/io/floci/gcp/services/gcs/GcsGrpcControllerTest.java b/src/test/java/io/floci/gcp/services/gcs/GcsGrpcControllerTest.java index 0bd7d8ba..609375a1 100644 --- a/src/test/java/io/floci/gcp/services/gcs/GcsGrpcControllerTest.java +++ b/src/test/java/io/floci/gcp/services/gcs/GcsGrpcControllerTest.java @@ -2,6 +2,7 @@ import com.google.protobuf.ByteString; import com.google.protobuf.FieldMask; +import com.google.protobuf.Timestamp; import com.google.storage.v2.BidiWriteObjectRequest; import com.google.storage.v2.BidiWriteObjectResponse; import com.google.storage.v2.Bucket; @@ -9,6 +10,7 @@ import com.google.storage.v2.ComposeObjectRequest; import com.google.storage.v2.CreateBucketRequest; import com.google.storage.v2.GetBucketRequest; +import com.google.storage.v2.GetObjectRequest; import com.google.storage.v2.ListBucketsRequest; import com.google.storage.v2.ListBucketsResponse; import com.google.storage.v2.ListObjectsRequest; @@ -32,6 +34,7 @@ import java.io.ByteArrayOutputStream; import java.nio.charset.StandardCharsets; +import java.time.Instant; import java.util.ArrayList; import java.util.List; import java.util.Map; @@ -331,6 +334,208 @@ void objectUpdateAcceptsSdkMetadataKeyMask() { assertEquals("true", response.single().getMetadataOrThrow("updated")); } + /** + * cacheControl and customTime are part of the system metadata a REST write already + * accepts. A gRPC-only client reads object metadata through GetObject and ListObjects, + * so both have to carry them rather than reporting an empty field. + */ + @Test + void writeCarriesCacheControlAndCustomTimeThroughGetAndList() { + createBucket("grpc-system-metadata-bucket"); + byte[] payload = "cached".getBytes(StandardCharsets.UTF_8); + Timestamp customTime = timestamp("2026-01-15T10:30:00Z"); + + RecordingObserver written = new RecordingObserver<>(); + StreamObserver stream = controller.writeObject(written); + stream.onNext(WriteObjectRequest.newBuilder() + .setWriteObjectSpec(WriteObjectSpec.newBuilder() + .setResource(object("grpc-system-metadata-bucket", "cached.txt").toBuilder() + .setCacheControl("public, max-age=3600") + .setCustomTime(customTime)) + .setObjectSize(payload.length)) + .setWriteOffset(0) + .setChecksummedData(data(payload)) + .setFinishWrite(true) + .build()); + stream.onCompleted(); + + assertNull(written.error); + assertEquals("public, max-age=3600", written.single().getResource().getCacheControl()); + assertEquals(customTime, written.single().getResource().getCustomTime()); + + RecordingObserver fetched = new RecordingObserver<>(); + controller.getObject(GetObjectRequest.newBuilder() + .setBucket("projects/_/buckets/grpc-system-metadata-bucket") + .setObject("cached.txt") + .build(), fetched); + assertNull(fetched.error); + assertEquals("public, max-age=3600", fetched.single().getCacheControl()); + assertEquals(customTime, fetched.single().getCustomTime()); + + RecordingObserver listed = new RecordingObserver<>(); + controller.listObjects(ListObjectsRequest.newBuilder() + .setParent("projects/_/buckets/grpc-system-metadata-bucket").build(), listed); + assertNull(listed.error); + assertEquals("public, max-age=3600", listed.single().getObjects(0).getCacheControl()); + } + + @Test + void objectUpdateSetsCacheControlAndCustomTime() { + createBucket("grpc-patch-metadata-bucket"); + service.putObject("grpc-patch-metadata-bucket", "object", "text/plain", + new byte[] {1}, BASE_URL); + Timestamp customTime = timestamp("2026-06-20T08:00:00Z"); + + RecordingObserver response = new RecordingObserver<>(); + controller.updateObject(UpdateObjectRequest.newBuilder() + .setObject(object("grpc-patch-metadata-bucket", "object").toBuilder() + .setCacheControl("no-store") + .setCustomTime(customTime)) + .setUpdateMask(FieldMask.newBuilder().addPaths("cache_control").addPaths("custom_time")) + .build(), response); + + assertNull(response.error); + assertEquals("no-store", response.single().getCacheControl()); + assertEquals(customTime, response.single().getCustomTime()); + assertEquals("no-store", + service.getObjectMeta("grpc-patch-metadata-bucket", "object").getCacheControl()); + } + + /** + * GCS never removes a custom time. Naming custom_time in the mask without a value, or + * sending a "*" mask with the field unset, leaves the existing value in place. + */ + @Test + void objectUpdateWithAnUnsetCustomTimeKeepsTheValue() { + createBucket("grpc-keep-custom-time-bucket"); + service.putObject("grpc-keep-custom-time-bucket", "object", "text/plain", + new byte[] {1}, BASE_URL); + Timestamp customTime = timestamp("2026-06-20T08:00:00Z"); + updateObject("grpc-keep-custom-time-bucket", + object("grpc-keep-custom-time-bucket", "object").toBuilder().setCustomTime(customTime), + "custom_time"); + + RecordingObserver masked = updateObject("grpc-keep-custom-time-bucket", + object("grpc-keep-custom-time-bucket", "object").toBuilder(), "custom_time"); + assertNull(masked.error); + assertEquals(customTime, masked.single().getCustomTime()); + + RecordingObserver star = updateObject("grpc-keep-custom-time-bucket", + object("grpc-keep-custom-time-bucket", "object").toBuilder().setCacheControl("max-age=1"), "*"); + assertNull(star.error); + assertEquals("max-age=1", star.single().getCacheControl()); + assertEquals(customTime, star.single().getCustomTime()); + } + + @Test + void objectUpdateRejectsADecreasedCustomTime() { + createBucket("grpc-decrease-custom-time-bucket"); + service.putObject("grpc-decrease-custom-time-bucket", "object", "text/plain", + new byte[] {1}, BASE_URL); + updateObject("grpc-decrease-custom-time-bucket", + object("grpc-decrease-custom-time-bucket", "object").toBuilder() + .setCustomTime(timestamp("2027-01-01T00:00:00Z")), + "custom_time"); + + RecordingObserver response = updateObject("grpc-decrease-custom-time-bucket", + object("grpc-decrease-custom-time-bucket", "object").toBuilder() + .setCustomTime(timestamp("2025-01-01T00:00:00Z")), + "custom_time"); + + Status status = Status.fromThrowable(response.error); + assertEquals(Status.Code.INVALID_ARGUMENT, status.getCode()); + assertEquals(""" + Custom time cannot be decreased. Previously: 2027-01-01T00:00:00+00:00. \ + Attempting to set: 2025-01-01T00:00:00+00:00.""", status.getDescription()); + } + + /** + * GCS keeps custom_time as int64 nanoseconds. A seconds value it cannot multiply is rejected + * as "too large" (INVALID_ARGUMENT on write, INTERNAL on update), a Timestamp that breaks the + * protobuf rules is an internal error, and values inside the range saturate at the limits. + */ + @Test + void malformedCustomTimeIsRejectedOrSaturatedLikeGcs() { + createBucket("grpc-custom-time-range-bucket"); + String tooLarge = "Invalid timestamp - too large to convert to nanoseconds."; + String internal = "We encountered an internal error. Please try again."; + + Status tooLargeWrite = Status.fromThrowable(writeWithCustomTime("too-large", 9_223_372_037L, 0).error); + assertEquals(Status.Code.INVALID_ARGUMENT, tooLargeWrite.getCode()); + assertEquals(tooLarge, tooLargeWrite.getDescription()); + + Status badNanos = Status.fromThrowable(writeWithCustomTime("bad-nanos", 0, 1_000_000_000).error); + assertEquals(Status.Code.INTERNAL, badNanos.getCode()); + assertEquals(internal, badNanos.getDescription()); + + Status beforeYearOne = Status.fromThrowable(writeWithCustomTime("year-zero", -62_135_596_801L, 0).error); + assertEquals(Status.Code.INTERNAL, beforeYearOne.getCode()); + assertEquals(internal, beforeYearOne.getDescription()); + + RecordingObserver saturatedHigh = writeWithCustomTime("high", 9_223_372_036L, 999_999_999); + assertNull(saturatedHigh.error); + assertEquals(timestamp("2262-04-11T23:47:16.854775807Z"), saturatedHigh.single().getResource().getCustomTime()); + + RecordingObserver saturatedLow = writeWithCustomTime("low", -62_135_596_800L, 0); + assertNull(saturatedLow.error); + assertEquals(timestamp("1677-09-21T00:12:43.145224192Z"), saturatedLow.single().getResource().getCustomTime()); + + RecordingObserver tooLargeUpdate = updateObject("grpc-custom-time-range-bucket", + object("grpc-custom-time-range-bucket", "low").toBuilder() + .setCustomTime(Timestamp.newBuilder().setSeconds(9_223_372_037L)), + "custom_time"); + Status updateStatus = Status.fromThrowable(tooLargeUpdate.error); + assertEquals(Status.Code.INTERNAL, updateStatus.getCode()); + assertEquals(tooLarge, updateStatus.getDescription()); + assertEquals("1677-09-21T00:12:43.145224192Z", + service.getObjectMeta("grpc-custom-time-range-bucket", "low").getCustomTime()); + } + + private RecordingObserver writeWithCustomTime(String name, long seconds, int nanos) { + byte[] payload = {1}; + RecordingObserver written = new RecordingObserver<>(); + StreamObserver stream = controller.writeObject(written); + stream.onNext(WriteObjectRequest.newBuilder() + .setWriteObjectSpec(WriteObjectSpec.newBuilder() + .setResource(object("grpc-custom-time-range-bucket", name).toBuilder() + .setCustomTime(Timestamp.newBuilder().setSeconds(seconds).setNanos(nanos))) + .setObjectSize(payload.length)) + .setWriteOffset(0) + .setChecksummedData(data(payload)) + .setFinishWrite(true) + .build()); + stream.onCompleted(); + return written; + } + + /** An empty cache_control under the mask unsets the field, so REST omits it afterwards. */ + @Test + void objectUpdateWithAnEmptyCacheControlUnsetsIt() { + createBucket("grpc-clear-cache-control-bucket"); + service.putObject("grpc-clear-cache-control-bucket", "object", "text/plain", + new byte[] {1}, BASE_URL); + updateObject("grpc-clear-cache-control-bucket", + object("grpc-clear-cache-control-bucket", "object").toBuilder().setCacheControl("no-store"), + "cache_control"); + + RecordingObserver response = updateObject("grpc-clear-cache-control-bucket", + object("grpc-clear-cache-control-bucket", "object").toBuilder(), "cache_control"); + + assertNull(response.error); + assertEquals("", response.single().getCacheControl()); + assertNull(service.getObjectMeta("grpc-clear-cache-control-bucket", "object").getCacheControl()); + } + + private RecordingObserver updateObject(String bucket, + com.google.storage.v2.Object.Builder object, String... mask) { + RecordingObserver response = new RecordingObserver<>(); + controller.updateObject(UpdateObjectRequest.newBuilder() + .setObject(object) + .setUpdateMask(FieldMask.newBuilder().addAllPaths(List.of(mask))) + .build(), response); + return response; + } + private void createBucket(String name) { service.createBucket(name, "test-project", BASE_URL, Map.of()); } @@ -343,6 +548,12 @@ private static com.google.storage.v2.Object object(String bucket, String name) { .build(); } + private static Timestamp timestamp(String iso) { + Instant instant = Instant.parse(iso); + return Timestamp.newBuilder() + .setSeconds(instant.getEpochSecond()).setNanos(instant.getNano()).build(); + } + private static ChecksummedData data(byte[] bytes) { CRC32C crc = new CRC32C(); crc.update(bytes, 0, bytes.length); diff --git a/src/test/java/io/floci/gcp/services/gcs/GcsGrpcIntegrationTest.java b/src/test/java/io/floci/gcp/services/gcs/GcsGrpcIntegrationTest.java index 7ff09e70..c76b3c1b 100644 --- a/src/test/java/io/floci/gcp/services/gcs/GcsGrpcIntegrationTest.java +++ b/src/test/java/io/floci/gcp/services/gcs/GcsGrpcIntegrationTest.java @@ -59,4 +59,51 @@ void grpcAndRestShareTheSinglePortAndStorageState() throws Exception { channel.shutdownNow().awaitTermination(5, TimeUnit.SECONDS); } } + + /** + * cacheControl only reaches the emulator over REST. A gRPC-only client never sees the + * REST representation, so the object it reads back has to carry the field too. + */ + @Test + void grpcGetObjectReportsCacheControlSetByARestUpload() throws Exception { + String bucket = "grpc-cache-control-bucket"; + ManagedChannel channel = ManagedChannelBuilder + .forAddress(endpoint.getHost(), endpoint.getPort()) + .usePlaintext() + .build(); + try { + StorageGrpc.StorageBlockingStub storage = StorageGrpc.newBlockingStub(channel); + storage.createBucket(CreateBucketRequest.newBuilder() + .setParent("projects/_") + .setBucketId(bucket) + .setBucket(Bucket.newBuilder().setProject("projects/test-project")) + .build()); + + String body = """ + --sysmeta + Content-Type: application/json + + {"name":"cached.txt","cacheControl":"public, max-age=3600"} + --sysmeta + Content-Type: text/plain + + cached + --sysmeta-- + """.replace("\n", "\r\n"); + given() + .queryParam("uploadType", "multipart") + .header("Content-Type", "multipart/related; boundary=sysmeta") + .body(body.getBytes(StandardCharsets.UTF_8)) + .when().post("/upload/storage/v1/b/" + bucket + "/o") + .then().statusCode(200); + + com.google.storage.v2.Object object = storage.getObject(GetObjectRequest.newBuilder() + .setBucket("projects/_/buckets/" + bucket) + .setObject("cached.txt") + .build()); + assertEquals("public, max-age=3600", object.getCacheControl()); + } finally { + channel.shutdownNow().awaitTermination(5, TimeUnit.SECONDS); + } + } } diff --git a/src/test/java/io/floci/gcp/services/gcs/GcsSystemMetadataRestIntegrationTest.java b/src/test/java/io/floci/gcp/services/gcs/GcsSystemMetadataRestIntegrationTest.java index 093b02d9..a7c68e5e 100644 --- a/src/test/java/io/floci/gcp/services/gcs/GcsSystemMetadataRestIntegrationTest.java +++ b/src/test/java/io/floci/gcp/services/gcs/GcsSystemMetadataRestIntegrationTest.java @@ -8,6 +8,7 @@ import static io.restassured.RestAssured.given; import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.nullValue; /** * System metadata a client sets at upload time (contentEncoding, customTime, ...) and @@ -32,20 +33,27 @@ void mediaUploadHonorsContentEncodingQueryParameter() { .body("contentEncoding", equalTo("gzip")); } + /** + * objects.insert takes contentEncoding on the URL but no other system metadata. GCS + * silently ignores the rest, so an object uploaded this way lands without them. + */ @Test - void mediaUploadHonorsCacheControlQueryParameter() { - // cacheControl is in the accepted system-metadata set, so it has to persist rather - // than being silently dropped on the way through. + void mediaUploadIgnoresOtherSystemMetadataQueryParameters() { ensureBucket(); given().contentType("text/plain").body("x") .when().post("/upload/storage/v1/b/" + BUCKET - + "/o?uploadType=media&name=cc-media&cacheControl=max-age=3600") + + "/o?uploadType=media&name=cc-media&cacheControl=max-age=3600" + + "&customTime=2026-01-15T10:30:00Z&contentLanguage=ja&storageClass=NEARLINE") .then().statusCode(200) - .body("cacheControl", equalTo("max-age=3600")); + .body("cacheControl", nullValue()) + .body("customTime", nullValue()) + .body("contentLanguage", nullValue()) + .body("storageClass", equalTo("STANDARD")); given().when().get("/storage/v1/b/" + BUCKET + "/o/cc-media") .then().statusCode(200) - .body("cacheControl", equalTo("max-age=3600")); + .body("cacheControl", nullValue()) + .body("customTime", nullValue()); } @Test @@ -60,34 +68,106 @@ void patchSetsCacheControl() { .body("cacheControl", equalTo("no-store")); } + /** GCS renders customTime in UTC with 0, 3, 6 or 9 fraction digits, whatever was sent. */ @Test - void mediaUploadHonorsCustomTimeQueryParameter() { + void customTimeIsRenderedTheWayGcsRendersIt() { ensureBucket(); - given().contentType("text/plain").body("x") - .when().post("/upload/storage/v1/b/" + BUCKET - + "/o?uploadType=media&name=ct-media&customTime=2026-01-15T10:30:00.000Z") + given().header("Content-Type", "multipart/related; boundary=ct") + .body(multipart("ct", "{\"name\":\"ct-render\",\"customTime\":\"2026-01-15T19:30:00.120+09:00\"}")) + .when().post("/upload/storage/v1/b/" + BUCKET + "/o?uploadType=multipart") + .then().statusCode(200) + .body("customTime", equalTo("2026-01-15T10:30:00.120Z")); + + given().contentType("application/json").body(Map.of("customTime", "2026-01-15T10:30:01.123456Z")) + .when().patch("/storage/v1/b/" + BUCKET + "/o/ct-render") + .then().statusCode(200) + .body("customTime", equalTo("2026-01-15T10:30:01.123456Z")); + } + + @Test + void patchWithANullCustomTimeKeepsTheValue() { + ensureBucket(); + given().header("Content-Type", "multipart/related; boundary=ct") + .body(multipart("ct", "{\"name\":\"ct-keep\",\"customTime\":\"2026-02-01T00:00:00Z\"}")) + .when().post("/upload/storage/v1/b/" + BUCKET + "/o?uploadType=multipart") + .then().statusCode(200); + + given().contentType("application/json").body("{\"customTime\":null}") + .when().patch("/storage/v1/b/" + BUCKET + "/o/ct-keep") .then().statusCode(200) - .body("customTime", equalTo("2026-01-15T10:30:00.000Z")); + .body("customTime", equalTo("2026-02-01T00:00:00Z")); + } + + @Test + void patchRejectsADecreasedCustomTime() { + ensureBucket(); + given().header("Content-Type", "multipart/related; boundary=ct") + .body(multipart("ct", "{\"name\":\"ct-decrease\",\"customTime\":\"2026-02-01T00:00:00.5Z\"}")) + .when().post("/upload/storage/v1/b/" + BUCKET + "/o?uploadType=multipart") + .then().statusCode(200); + + String message = """ + Custom time cannot be decreased. Previously: 2026-02-01T00:00:00.5+00:00. \ + Attempting to set: 2026-02-01T00:00:00.25+00:00."""; + given().contentType("application/json").body(Map.of("customTime", "2026-02-01T00:00:00.25Z")) + .when().patch("/storage/v1/b/" + BUCKET + "/o/ct-decrease") + .then().statusCode(400) + .body("error.message", equalTo(message)) + .body("error.errors[0].reason", equalTo("invalid")); + + given().contentType("application/json").body(Map.of("customTime", "2026-02-01T00:00:00.500Z")) + .when().patch("/storage/v1/b/" + BUCKET + "/o/ct-decrease") + .then().statusCode(200) + .body("customTime", equalTo("2026-02-01T00:00:00.500Z")); + } + + @Test + void unparsableCustomTimeIsRejectedWithTheGcsMessage() { + ensureBucket(); + String message = """ + Parse Error: Invalid value for type.googleapis.com/google.protobuf.Timestamp field: \ + 'Field 'customTime', Illegal timestamp format; timestamps must end with 'Z' or have \ + a valid timezone offset.'."""; + given().header("Content-Type", "multipart/related; boundary=ct") + .body(multipart("ct", "{\"name\":\"ct-invalid\",\"customTime\":\"\"}")) + .when().post("/upload/storage/v1/b/" + BUCKET + "/o?uploadType=multipart") + .then().statusCode(400) + .body("error.message", equalTo(message)); + + given().contentType("text/plain").body("x") + .when().post("/upload/storage/v1/b/" + BUCKET + "/o?uploadType=media&name=ct-invalid") + .then().statusCode(200); + given().contentType("application/json").body(Map.of("customTime", "2027-02-01T00:00:00")) + .when().patch("/storage/v1/b/" + BUCKET + "/o/ct-invalid") + .then().statusCode(400) + .body("error.message", equalTo(message)) + .body("error.errors[0].reason", equalTo("invalid")); + } + + private static byte[] multipart(String boundary, String metadataJson) { + return """ + --%1$s + Content-Type: application/json + + %2$s + --%1$s + Content-Type: text/plain + + x + --%1$s-- + """.formatted(boundary, metadataJson).replace("\n", "\r\n").getBytes(StandardCharsets.UTF_8); } @Test void multipartUploadHonorsContentEncodingInTheMetadataPart() { ensureBucket(); - var boundary = "sysmeta"; - var body = "--" + boundary + "\r\n" - + "Content-Type: application/json\r\n\r\n" - + "{\"name\":\"ce-multipart\",\"contentEncoding\":\"gzip\",\"customTime\":\"2026-01-15T10:30:00.000Z\"}\r\n" - + "--" + boundary + "\r\n" - + "Content-Type: text/plain\r\n\r\n" - + "compressed-bytes\r\n" - + "--" + boundary + "--\r\n"; - - given().header("Content-Type", "multipart/related; boundary=" + boundary) - .body(body.getBytes(StandardCharsets.UTF_8)) + given().header("Content-Type", "multipart/related; boundary=sysmeta") + .body(multipart("sysmeta", """ + {"name":"ce-multipart","contentEncoding":"gzip","customTime":"2026-01-15T10:30:00.000Z"}""")) .when().post("/upload/storage/v1/b/" + BUCKET + "/o?uploadType=multipart") .then().statusCode(200) .body("contentEncoding", equalTo("gzip")) - .body("customTime", equalTo("2026-01-15T10:30:00.000Z")); + .body("customTime", equalTo("2026-01-15T10:30:00Z")); } @Test @@ -99,7 +179,7 @@ void patchSetsCustomTime() { given().contentType("application/json").body(Map.of("customTime", "2026-06-20T08:00:00.000Z")) .when().patch("/storage/v1/b/" + BUCKET + "/o/ct-patch") .then().statusCode(200) - .body("customTime", equalTo("2026-06-20T08:00:00.000Z")); + .body("customTime", equalTo("2026-06-20T08:00:00Z")); } @Test