diff --git a/storage/src/main/java/com/epam/aidial/core/storage/data/ResourceUpload.java b/storage/src/main/java/com/epam/aidial/core/storage/data/ResourceUpload.java index 20ec9ceb0..b69147017 100644 --- a/storage/src/main/java/com/epam/aidial/core/storage/data/ResourceUpload.java +++ b/storage/src/main/java/com/epam/aidial/core/storage/data/ResourceUpload.java @@ -1,6 +1,7 @@ package com.epam.aidial.core.storage.data; import com.epam.aidial.core.storage.blobstore.BlobStorage; +import com.epam.aidial.core.storage.util.EtagBuilder; import io.netty.buffer.ByteBuf; import io.netty.buffer.ByteBufInputStream; import lombok.Data; @@ -21,8 +22,10 @@ public class ResourceUpload { private final String contentType; private final long updatedAt; private final long createdAt; + private final EtagBuilder etagBuilder; private long contentLength; private int chunkNumber = 0; + private String etag; public ResourceUpload(BlobStorage blobStorage, MultipartUpload mpu, String contentType, long createdAt, long updatedAt) { this.multipartUpload = mpu; @@ -30,6 +33,7 @@ public ResourceUpload(BlobStorage blobStorage, MultipartUpload mpu, String conte this.contentType = contentType; this.createdAt = createdAt; this.updatedAt = updatedAt; + this.etagBuilder = new EtagBuilder(); } public void addChunk(ByteBuf chunk) throws IOException { @@ -37,9 +41,17 @@ public void addChunk(ByteBuf chunk) throws IOException { MultipartPart part = blobStorage.storeMultipartPart(multipartUpload, ++chunkNumber, payload); parts.add(part); } + etagBuilder.append(chunk.nioBuffer()); contentLength += chunk.readableBytes(); } + public String calculateEtag() { + if (etag == null) { + etag = etagBuilder.build(); + } + return etag; + } + public void abort() { blobStorage.abortMultipartUpload(multipartUpload); } diff --git a/storage/src/main/java/com/epam/aidial/core/storage/service/ResourceService.java b/storage/src/main/java/com/epam/aidial/core/storage/service/ResourceService.java index 0c9c1fb21..9147b13e2 100644 --- a/storage/src/main/java/com/epam/aidial/core/storage/service/ResourceService.java +++ b/storage/src/main/java/com/epam/aidial/core/storage/service/ResourceService.java @@ -495,7 +495,15 @@ public FileMetadata finishFileUpload(ResourceDescriptor descriptor, ResourceUplo long updatedAt = resourceUpload.getUpdatedAt(); Long createdAt = resourceUpload.getCreatedAt(); - String etag = blobStore.completeMultipartUpload(multipartUpload, parts); + String etag = resourceUpload.calculateEtag(); + + // try to update user metadata with the computed e-tag. + // Note. That doesn't work in AWS S3. Instead of that we take e-tag to be provided by AWS S3. + // AWS S3 computes e-tag based blob content. + Map userMetadata = multipartUpload.blobMetadata().getUserMetadata(); + userMetadata.put(ETAG_ATTRIBUTE, etag); + + blobStore.completeMultipartUpload(multipartUpload, parts); ResourceEvent.Action action = metadata == null ? ResourceEvent.Action.CREATE @@ -901,6 +909,20 @@ static String decode(String val) { return new String(Base58.decode(decoded), StandardCharsets.UTF_8); } + /** + * Extracts e-tag from the blob metadata. + * + *

+ * There are two cases: + *

+ *

+ * + * @param meta - blob metadata + * @return e-tag + */ private static String extractEtag(BlobMetadata meta) { Map attributes = meta.getUserMetadata(); String etag = attributes.get(ETAG_ATTRIBUTE);