From c8660d498176f3c5a500489e22196a1aa8ad09fd Mon Sep 17 00:00:00 2001 From: Gerald Hansen Date: Wed, 26 Sep 2018 15:27:56 +0200 Subject: [PATCH] Add ability to handle sse-c with add-header This will suppress md5-sum checks for sse-c uploaded and downloaded files. And use extra-headers also for get and info requests to download sse-c enrypted files again. --- S3/S3.py | 24 ++++++++++++++++++------ s3cmd | 10 ++++++++-- 2 files changed, 26 insertions(+), 8 deletions(-) diff --git a/S3/S3.py b/S3/S3.py index 13da91bd7..d3ea31941 100644 --- a/S3/S3.py +++ b/S3/S3.py @@ -706,10 +706,13 @@ def object_put(self, filename, uri, extra_headers = None, extra_label = ""): response = self.send_file(request, src_stream, labels) return response - def object_get(self, uri, stream, dest_name, start_position = 0, extra_label = ""): + def object_get(self, uri, stream, dest_name, start_position = 0, extra_headers = None, extra_label = ""): + headers = SortedDict(ignore_case = True) + if extra_headers: + headers.update(extra_headers) if uri.type != "s3": raise ValueError("Expected URI type 's3', got '%s'" % uri.type) - request = self.create_request("OBJECT_GET", uri = uri) + request = self.create_request("OBJECT_GET", uri = uri, headers = headers) labels = { 'source' : uri.uri(), 'destination' : dest_name, 'extra' : extra_label } response = self.recv_file(request, stream, labels, start_position) return response @@ -924,8 +927,12 @@ def object_move(self, src_uri, dst_uri, extra_headers = None): debug("Object '%s' NOT deleted because of an unexepected response data content.", src_uri) return response_copy - def object_info(self, uri): - request = self.create_request("OBJECT_HEAD", uri = uri) + def object_info(self, uri, extra_headers = None): + headers = SortedDict(ignore_case = True) + if extra_headers: + headers.update(extra_headers) + + request = self.create_request("OBJECT_HEAD", uri = uri, headers=headers) response = self.send_request(request) return response @@ -1583,7 +1590,10 @@ def send_file(self, request, stream, labels, buffer = '', throttle = 0, debug("MD5 sums: computed=%s, received=%s" % (md5_computed, response["headers"].get('etag', '').strip('"\''))) ## when using KMS encryption, MD5 etag value will not match md5_from_s3 = response["headers"].get("etag", "").strip('"\'') - if ('-' not in md5_from_s3) and (md5_from_s3 != md5_hash.hexdigest()) and response["headers"].get("x-amz-server-side-encryption") != 'aws:kms': + if (('-' not in md5_from_s3) and + (md5_from_s3 != md5_hash.hexdigest()) and + response["headers"].get("x-amz-server-side-encryption") != 'aws:kms' and + response["headers"].get("x-amz-server-side​-encryption​-customer-key-md5") == ""): warning("MD5 Sums don't match!") if retries: warning("Retrying upload of %s" % (filename)) @@ -1821,7 +1831,9 @@ def recv_file(self, request, stream, labels, start_position = 0, retries = _max_ start_position + int(response["headers"]["content-length"]), response["size"])) debug("ReceiveFile: Computed MD5 = %s" % response.get("md5")) # avoid ETags from multipart uploads that aren't the real md5 - if ('-' not in md5_from_s3 and not response["md5match"]) and (response["headers"].get("x-amz-server-side-encryption") != 'aws:kms'): + if (('-' not in md5_from_s3 and not response["md5match"]) and + response["headers"].get("x-amz-server-side-encryption") != 'aws:kms' and + response["headers"].get("x-amz-server-side​-encryption​-customer-key-md5") == ""): warning("MD5 signatures do not match: computed=%s, received=%s" % ( response.get("md5"), md5_from_s3)) return response diff --git a/s3cmd b/s3cmd index cef1e9c55..60a9e8185 100755 --- a/s3cmd +++ b/s3cmd @@ -548,6 +548,8 @@ def cmd_object_get(args): warning(u"Exiting now because of --dry-run") return EX_OK + extra_headers = copy(cfg.extra_headers) + seq = 0 ret = EX_OK for key in remote_list: @@ -599,7 +601,10 @@ def cmd_object_get(args): continue try: try: - response = s3.object_get(uri, dst_stream, destination, start_position = start_position, extra_label = seq_label) + response = s3.object_get(uri, dst_stream, destination, + start_position=start_position, + extra_headers=extra_headers, + extra_label=seq_label) finally: dst_stream.close() except S3DownloadError as e: @@ -916,6 +921,7 @@ def cmd_mv(args): def cmd_info(args): cfg = Config() s3 = S3(cfg) + extra_headers = copy(cfg.extra_headers) while (len(args)): uri_arg = args.pop(0) @@ -925,7 +931,7 @@ def cmd_info(args): try: if uri.has_object(): - info = s3.object_info(uri) + info = s3.object_info(uri, extra_headers=extra_headers) output(u"%s (object):" % uri.uri()) output(u" File size: %s" % info['headers']['content-length']) output(u" Last mod: %s" % info['headers']['last-modified'])