Skip to content

Add ability to handle sse-c with add-header #1004

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 1 commit into
base: master
Choose a base branch
from
Open

Add ability to handle sse-c with add-header #1004

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
24 changes: 18 additions & 6 deletions S3/S3.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -706,10 +706,13 @@ def object_put(self, filename, uri, extra_headers = None, extra_label = ""):
response = self.send_file(request, src_stream, labels)
return response

def object_get(self, uri, stream, dest_name, start_position = 0, extra_label = ""):
def object_get(self, uri, stream, dest_name, start_position = 0, extra_headers = None, extra_label = ""):
headers = SortedDict(ignore_case = True)
if extra_headers:
headers.update(extra_headers)
if uri.type != "s3":
raise ValueError("Expected URI type 's3', got '%s'" % uri.type)
request = self.create_request("OBJECT_GET", uri = uri)
request = self.create_request("OBJECT_GET", uri = uri, headers = headers)
labels = { 'source' : uri.uri(), 'destination' : dest_name, 'extra' : extra_label }
response = self.recv_file(request, stream, labels, start_position)
return response
Expand Down Expand Up @@ -924,8 +927,12 @@ def object_move(self, src_uri, dst_uri, extra_headers = None):
debug("Object '%s' NOT deleted because of an unexepected response data content.", src_uri)
return response_copy

def object_info(self, uri):
request = self.create_request("OBJECT_HEAD", uri = uri)
def object_info(self, uri, extra_headers = None):
headers = SortedDict(ignore_case = True)
if extra_headers:
headers.update(extra_headers)

request = self.create_request("OBJECT_HEAD", uri = uri, headers=headers)
response = self.send_request(request)
return response

Expand Down Expand Up @@ -1583,7 +1590,10 @@ def send_file(self, request, stream, labels, buffer = '', throttle = 0,
debug("MD5 sums: computed=%s, received=%s" % (md5_computed, response["headers"].get('etag', '').strip('"\'')))
## when using KMS encryption, MD5 etag value will not match
md5_from_s3 = response["headers"].get("etag", "").strip('"\'')
if ('-' not in md5_from_s3) and (md5_from_s3 != md5_hash.hexdigest()) and response["headers"].get("x-amz-server-side-encryption") != 'aws:kms':
if (('-' not in md5_from_s3) and
(md5_from_s3 != md5_hash.hexdigest()) and
response["headers"].get("x-amz-server-side-encryption") != 'aws:kms' and
response["headers"].get("x-amz-server-side​-encryption​-customer-key-md5") == ""):
warning("MD5 Sums don't match!")
if retries:
warning("Retrying upload of %s" % (filename))
Expand Down Expand Up @@ -1821,7 +1831,9 @@ def recv_file(self, request, stream, labels, start_position = 0, retries = _max_
start_position + int(response["headers"]["content-length"]), response["size"]))
debug("ReceiveFile: Computed MD5 = %s" % response.get("md5"))
# avoid ETags from multipart uploads that aren't the real md5
if ('-' not in md5_from_s3 and not response["md5match"]) and (response["headers"].get("x-amz-server-side-encryption") != 'aws:kms'):
if (('-' not in md5_from_s3 and not response["md5match"]) and
response["headers"].get("x-amz-server-side-encryption") != 'aws:kms' and
response["headers"].get("x-amz-server-side​-encryption​-customer-key-md5") == ""):
warning("MD5 signatures do not match: computed=%s, received=%s" % (
response.get("md5"), md5_from_s3))
return response
Expand Down
10 changes: 8 additions & 2 deletions s3cmd
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -548,6 +548,8 @@ def cmd_object_get(args):
warning(u"Exiting now because of --dry-run")
return EX_OK

extra_headers = copy(cfg.extra_headers)

seq = 0
ret = EX_OK
for key in remote_list:
Expand Down Expand Up @@ -599,7 +601,10 @@ def cmd_object_get(args):
continue
try:
try:
response = s3.object_get(uri, dst_stream, destination, start_position = start_position, extra_label = seq_label)
response = s3.object_get(uri, dst_stream, destination,
start_position=start_position,
extra_headers=extra_headers,
extra_label=seq_label)
finally:
dst_stream.close()
except S3DownloadError as e:
Expand Down Expand Up @@ -916,6 +921,7 @@ def cmd_mv(args):
def cmd_info(args):
cfg = Config()
s3 = S3(cfg)
extra_headers = copy(cfg.extra_headers)

while (len(args)):
uri_arg = args.pop(0)
Expand All @@ -925,7 +931,7 @@ def cmd_info(args):

try:
if uri.has_object():
info = s3.object_info(uri)
info = s3.object_info(uri, extra_headers=extra_headers)
output(u"%s (object):" % uri.uri())
output(u" File size: %s" % info['headers']['content-length'])
output(u" Last mod: %s" % info['headers']['last-modified'])
Expand Down