add support of aws server-side encryption with customer-provided encryption keys (SSE-C)

Signed-off-by: wangyunqing <[email protected]>

Author: inspur-wyq

S3/Config.py

@@ -93,6 +93,7 @@ class Config(object):
     host_base = u"s3.amazonaws.com"
     host_bucket = u"%(bucket)s.s3.amazonaws.com"
     kms_key = u""    #can't set this and Server Side Encryption at the same time
+    customer_key = u""
     # simpledb_host looks useless, legacy? to remove?
     simpledb_host = u"sdb.amazonaws.com"
     cloudfront_host = u"cloudfront.amazonaws.com"
@@ -247,8 +248,10 @@ def __init__(self, configfile = None, access_key=None, secret_key=None, access_t
                     self.role_config()
 
             #TODO check KMS key is valid
-            if self.kms_key and self.server_side_encryption == True:
-                warning('Cannot have server_side_encryption (S3 SSE) and KMS_key set (S3 KMS). KMS encryption will be used. Please set server_side_encryption to False')
+            if (self.kms_key and self.server_side_encryption == True) or \
+               (self.kms_key and self.customer_key) or \
+               (self.customer_key and self.server_side_encryption == True):
+                warning('Cannot have server_side_encryption (S3 SSE) and KMS_key set (S3 KMS) and server_side_encryption_customer (S3 SSE-C). KMS encryption will be used. Please set one')
             if self.kms_key and self.signature_v2 == True:
                 raise Exception('KMS encryption requires signature v4. Please set signature_v2 to False')
 

S3/MultiPart.py

@@ -167,7 +167,8 @@ def upload_part(self, seq, offset, chunk_size, labels, buffer = '', remote_statu
                 warning("MultiPart: size (%d vs %d) does not match for %s part %d, reuploading."
                         % (int(remote_status['size']), chunk_size, self.uri, seq))
 
-        headers = { "content-length": str(chunk_size) }
+        headers = self.headers_baseline
+        headers["content-length"] = str(chunk_size)
         query_string_params = {'partNumber':'%s' % seq,
                                'uploadId': self.upload_id}
         request = self.s3.create_request("OBJECT_PUT", uri = self.uri,
@@ -190,7 +191,8 @@ def complete_multipart_upload(self):
             parts_xml.append(part_xml % (seq, etag))
         body = "<CompleteMultipartUpload>%s</CompleteMultipartUpload>" % ("".join(parts_xml))
 
-        headers = { "content-length": str(len(body)) }
+        headers = self.headers_baseline
+        headers["content-length"] = str(len(body))
         request = self.s3.create_request("OBJECT_POST", uri = self.uri,
                                          headers = headers, body = body,
                                          uri_params = {'uploadId': self.upload_id})

S3/S3.py

@@ -664,6 +664,12 @@ def object_put(self, filename, uri, extra_headers = None, extra_label = ""):
             headers['x-amz-server-side-encryption'] = 'aws:kms'
             headers['x-amz-server-side-encryption-aws-kms-key-id'] = self.config.kms_key
 
+        ## Set server side encryption customer
+        if self.config.customer_key:
+            headers["x-amz-server-side-encryption-customer-algorithm"] = "AES256"
+            headers["x-amz-server-side-encryption-customer-key"] = base64.b64encode(self.config.customer_key)
+            headers["x-amz-server-side-encryption-customer-key-MD5"] = compute_content_md5(self.config.customer_key)
+
         ## MIME-type handling
         headers["content-type"] = self.content_type(filename=filename)
 
@@ -721,7 +727,14 @@ def object_put(self, filename, uri, extra_headers = None, extra_label = ""):
     def object_get(self, uri, stream, dest_name, start_position = 0, extra_label = ""):
         if uri.type != "s3":
             raise ValueError("Expected URI type 's3', got '%s'" % uri.type)
-        request = self.create_request("OBJECT_GET", uri = uri)
+
+        headers = SortedDict(ignore_case = True)
+        if self.config.customer_key:
+            headers["x-amz-server-side-encryption-customer-algorithm"] = "AES256"
+            headers["x-amz-server-side-encryption-customer-key"] = base64.b64encode(self.config.customer_key)
+            headers["x-amz-server-side-encryption-customer-key-MD5"] = compute_content_md5(self.config.customer_key)
+
+        request = self.create_request("OBJECT_GET", uri = uri, headers = headers)
         labels = { 'source' : uri.uri(), 'destination' : dest_name, 'extra' : extra_label }
         response = self.recv_file(request, stream, labels, start_position)
         return response
@@ -843,6 +856,12 @@ def object_copy(self, src_uri, dst_uri, extra_headers = None):
             headers['x-amz-server-side-encryption'] = 'aws:kms'
             headers['x-amz-server-side-encryption-aws-kms-key-id'] = self.config.kms_key
 
+        ## Set server side encryption customer
+        if self.config.customer_key:
+            headers["x-amz-server-side-encryption-customer-algorithm"] = "AES256"
+            headers["x-amz-server-side-encryption-customer-key"] = base64.b64encode(self.config.customer_key)
+            headers["x-amz-server-side-encryption-customer-key-MD5"] = compute_content_md5(self.config.customer_key)
+
         if extra_headers:
             headers.update(extra_headers)
 
@@ -900,6 +919,12 @@ def object_modify(self, src_uri, dst_uri, extra_headers = None):
             headers['x-amz-server-side-encryption'] = 'aws:kms'
             headers['x-amz-server-side-encryption-aws-kms-key-id'] = self.config.kms_key
 
+        ## Set server side encryption customer
+        if self.config.customer_key:
+            headers["x-amz-server-side-encryption-customer-algorithm"] = "AES256"
+            headers["x-amz-server-side-encryption-customer-key"] = base64.b64encode(self.config.customer_key)
+            headers["x-amz-server-side-encryption-customer-key-MD5"] = compute_content_md5(self.config.customer_key)
+
         if extra_headers:
             headers.update(extra_headers)
 

s3cmd

@@ -2717,6 +2717,7 @@ def main():
 
     optparser.add_option(      "--server-side-encryption", dest="server_side_encryption", action="store_true", help="Specifies that server-side encryption will be used when putting objects. [put, sync, cp, modify]")
     optparser.add_option(      "--server-side-encryption-kms-id", dest="kms_key", action="store", help="Specifies the key id used for server-side encryption with AWS KMS-Managed Keys (SSE-KMS) when putting objects. [put, sync, cp, modify]")
+    optparser.add_option(      "--server-side-encryption-customer-key", dest="customer_key", action="store", help="Specifies the key used for server-side encryption with customer-provided encryption keys (SSE-C) when putting objects or get SSE-C object. [put, sync, cp, modify, get]")
 
     optparser.add_option(      "--encoding", dest="encoding", metavar="ENCODING", help="Override autodetected terminal and filesystem encoding (character set). Autodetected: %s" % autodetected_encoding)
     optparser.add_option(      "--add-encoding-exts", dest="add_encoding_exts", metavar="EXTENSIONs", help="Add encoding to these comma delimited extensions i.e. (css,js,html) when uploading to S3 )")