Add support for SSE-C encryption

Changes implement 2 new flags --sse-customer-key and
--sse-copy-source-customer-key that can be used by user to
provide a key for server side encryption.

Once these options are set extra headers are added to request
accordingly to SSE-C specification [1]

This PR squashes and rebases on current master changes
implemented by @jheller

[1] https://docs.aws.amazon.com/AmazonS3/latest/userguide/specifying-s3-c-encryption.html

Author: jheller

S3/Config.py

@@ -132,6 +132,7 @@ class Config(object):
     extra_headers = SortedDict(ignore_case = True)
     force = False
     server_side_encryption = False
+    sse_customer_key = ""
     enable = None
     get_continue = False
     put_continue = False

S3/FileDict.py

@@ -43,7 +43,7 @@ def get_md5(self, relative_file):
         if 'md5' in self[relative_file]:
             return self[relative_file]['md5']
         md5 = self.get_hardlink_md5(relative_file)
-        if md5 is None and 'md5' in cfg.sync_checks:
+        if md5 is None and 'md5' in cfg.preserve_attrs_list:
             logging.debug(u"doing file I/O to read md5 of %s" % relative_file)
             md5 = Utils.hash_file_md5(self[relative_file]['full_name'])
         self.record_md5(relative_file, md5)

S3/FileLists.py

@@ -552,6 +552,23 @@ def _compare(src_list, dst_lst, src_remote, dst_remote, file):
                 attribs_match = False
                 debug(u"XFER: %s (md5 mismatch: src=%s dst=%s)" % (file, src_md5, dst_md5))
 
+        # Check mtime. This compares local mtime to the upload time of remote file
+        compare_mtime = 'mtime' in cfg.sync_checks
+        if attribs_match and compare_mtime:
+            try:
+                src_mtime = src_list[file]['mtime']
+                dst_mtime = dst_list[file]['timestamp']
+            except (IOError,OSError):
+                # mtime sum verification failed - ignore that file altogether
+                debug(u"IGNR: %s (disappeared)" % (file))
+                warning(u"%s: file disappeared, ignoring." % (file))
+                raise
+
+            if src_mtime > dst_mtime:
+                ## checksums are different.
+                attribs_match = False
+                debug(u"XFER: %s (mtime newer than last upload: src=%s dst=%s)" % (file, src_mtime, dst_mtime))
+
         return attribs_match
 
     # we don't support local->local sync, use 'rsync' or something like that instead ;-)

S3/S3.py

@@ -1828,19 +1828,32 @@ def send_file(self, request, stream, labels, buffer = '', throttle = 0,
             ## Non-recoverable error
             raise S3Error(response)
 
-        debug("MD5 sums: computed=%s, received=%s" % (md5_computed, response["headers"].get('etag', '').strip('"\'')))
-        ## when using KMS encryption, MD5 etag value will not match
-        md5_from_s3 = response["headers"].get("etag", "").strip('"\'')
-        if ('-' not in md5_from_s3) and (md5_from_s3 != md5_hash.hexdigest()) and response["headers"].get("x-amz-server-side-encryption") != 'aws:kms':
-            warning("MD5 Sums don't match!")
-            if retries:
-                warning("Retrying upload of %s" % (filename))
-                return self.send_file(request, stream, labels, buffer, throttle,
-                                      retries - 1, offset, chunk_size, use_expect_continue)
+        if self.config.sse_customer_key:
+            if response["headers"]["x-amz-server-side-encryption-customer-key-md5"] != \
+                self.config.extra_headers["x-amz-server-side-encryption-customer-key-md5"]:
+                warning("MD5 of customer key don't match!")
+                if retries:
+                    warning("Retrying upload of %s" % (filename))
+                    return self.send_file(request, stream, labels, buffer, throttle, retries - 1, offset, chunk_size)
+                else:
+                    warning("Too many failures. Giving up on '%s'" % (filename))
+                    raise S3UploadError
             else:
-                warning("Too many failures. Giving up on '%s'" % (filename))
-                raise S3UploadError("Too many failures. Giving up on '%s'"
-                                    % filename)
+                debug("Match of x-amz-server-side-encryption-customer-key-md5")
+        else:
+            debug("MD5 sums: computed=%s, received=%s" % (md5_computed, response["headers"].get('etag', '').strip('"\'')))
+            ## when using KMS encryption, MD5 etag value will not match
+            md5_from_s3 = response["headers"].get("etag", "").strip('"\'')
+            if ('-' not in md5_from_s3) and (md5_from_s3 != md5_hash.hexdigest()) and response["headers"].get("x-amz-server-side-encryption") != 'aws:kms':
+                warning("MD5 Sums don't match!")
+                if retries:
+                    warning("Retrying upload of %s" % (filename))
+                    return self.send_file(request, stream, labels, buffer, throttle,
+                                        retries - 1, offset, chunk_size, use_expect_continue)
+                else:
+                    warning("Too many failures. Giving up on '%s'" % (filename))
+                    raise S3UploadError("Too many failures. Giving up on '%s'"
+                                        % filename)
 
         return response
 

s3cmd

@@ -30,9 +30,11 @@ if sys.version_info < (2, 6):
 
 PY3 = (sys.version_info >= (3, 0))
 
+import base64
 import codecs
 import errno
 import glob
+import hashlib
 import io
 import locale
 import logging
@@ -1920,6 +1922,15 @@ def cmd_sync_local2remote(args):
         error(u"or disable encryption with --no-encrypt parameter.")
         sys.exit(EX_USAGE)
 
+    # Disable md5 checks if using SSE-C. Add mtime check
+    if cfg.sse_customer_key:
+        try:
+            cfg.sync_checks.remove("md5")
+        except Exception:
+            pass
+        if cfg.sync_checks.count("mtime") == 0:
+            cfg.sync_checks.append("mtime")
+
     for arg in args[:-1]:
         if not os.path.exists(deunicodise(arg)):
             raise ParameterError("Invalid source: '%s' is not an existing file or directory" % arg)
@@ -2365,6 +2376,7 @@ def run_configure(config_file, args):
         ("gpg_passphrase", "Encryption password", "Encryption password is used to protect your files from reading\nby unauthorized persons while in transfer to S3"),
         ("gpg_command", "Path to GPG program"),
         ("use_https", "Use HTTPS protocol", "When using secure HTTPS protocol all communication with Amazon S3\nservers is protected from 3rd party eavesdropping. This method is\nslower than plain HTTP, and can only be proxied with Python 2.7 or newer"),
+        ("sse_customer_key", "Encryption key for server-side-encryption with customer key.\nMust be 32 characters"),
         ("proxy_host", "HTTP Proxy server name", "On some networks all internet access must go through a HTTP proxy.\nTry setting it here if you can't connect to S3 directly"),
         ("proxy_port", "HTTP Proxy server port"),
         ]
@@ -2804,6 +2816,8 @@ def main():
 
     optparser.add_option(      "--server-side-encryption", dest="server_side_encryption", action="store_true", help="Specifies that server-side encryption will be used when putting objects. [put, sync, cp, modify]")
     optparser.add_option(      "--server-side-encryption-kms-id", dest="kms_key", action="store", help="Specifies the key id used for server-side encryption with AWS KMS-Managed Keys (SSE-KMS) when putting objects. [put, sync, cp, modify]")
+    optparser.add_option(      "--sse-customer-key", dest="sse_customer_key", action="store", metavar="12345678901234567890123456789012", help="Specifies a customer provided key for server-side encryption. Must be 32 character string.")
+    optparser.add_option(      "--sse-copy-source-customer-key", dest="sse_copy_source_customer_key", action="store", metavar="12345678901234567890123456789012", help="Specifies the encryption key for copying or moving objects with a customer provided key for server-side encryption.")
 
     optparser.add_option(      "--encoding", dest="encoding", metavar="ENCODING", help="Override autodetected terminal and filesystem encoding (character set). Autodetected: %s" % autodetected_encoding)
     optparser.add_option(      "--add-encoding-exts", dest="add_encoding_exts", metavar="EXTENSIONs", help="Add encoding to these comma delimited extensions i.e. (css,js,html) when uploading to S3 )")
@@ -2917,6 +2931,41 @@ def main():
             error(u"Option --progress is not yet supported on MS Windows platform. Assuming --no-progress.")
             cfg.progress_meter = False
 
+    if options.sse_customer_key is not None:
+        if len(options.sse_customer_key) == 32:
+            cfg.sse_customer_key = options.sse_customer_key
+        else:
+            error(u"sse-customer-key must be 32 characters")
+            sys.exit(EX_CONFIG)
+
+    if cfg.sse_customer_key:
+        md5 = hashlib.md5()
+        sse_customer_key = cfg.sse_customer_key.encode()
+        md5.update(sse_customer_key)
+        md5_encoded = base64.b64encode(md5.digest())
+        encoded = base64.b64encode(sse_customer_key)
+        cfg.extra_headers["x-amz-server-side-encryption-customer-algorithm"] = "AES256"
+        cfg.extra_headers["x-amz-server-side-encryption-customer-key"] = encoded.decode()
+        cfg.extra_headers["x-amz-server-side-encryption-customer-key-md5"] = md5_encoded.decode()
+
+        debug(u"Updating Config.Config extra_headers[%s] -> %s" % ("x-amz-server-side-encryption-customer-algorithm", "AES256"))
+        debug(u"Updating Config.Config extra_headers[%s] -> %s" % ("x-amz-server-side-encryption-customer-key", encoded))
+        debug(u"Updating Config.Config extra_headers[%s] -> %s" % ("x-amz-server-side-encryption-customer-key-md5", md5_encoded))
+
+    if options.sse_copy_source_customer_key is not None:
+        md5 = hashlib.md5()
+        sse_copy_source_customer_key = options.sse_copy_source_customer_key.encode()
+        md5.update(sse_copy_source_customer_key)
+        md5_encoded = base64.b64encode(md5.digest())
+        encoded = base64.b64encode(sse_copy_source_customer_key)
+        cfg.extra_headers["x-amz-copy-source-server-side-encryption-customer-algorithm"] = "AES256"
+        cfg.extra_headers["x-amz-copy-source-server-side-encryption-customer-key"] = encoded.decode()
+        cfg.extra_headers["x-amz-copy-source-server-side-encryption-customer-key-md5"] = md5_encoded.decode()
+
+        debug(u"Updating Config.Config extra_headers[%s] -> %s" % ("x-amz-copy-source-server-side-encryption-customer-algorithm", "AES256"))
+        debug(u"Updating Config.Config extra_headers[%s] -> %s" % ("x-amz-copy-source-server-side-encryption-customer-key", encoded))
+        debug(u"Updating Config.Config extra_headers[%s] -> %s" % ("x-amz-copy-source-server-side-encryption-customer-key-md5", md5_encoded))
+
     ## Pre-process --add-header's and put them to Config.extra_headers SortedDict()
     if options.add_header:
         for hdr in options.add_header:

s3cmd.1

@@ -417,6 +417,14 @@ Specifies the key id used for server\-side encryption
 with AWS KMS\-Managed Keys (SSE\-KMS) when putting
 objects. [put, sync, cp, modify]
 .TP
+\fB\-\-sse\-customer\-key\fR=12345678901234567890123456789012
+Specifies a customer key for server-side encryption, to be used
+when putting objects. Must be 32 characters.
+.TP
+\fB\-\-sse\-copy\-source\-customer\-key\fR=12345678901234567890123456789012
+Specifies the key for copying objects with server-side
+encryption customer key. Must be 32 characters.
+.TP
 \fB\-\-encoding\fR=ENCODING
 Override autodetected terminal and filesystem encoding
 (character set). Autodetected: UTF\-8