update 3.25.8

Author: zhangtingwei998

README.md

@@ -1,3 +1,13 @@
+Version 3.25.8
+
+New features:
+
+When new ObsClient with ECS agency, the more secure IMDSv2 interface will be prioritized for interacting with ECS services.
+
+Fix problem:
+
+1. Fix the issue where the ObsClient.downloadFile interface cannot retry after enabling crc64, if the verification fails.
+-------------------------------------------------------------------------------------------------
 Version 3.25.3
 
 New Features:

README_CN.md

@@ -1,4 +1,14 @@
-Version 3.25.3
+Version 3.25.8
+
+新特性：
+
+1. 使用ECS委托初始化ObsClient时会优先使用更安全的IMDSv2接口与ECS服务交互。
+
+修复问题:
+
+1. 修复ObsClient.downloadFile接口开启crc64和断点续传后，如果校验失败，无法重试的问题。
+-------------------------------------------------------------------------------------------------
+Version 3.25.3
 
 新特性：
 

src/obs/__init__.py

@@ -23,7 +23,7 @@
 from obs.model import Options, PutObjectHeader, AppendObjectHeader, AppendObjectContent, RedirectAllRequestTo
 from obs.model import Redirect, RoutingRule, Tag, TagInfo, Transition, NoncurrentVersionTransition, Rule, Versions, AbortIncompleteMultipartUpload
 from obs.model import Object, WebsiteConfiguration, Logging, CompleteMultipartUploadRequest, DeleteObjectsRequest,CustomDomainConfiguration
-from obs.model import ListMultipartUploadsRequest, GetObjectRequest, UploadFileHeader, Payer
+from obs.model import ListMultipartUploadsRequest, GetObjectRequest, UploadFileHeader, Payer, ClientVerify
 from obs.model import ExtensionHeader, FetchStatus, BucketAliasModel, ListBucketAliasModel
 from obs.workflow import WorkflowClient
 from obs.crypto_client import CryptoObsClient
@@ -97,5 +97,6 @@
     'CtrRSACipherGenerator',
     'BucketAliasModel',
     'ListBucketAliasModel',
-    'CustomDomainConfiguration'
+    'CustomDomainConfiguration',
+    'ClientVerify'
 ]

src/obs/client.py

@@ -199,9 +199,9 @@ def __getattr__(self, item):
 
 class _BasicClient(object):
     def __init__(self, access_key_id='', secret_access_key='', is_secure=True, server=None,
-                 signature='obs', region='region', path_style=False, ssl_verify=False,
+                 signature='obs', region='region', path_style=False, ssl_verify=False, is_use_gmssl=False,
                  port=None, max_retry_count=3, timeout=60, chunk_size=const.READ_ONCE_LENGTH,
-                 long_conn_mode=False, proxy_host=None, proxy_port=None,
+                 long_conn_mode=False, proxy_host=None, proxy_port=None, client_verify=None,
                  proxy_username=None, proxy_password=None, security_token=None,
                  custom_ciphers=None, use_http2=False, is_signature_negotiation=True, is_cname=False,
                  max_redirect_count=10, security_providers=None, security_provider_policy=None, client_mode='obs',
@@ -239,7 +239,8 @@ def __init__(self, access_key_id='', secret_access_key='', is_secure=True, serve
         self.is_signature_negotiation = is_signature_negotiation
         self.is_cname = is_cname
         self.max_redirect_count = max_redirect_count
-
+        self.is_use_gmssl = is_use_gmssl
+        self.client_verify = client_verify
         if client_mode == 'obs':
             if self.path_style or self.is_cname:
                 self.is_signature_negotiation = False
@@ -364,30 +365,45 @@ def _init_connHolder(self):
         self.connHolder = {'connSet': Queue(), 'lock': threading.Lock()}
 
     def _init_ssl_context(self, custom_ciphers):
-        try:
-            import ssl
-            if hasattr(ssl, 'SSLContext'):
+        import ssl
+        if hasattr(ssl, 'SSLContext'):
+            if self.is_use_gmssl:
+                if not hasattr(ssl, 'PROTOCOL_GMTLS'):
+                    raise Exception('ssl not support PROTOCOL_GMTLS.')
+                context = ssl.SSLContext(ssl.PROTOCOL_GMTLS)
+                context.set_ciphers('ECC-SM4-SM3:ECDHE-SM4-SM3')
+            else:
                 context = ssl.SSLContext(ssl.PROTOCOL_SSLv23)
-                context.options |= ssl.OP_NO_SSLv2
-                context.options |= ssl.OP_NO_SSLv3
-                if custom_ciphers is not None:
-                    custom_ciphers = util.to_string(custom_ciphers).strip()
-                    if custom_ciphers != '' and hasattr(context, 'set_ciphers') and callable(context.set_ciphers):
-                        context.set_ciphers(custom_ciphers)
-                if self.ssl_verify:
-                    import _ssl
-                    cafile = util.to_string(self.ssl_verify)
-                    context.options |= getattr(_ssl, "OP_NO_COMPRESSION", 0)
-                    context.verify_mode = ssl.CERT_REQUIRED
-                    if os.path.isfile(cafile):
-                        context.load_verify_locations(cafile)
-                else:
-                    context.verify_mode = ssl.CERT_NONE
-                if hasattr(context, 'check_hostname'):
-                    context.check_hostname = False
-                self.context = context
-        except Exception:
-            print(traceback.format_exc())
+            context.options |= ssl.OP_NO_SSLv2
+            context.options |= ssl.OP_NO_SSLv3
+            if custom_ciphers is not None:
+                custom_ciphers = util.to_string(custom_ciphers).strip()
+                if custom_ciphers != '' and hasattr(context, 'set_ciphers') and callable(context.set_ciphers):
+                    context.set_ciphers(custom_ciphers)
+            if self.ssl_verify:
+                import _ssl
+                cafile = util.to_string(self.ssl_verify)
+                context.options |= getattr(_ssl, "OP_NO_COMPRESSION", 0)
+                context.verify_mode = ssl.CERT_REQUIRED
+                if os.path.isfile(cafile):
+                    context.load_verify_locations(cafile)
+            else:
+                context.verify_mode = ssl.CERT_NONE
+            is_client_sign_verify = self.client_verify and self.client_verify.clientCert and \
+                                    self.client_verify.clientKey
+            is_client_enc_verify = self.client_verify and self.client_verify.clientEncCert and \
+                                   self.client_verify.clientEncKey
+            if is_client_sign_verify:
+                context.load_cert_chain(certfile=util.to_string(self.client_verify.clientCert),
+                                        keyfile=util.to_string(self.client_verify.clientKey),
+                                        password=util.to_string(self.client_verify.clientKeyPassword))
+                if is_client_enc_verify:
+                    context.load_cert_chain(certfile=util.to_string(self.client_verify.clientEncCert),
+                                            keyfile=util.to_string(self.client_verify.clientEncKey),
+                                            password=util.to_string(self.client_verify.clientEncKeyPassword))
+            if hasattr(context, 'check_hostname'):
+                context.check_hostname = False
+            self.context = context
 
     def close(self):
         if self.connHolder is not None:
@@ -502,7 +518,7 @@ def _make_request_with_retry(self, methodType, bucketName, objectKey=None, pathA
                 if flag >= self.max_retry_count or readable:
                     return self._make_error_result(e, ret)
                 flag += 1
-                time.sleep(math.pow(2, flag) * 0.05)
+                time.sleep(math.pow(2, flag) * 0.1)
                 self.log_client.log(WARNING, 'request again, time:%d' % int(flag))
                 continue
 
@@ -824,7 +840,9 @@ def _parse_content(self, objectKey, conn, response, download_start='',
                     result_wrapper = ResponseWrapper(conn, response, self.connHolder, content_length, notifier, obs_crc64=obs_crc64)
                     self.log_client.log(DEBUG, 'CRC64 from the server is {0}'.format(obs_crc64))
                 else:
-                    raise Exception('No CRC64 is obtained from the server.')
+                    result_wrapper = ResponseWrapper(conn, response, self.connHolder, content_length, notifier)
+                    self.log_client.log(WARNING, 'object {0} not get CRC64 from the server.'.format(objectKey))
+
             else:
                 result_wrapper = ResponseWrapper(conn, response, self.connHolder, content_length, notifier)
             if loadStreamInMemory:
@@ -1248,9 +1266,7 @@ def _createPostSignature(self, bucketName=None, objectKey=None, expires=300, for
         if matchAnyKey:
             policy.append('["starts-with", "$key", ""],')
 
-        policy.append(']}')
-
-        originPolicy = ''.join(policy)
+        originPolicy = ''.join(policy).rstrip(',') + ']}'
 
         policy = util.base64_encode(originPolicy)
 

src/obs/const.py

@@ -101,7 +101,7 @@
 DEFAULT_TASK_QUEUE_SIZE = 20000
 CONNECTION_POOL_SIZE = 10
 
-OBS_SDK_VERSION = '3.25.3'
+OBS_SDK_VERSION = '3.25.8'
 
 V2_META_HEADER_PREFIX = 'x-amz-meta-'
 V2_HEADER_PREFIX = 'x-amz-'

src/obs/convertor.py

@@ -641,6 +641,14 @@ def trans_custom_domain_cert(self, certificate, domainName):
             if certificate.get("certificateChain", None):
                 ET.SubElement(root, "CertificateChain").text = util.to_string(certificate.get("certificateChain", None))
             ET.SubElement(root, "PrivateKey").text = util.to_string(certificate.get("privateKey"))
+            if certificate.get("certificateType", None):
+                ET.SubElement(root, "CertificateType").text = util.to_string(certificate.get("certificateType", None))
+            if certificate.get("encCertificate", None):
+                ET.SubElement(root, "ENCCertificate").text = util.to_string(certificate.get("encCertificate", None))
+            if certificate.get("encPrivateKey", None):
+                ET.SubElement(root, "ENCPrivateKey").text = util.to_string(certificate.get("encPrivateKey", None))
+            if util.to_string(certificate.get("deleteCertificate", None)):
+                ET.SubElement(root, "DeleteCertificate").text = util.to_string(certificate.get("deleteCertificate", None))
             entity = ET.tostring(root, "UTF-8")
             if len(entity) > const.MAX_CERT_XML_BODY_SIZE:
                 error_message = "XML body size exceeds {} KB limit".format(const.MAX_CERT_XML_BODY_SIZE / 1024)
@@ -1336,7 +1344,12 @@ def parseGetBucketCustomDomain(self, xml, headers=None):
             d = self._find_item(domain, "CreateTime")
             certificate_id = self._find_item(domain, "CertificateId")
             create_time = DateTime.UTCToLocal(d)
-            curr_bucket = BucketCustomDomain(domain_name=domain_name,create_time=create_time,certificate_id=certificate_id)
+            name = self._find_item(domain, "Name")
+            certificate_type = self._find_item(domain, "CertificateType")
+            e = self._find_item(domain, "ExpiredTime")
+            expired_time = DateTime.UTCToLocal(e)
+            curr_bucket = BucketCustomDomain(domainName=domain_name, createTime=create_time, certificateId=certificate_id,
+                                             name=name, certificateType=certificate_type, expiredTime=expired_time)
             entries.append(curr_bucket)
 
         return ListBucketCustomDomainsResponse(domains=entries)