diff --git a/s3cmd b/s3cmd index 77b4e74f..4ad686d0 100755 --- a/s3cmd +++ b/s3cmd @@ -192,12 +192,6 @@ def subcmd_bucket_list(s3, uri, limit): debug(u"Bucket 's3://%s':" % bucket) if prefix.endswith('*'): prefix = prefix[:-1] - try: - response = s3.bucket_list(bucket, prefix = prefix, limit = limit) - except S3Error as e: - if e.info["Code"] in S3.codes: - error(S3.codes[e.info["Code"]] % bucket) - raise # md5 are 32 char long, but for multipart there could be a suffix if Config().human_readable_sizes: @@ -214,38 +208,51 @@ def subcmd_bucket_list(s3, uri, limit): else: format_string = u"%(timestamp)16s %(size)s %(uri)s" - for prefix in response['common_prefixes']: - output(format_string % { - "timestamp": "", - "size": dir_str, - "md5": "", - "storageclass": "", - "uri": uri.compose_uri(bucket, prefix["Prefix"])}) - - for object in response["list"]: - md5 = object.get('ETag', '').strip('"\'') - storageclass = object.get('StorageClass','') - - if cfg.list_md5: - if '-' in md5: # need to get md5 from the object - object_uri = uri.compose_uri(bucket, object["Key"]) - info_response = s3.object_info(S3Uri(object_uri)) - try: - md5 = info_response['s3cmd-attrs']['md5'] - except KeyError: - pass + truncated = False + common_prefixes = [] + + try: + for _trunkated, _common_prefixes, _objects in s3.bucket_list_streaming(bucket, prefix = prefix, limit = limit): + truncated = _trunkated + + if common_prefixes != _common_prefixes: + common_prefixes = _common_prefixes + for prefix in common_prefixes: + output(format_string % { + "timestamp": "", + "size": dir_str, + "md5": "", + "storageclass": "", + "uri": uri.compose_uri(bucket, prefix["Prefix"])}) + + for object in _objects: + md5 = object.get('ETag', '').strip('"\'') + storageclass = object.get('StorageClass','') + + if cfg.list_md5: + if '-' in md5: # need to get md5 from the object + object_uri = uri.compose_uri(bucket, object["Key"]) + info_response = s3.object_info(S3Uri(object_uri)) + try: + md5 = info_response['s3cmd-attrs']['md5'] + except KeyError: + pass + + size_and_coeff = formatSize(object["Size"], + Config().human_readable_sizes) + output(format_string % { + "timestamp": formatDateTime(object["LastModified"]), + "size" : format_size % size_and_coeff, + "md5" : md5, + "storageclass" : storageclass, + "uri": uri.compose_uri(bucket, object["Key"]), + }) + except S3Error as e: + if e.info["Code"] in S3.codes: + error(S3.codes[e.info["Code"]] % bucket) + raise - size_and_coeff = formatSize(object["Size"], - Config().human_readable_sizes) - output(format_string % { - "timestamp": formatDateTime(object["LastModified"]), - "size" : format_size % size_and_coeff, - "md5" : md5, - "storageclass" : storageclass, - "uri": uri.compose_uri(bucket, object["Key"]), - }) - - if response["truncated"]: + if truncated: warning(u"The list is truncated because the settings limit was reached.") def cmd_bucket_create(args):