diff --git a/whisperbackup/noop.py b/whisperbackup/noop.py index 660c67c..79a3a5e 100644 --- a/whisperbackup/noop.py +++ b/whisperbackup/noop.py @@ -23,8 +23,7 @@ class NoOP(object): def __init__(self, bucket, noop): - """Setup the S3 storage backend with the bucket we will use and - optional region.""" + """Setup the S3 storage backend with the bucket we will use.""" self.bucket = bucket self.noop = noop diff --git a/whisperbackup/s3.py b/whisperbackup/s3.py index a7dc01d..8c5e798 100644 --- a/whisperbackup/s3.py +++ b/whisperbackup/s3.py @@ -23,32 +23,42 @@ logger = logging.getLogger(__main__.__name__) + class S3(object): + """AWS S3 storage object + + References: - def __init__(self, bucket, region="us-east-1", noop=False): - """Setup the S3 storage backend with the bucket we will use and - optional region.""" + - http://boto.readthedocs.org/en/latest/ref/boto.html#boto.connect_s3 + - http://boto.readthedocs.org/en/latest/ref/s3.html + """ + + def __init__(self, bucket, key_prefix=None, noop=False): + """Setup the S3 storage backend with the bucket we will use.""" self.conn = boto.connect_s3() self.bucket = bucket + self.key_prefix = key_prefix self.noop = noop - b = self.conn.lookup(self.bucket) if not noop and b is None: # Create the bucket if it doesn't exist self.conn.create_bucket(self.bucket) - self.__b = self.conn.get_bucket(self.bucket) def list(self, prefix=""): """Return all keys in this bucket.""" + prefix = prefix or "" + if self.key_prefix: + prefix = self.key_prefix + "/" + prefix for i in self.__b.list(prefix): yield i.key def get(self, src): """Return the contents of src from S3 as a string.""" + if self.key_prefix: + src = self.key_prefix + "/" + src if self.__b.get_key(src) is None: return None - k = Key(self.__b) k.key = src return k.get_contents_as_string() @@ -56,7 +66,8 @@ def get(self, src): def put(self, dst, data): """Store the contents of the string data at a key named by dst in S3.""" - + if self.key_prefix: + dst = self.key_prefix + "/" + dst if self.noop: logger.info("No-Op Put: %s" % dst) else: @@ -66,7 +77,8 @@ def put(self, dst, data): def delete(self, src): """Delete the object in S3 referenced by the key name src.""" - + if self.key_prefix: + src = self.key_prefix + "/" + src if self.noop: logger.info("No-Op Delete: %s" % src) else: diff --git a/whisperbackup/swift.py b/whisperbackup/swift.py index 0f2fe82..d0ac759 100644 --- a/whisperbackup/swift.py +++ b/whisperbackup/swift.py @@ -28,8 +28,7 @@ class Swift(object): def __init__(self, bucket, noop): - """Setup the S3 storage backend with the bucket we will use and - optional region.""" + """Setup the S3 storage backend with the bucket we will use.""" # This is our Swift container self.bucket = bucket diff --git a/whisperbackup/whisperbackup.py b/whisperbackup/whisperbackup.py index 60f9b14..631ad60 100644 --- a/whisperbackup/whisperbackup.py +++ b/whisperbackup/whisperbackup.py @@ -71,11 +71,9 @@ def storageBackend(script): return noop.NoOP(script.options.bucket, script.options.noop) if script.args[1].lower() == "s3": import s3 - if len(script.args) > 2: - region = script.args[2] - else: - region = "us-east-1" - return s3.S3(script.options.bucket, region, script.options.noop) + return s3.S3(script.options.bucket, + script.options.keyprefix, + script.options.noop) if script.args[1].lower() == "swift": import swift return swift.Swift(script.options.bucket, script.options.noop) @@ -400,6 +398,9 @@ def main(): options.append(make_option("-b", "--bucket", type="string", default="graphite-backups", help="The AWS S3 bucket name or Swift container to use, default %default")) + options.append(make_option("-k", "--keyprefix", type="string", + default=None, + help="The AWS S3 bucket key prefix to use - if empty, all files will be put into the root of the bucket %default")) options.append(make_option("-m", "--metrics", type="string", default="*", help="Glob pattern of metric names to backup or restore, default %default"))