Added --keyprefix option

whisperbackup/noop.py

@@ -23,8 +23,7 @@
 class NoOP(object):
 
     def __init__(self, bucket, noop):
-        """Setup the S3 storage backend with the bucket we will use and
-           optional region."""
+        """Setup the S3 storage backend with the bucket we will use."""
         self.bucket = bucket
         self.noop = noop
 

whisperbackup/s3.py

@@ -23,40 +23,51 @@
 
 logger = logging.getLogger(__main__.__name__)
 
+
 class S3(object):
+    """AWS S3 storage object
+
+    References:
 
-    def __init__(self, bucket, region="us-east-1", noop=False):
-        """Setup the S3 storage backend with the bucket we will use and
-           optional region."""
+    - http://boto.readthedocs.org/en/latest/ref/boto.html#boto.connect_s3
+    - http://boto.readthedocs.org/en/latest/ref/s3.html
+    """
+
+    def __init__(self, bucket, key_prefix=None, noop=False):
+        """Setup the S3 storage backend with the bucket we will use."""
         self.conn = boto.connect_s3()
         self.bucket = bucket
+        self.key_prefix = key_prefix
         self.noop = noop
-
         b = self.conn.lookup(self.bucket)
         if not noop and b is None:
             # Create the bucket if it doesn't exist
             self.conn.create_bucket(self.bucket)
-
         self.__b = self.conn.get_bucket(self.bucket)
 
     def list(self, prefix=""):
         """Return all keys in this bucket."""
+        prefix = prefix or ""
+        if self.key_prefix:
+            prefix = self.key_prefix + "/" + prefix
         for i in self.__b.list(prefix):
             yield i.key
 
     def get(self, src):
         """Return the contents of src from S3 as a string."""
+        if self.key_prefix:
+            src = self.key_prefix + "/" + src
         if self.__b.get_key(src) is None:
             return None
-
         k = Key(self.__b)
         k.key = src
         return k.get_contents_as_string()
 
     def put(self, dst, data):
         """Store the contents of the string data at a key named by dst
            in S3."""
-
+        if self.key_prefix:
+            dst = self.key_prefix + "/" + dst
         if self.noop:
             logger.info("No-Op Put: %s" % dst)
         else:
@@ -66,7 +77,8 @@ def put(self, dst, data):
 
     def delete(self, src):
         """Delete the object in S3 referenced by the key name src."""
-
+        if self.key_prefix:
+            src = self.key_prefix + "/" + src
         if self.noop:
             logger.info("No-Op Delete: %s" % src)
         else:

whisperbackup/swift.py

@@ -28,8 +28,7 @@
 class Swift(object):
 
     def __init__(self, bucket, noop):
-        """Setup the S3 storage backend with the bucket we will use and
-           optional region."""
+        """Setup the S3 storage backend with the bucket we will use."""
 
         # This is our Swift container
         self.bucket = bucket

whisperbackup/whisperbackup.py

@@ -71,11 +71,9 @@ def storageBackend(script):
         return noop.NoOP(script.options.bucket, script.options.noop)
     if script.args[1].lower() == "s3":
         import s3
-        if len(script.args) > 2:
-            region = script.args[2]
-        else:
-            region = "us-east-1"
-        return s3.S3(script.options.bucket, region, script.options.noop)
+        return s3.S3(script.options.bucket,
+                     script.options.keyprefix,
+                     script.options.noop)
     if script.args[1].lower() == "swift":
         import swift
         return swift.Swift(script.options.bucket, script.options.noop)
@@ -400,6 +398,9 @@ def main():
     options.append(make_option("-b", "--bucket", type="string",
         default="graphite-backups",
         help="The AWS S3 bucket name or Swift container to use, default %default"))
+    options.append(make_option("-k", "--keyprefix", type="string",
+        default=None,
+        help="The AWS S3 bucket key prefix to use - if empty, all files will be put into the root of the bucket %default"))
     options.append(make_option("-m", "--metrics", type="string",
         default="*",
         help="Glob pattern of metric names to backup or restore, default %default"))