Skip to content

Fixed flush_namespace and using namespace for cache json #44

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
tuxskar wants to merge 2 commits into
base: master
Choose a base branch
from
Open

Fixed flush_namespace and using namespace for cache json #44

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
a5fdb25
This fixes vivekn/redis-simple-cache#32, updated flush_namespace to r…
tuxskar Mar 4, 2017
6706bd8
This fixes vivekn/redis-simple-cache#43, using namespaces to cache json
tuxskar Mar 4, 2017
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
64 changes: 40 additions & 24 deletions redis_cache/rediscache.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ class RedisConnect(object):
This makes the Simple Cache class a little more flexible, for cases
where redis connection configuration needs customizing.
"""

def __init__(self, host=None, port=None, db=None, password=None):
self.host = host if host else 'localhost'
self.port = port if port else 6379
Expand All @@ -36,7 +37,7 @@ def connect(self):
raise RedisNoConnException("Failed to create connection to redis",
(self.host,
self.port)
)
)
return redis.StrictRedis(host=self.host,
port=self.port,
db=self.db,
Expand Down Expand Up @@ -90,21 +91,34 @@ def __init__(self,
port=self.port,
db=self.db,
password=password).connect()
except RedisNoConnException, e:
except RedisNoConnException as e:
self.connection = None
pass

# Should we hash keys? There is a very small risk of collision invloved.
# Should we hash keys? There is a very small risk of collision involved.
self.hashkeys = hashkeys

@property
def namespace_prefix(self):
return "SimpleCache-{0}".format(self.prefix)

def make_key(self, key):
return "SimpleCache-{0}:{1}".format(self.prefix, key)
return "{0}:{1}".format(self.namespace_prefix, key)

def namespace_key(self, namespace):
return self.make_key(namespace + ':*')

def get_set_name(self):
return "SimpleCache-{0}-keys".format(self.prefix)
return "{0}-keys".format(self.namespace_prefix)

def get_key_without_namespace(self, key_with_namespace):
"""
Removes the prefix of the namespace from the namespace_key as parameter
:param key_with_namespace: '<namespace>:<key>'
:return: String
"""
namespace_prefix = '{}:'.format(self.namespace_prefix)
return key_with_namespace[len(namespace_prefix):]

def store(self, key, value, expire=None):
"""
Expand Down Expand Up @@ -134,7 +148,6 @@ def store(self, key, value, expire=None):
pipe.sadd(set_name, key)
pipe.execute()


def expire_all_in_set(self):
"""
Method expires all keys in the namespace of this object.
Expand All @@ -147,7 +160,7 @@ def expire_all_in_set(self):
:return: int, int
"""
all_members = self.keys()
keys = [self.make_key(k) for k in all_members]
keys = [self.make_key(k) for k in all_members]

with self.connection.pipeline() as pipe:
pipe.delete(*keys)
Expand Down Expand Up @@ -182,11 +195,11 @@ def isexpired(self, key):
:return: bool (True) if expired, or int representing current time-to-live (ttl) value
"""
ttl = self.connection.pttl("SimpleCache-{0}".format(key))
if ttl == -2: # not exist
if ttl == -2: # not exist
ttl = self.connection.pttl(self.make_key(key))
elif ttl == -1:
return True
if not ttl is None:
if ttl is not None:
return ttl
else:
return self.connection.pttl("{0}:{1}".format(self.prefix, key))
Expand All @@ -202,7 +215,7 @@ def get(self, key):
if key: # No need to validate membership, which is an O(1) operation, but seems we can do without.
value = self.connection.get(self.make_key(key))
if value is None: # expired key
if not key in self: # If key does not exist at all, it is a straight miss.
if key not in self: # If key does not exist at all, it is a straight miss.
raise CacheMissException

self.connection.srem(self.get_set_name(), key)
Expand Down Expand Up @@ -267,16 +280,15 @@ def __iter__(self):
return iter([])
return iter(
["{0}:{1}".format(self.prefix, x)
for x in self.connection.smembers(self.get_set_name())
])
for x in self.connection.smembers(self.get_set_name())
])

def __len__(self):
return self.connection.scard(self.get_set_name())

def keys(self):
return self.connection.smembers(self.get_set_name())


def flush(self):
keys = list(self.keys())
keys.append(self.get_set_name())
Expand All @@ -287,10 +299,11 @@ def flush(self):
def flush_namespace(self, space):
namespace = self.namespace_key(space)
setname = self.get_set_name()
keys = list(self.connection.keys(namespace))
keys_with_namespace = list(self.connection.keys(namespace))
keys = map(self.get_key_without_namespace, keys_with_namespace)
with self.connection.pipeline() as pipe:
pipe.delete(*keys)
pipe.srem(setname, *space)
pipe.delete(*keys_with_namespace)
pipe.srem(setname, *keys)
pipe.execute()

def get_hash(self, args):
Expand All @@ -310,8 +323,9 @@ def cache_it(limit=10000, expire=DEFAULT_EXPIRY, cache=None,
:param cache: SimpleCache object, if created separately
:return: decorated function
"""
cache_ = cache ## Since python 2.x doesn't have the nonlocal keyword, we need to do this
expire_ = expire ## Same here.
cache_ = cache # Since python 2.x doesn't have the nonlocal keyword, we need to do this
expire_ = expire # Same here.

def decorator(function):
cache, expire = cache_, expire_
if cache is None:
Expand All @@ -323,7 +337,7 @@ def decorator(function):

@wraps(function)
def func(*args, **kwargs):
## Handle cases where caching is down or otherwise not available.
# Handle cases where caching is down or otherwise not available.
if cache.connection is None:
result = function(*args, **kwargs)
return result
Expand All @@ -332,8 +346,8 @@ def func(*args, **kwargs):
fetcher = cache.get_json if use_json else cache.get_pickle
storer = cache.store_json if use_json else cache.store_pickle

## Key will be either a md5 hash or just pickle object,
## in the form of `function name`:`key`
# Key will be either a md5 hash or just pickle object,
# in the form of `function name`:`key`
key = cache.get_hash(serializer.dumps([args, kwargs]))
cache_key = '{func_name}:{key}'.format(func_name=function.__name__,
key=key)
Expand All @@ -345,7 +359,7 @@ def func(*args, **kwargs):
try:
return fetcher(cache_key)
except (ExpiredKeyException, CacheMissException) as e:
## Add some sort of cache miss handing here.
# Add some sort of cache miss handing here.
pass
except:
logging.exception("Unknown redis-simple-cache error. Please check your Redis free space.")
Expand All @@ -361,9 +375,10 @@ def func(*args, **kwargs):
logging.exception(e)

return result

return func
return decorator

return decorator


def cache_it_json(limit=10000, expire=DEFAULT_EXPIRY, cache=None, namespace=None):
Expand All @@ -372,10 +387,11 @@ def cache_it_json(limit=10000, expire=DEFAULT_EXPIRY, cache=None, namespace=None
:param limit: maximum number of keys to maintain in the set
:param expire: period after which an entry in cache is considered expired
:param cache: SimpleCache object, if created separately
:param namespace: redis namespace to store the json
:return: decorated function
"""
return cache_it(limit=limit, expire=expire, use_json=True,
cache=cache, namespace=None)
cache=cache, namespace=namespace)


def to_unicode(obj, encoding='utf-8'):
Expand Down
30 changes: 20 additions & 10 deletions redis_cache/test_rediscache.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,10 +1,12 @@
#SimpleCache Tests
#~~~~~~~~~~~~~~~~~~~
# SimpleCache Tests
# ~~~~~~~~~~~~~~~~~~~
from datetime import timedelta
from rediscache import SimpleCache, RedisConnect, cache_it, cache_it_json, CacheMissException, ExpiredKeyException, DoNotCache
from rediscache import SimpleCache, RedisConnect, cache_it, cache_it_json, CacheMissException, ExpiredKeyException, \
DoNotCache
from unittest import TestCase, main
import time


class ComplexNumber(object): # used in pickle test
def __init__(self, real, imag):
self.real = real
Expand All @@ -15,11 +17,11 @@ def __eq__(self, other):


class SimpleCacheTest(TestCase):

def setUp(self):
self.c = SimpleCache(10) # Cache that has a maximum limit of 10 keys
self.assertIsNotNone(self.c.connection)
self.redis = RedisConnect().connect()

def test_expire(self):
quick_c = SimpleCache()

Expand All @@ -40,6 +42,7 @@ def test_kwargs_decorator(self):
@cache_it_json(cache=self.c)
def add_it(a, b=10, c=5):
return a + b + c

add_it(3)
self.assertEqual(add_it(3), 18)
add_it(5, b=7)
Expand All @@ -58,17 +61,19 @@ def test_json(self):
self.assertEqual(self.c.get_json("json"), payload)

def test_pickle(self):
payload = ComplexNumber(3,4)
payload = ComplexNumber(3, 4)
self.c.store_pickle("pickle", payload)
self.assertEqual(self.c.get_pickle("pickle"), payload)

def test_decorator(self):
self.redis.flushall()
mutable = []

@cache_it(cache=self.c)
def append(n):
mutable.append(n)
return mutable

append(1)
len_before = len(mutable)
mutable_cached = append(1)
Expand Down Expand Up @@ -142,10 +147,12 @@ def test_decorator_json(self):
import random

mutable = {}

@cache_it_json(cache=self.c)
def set_key(n):
mutable[str(random.random())] = n
return mutable

set_key('a')
len_before = len(mutable)
mutable_cached = set_key('a')
Expand All @@ -160,11 +167,12 @@ def test_decorator_complex_type(self):
@cache_it(cache=self.c)
def add(x, y):
return ComplexNumber(x.real + y.real, x.imag + y.imag)
result = add(ComplexNumber(3,4), ComplexNumber(4,5))
result_cached = add(ComplexNumber(3,4), ComplexNumber(4,5))

result = add(ComplexNumber(3, 4), ComplexNumber(4, 5))
result_cached = add(ComplexNumber(3, 4), ComplexNumber(4, 5))
self.assertNotEqual(id(result), id(result_cached))
self.assertEqual(result, result_cached)
self.assertEqual(result, complex(3,4) + complex(4,5))
self.assertEqual(result, complex(3, 4) + complex(4, 5))

def test_cache_limit(self):
for i in range(100):
Expand All @@ -189,7 +197,7 @@ def test_flush(self):
connection.delete("will_not_be_deleted")

def test_flush_namespace(self):
self.redis.flushall()
self.redis.flushall()
self.c.store("foo:one", "bir")
self.c.store("foo:two", "bor")
self.c.store("fii", "bur")
Expand Down Expand Up @@ -295,4 +303,6 @@ def test_invalidate_key(self):
def tearDown(self):
self.c.flush()

main()

if __name__ == '__main__':
main()