Skip to content

Commit 973d431

Browse files
Add async support to semantic cache extension (#214)
This PR introduces new async compliant methods to the semantic cache class using lazy index construction. Because the `AsyncSearchIndex` requires an async redis python client, we needed to construct that class lazily upon first usage within the semantic cache class. This PR fixes some unclosed connection errors and is also in support of BerriAI/litellm#5412 at LiteLLM.
1 parent 5f1e046 commit 973d431

File tree

10 files changed

+588
-109
lines changed

10 files changed

+588
-109
lines changed

redisvl/extensions/llmcache/base.py

Lines changed: 28 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,11 @@ def set_ttl(self, ttl: Optional[int] = None):
3131
self._ttl = None
3232

3333
def clear(self) -> None:
34-
"""Clear the LLMCache of all keys in the index."""
34+
"""Clear the cache of all keys in the index."""
35+
raise NotImplementedError
36+
37+
async def aclear(self) -> None:
38+
"""Async clear the cache of all keys in the index."""
3539
raise NotImplementedError
3640

3741
def check(
@@ -41,6 +45,17 @@ def check(
4145
num_results: int = 1,
4246
return_fields: Optional[List[str]] = None,
4347
) -> List[dict]:
48+
"""Check the cache based on a prompt or vector."""
49+
raise NotImplementedError
50+
51+
async def acheck(
52+
self,
53+
prompt: Optional[str] = None,
54+
vector: Optional[List[float]] = None,
55+
num_results: int = 1,
56+
return_fields: Optional[List[str]] = None,
57+
) -> List[dict]:
58+
"""Async check the cache based on a prompt or vector."""
4459
raise NotImplementedError
4560

4661
def store(
@@ -50,7 +65,18 @@ def store(
5065
vector: Optional[List[float]] = None,
5166
metadata: Optional[dict] = {},
5267
) -> str:
53-
"""Stores the specified key-value pair in the cache along with
68+
"""Store the specified key-value pair in the cache along with
69+
metadata."""
70+
raise NotImplementedError
71+
72+
async def astore(
73+
self,
74+
prompt: str,
75+
response: str,
76+
vector: Optional[List[float]] = None,
77+
metadata: Optional[dict] = {},
78+
) -> str:
79+
"""Async store the specified key-value pair in the cache along with
5480
metadata."""
5581
raise NotImplementedError
5682

0 commit comments

Comments
 (0)