Skip to content

Commit

Permalink
Merge pull request #304 from danschultzer/refactor-credentials-cache
Browse files Browse the repository at this point in the history 
Refactor cache stores
  • Loading branch information
@danschultzer
danschultzer authored Oct 19, 2019
2 parents 2606d81 + b5b0566 commit c29269d
Show file tree
Hide file tree
Showing 19 changed files with 947 additions and 439 deletions.
30 changes: 30 additions & 0 deletions CHANGELOG.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,36 @@
* `Pow.Plug.Session` now stores a keyword list with metadata for the session rather than just the timestamp
* `Pow.Phoenix.Router` now only filters routes that has equal number of bindings
* `Pow.Phoenix.Routes.user_not_authenticated_path/1` now only puts the `:request_path` param if the request is using "GET" method
* The stores has been refactored so the command conforms with ETS store. This means that put commands now accept `{key, value}` record element(s), and keys may be list for easier lookup.
* `Pow.Store.Backend.Base` behaviour now requires to;
* Accept `Pow.Store.Backend.Base.record/0` values for `put/2`
* Accept `Pow.Store.Backend.Base.key/0` for `delete/2` and `get/2`
* Implement `all/2`
* Remove `keys/1`
* Remove `put/3`
* `Pow.Store.Backend.EtsCache.keys/1` deprecated
* `Pow.Store.Backend.EtsCache.put/3` deprecated
* `Pow.Store.Backend.EtsCache` now uses `:ordered_set` instead of `:set` for efficiency
* `Pow.Store.Backend.MnesiaCache.keys/1` deprecated
* `Pow.Store.Backend.MnesiaCache.put/3` deprecated
* `Pow.Store.Backend.MnesiaCache` now uses `:ordered_set` instead of `:set` for efficiency
* `Pow.Store.Backend.MnesiaCache` will delete all binary key records when initialized
* `Pow.Store.Base` behaviour now requires to;
* Accept erlang term value for keys in all methods
* Implement `put/3` instead of `put/4`
* Implement `delete/2` instead of `put/3`
* Implement `get/2` instead of `put/3`
* Remove `keys/2`
* `Pow.Store.Base.all/3` added
* `Pow.Store.Base.put/3` added
* `Pow.Store.Base.keys/2` deprecated
* `Pow.Store.Base.put/4` deprecated
* `Pow.Store.Base` will use binary key rather than key list if `all/2` doesn't exist in the backend cache
* Added `Pow.Store.CredentialsCache.users/2`
* Added `Pow.Store.CredentialsCache.sessions/2`
* Deprecated `Pow.Store.CredentialsCache.user_session_keys/3`
* Deprecated `Pow.Store.CredentialsCache.sessions/3`
* `Pow.Store.CredentialsCache` now adds a session key rather than appending to a list for the user key to prevent race condition

## v1.0.13 (2019-08-25)

Expand Down
171 changes: 146 additions & 25 deletions guides/redis_cache_store_backend.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,50 +23,147 @@ defmodule MyAppWeb.PowRedisCache do

@redix_instance_name :redix

def put(config, key, value) do
key = redis_key(config, key)
ttl = Config.get(config, :ttl)
value = :erlang.term_to_binary(value)
command = put_command(key, value, ttl)

Redix.noreply_command(@redix_instance_name, command)
@impl true
def put(config, record_or_records) do
ttl = Config.get(config, :ttl) || raise_ttl_error()
commands =
record_or_records
|> List.wrap()
|> Enum.map(fn {key, value} ->
config
|> binary_redis_key(key)
|> put_command(value, ttl)
end)

Redix.noreply_pipeline(@redix_instance_name, commands)
end

defp put_command(key, value, ttl) when is_integer(ttl) and ttl > 0, do: ["SET", key, value, "PX", ttl]
defp put_command(key, value, _ttl), do: ["SET", key, value]
defp put_command(key, value, ttl) do
value = :erlang.term_to_binary(value)

["SET", key, value, "PX", ttl]
end

@impl true
def delete(config, key) do
key = redis_key(config, key)
key =
config
|> redis_key(key)
|> to_binary_redis_key()

Redix.noreply_command(@redix_instance_name, ["DEL", key])
end

@impl true
def get(config, key) do
key = redis_key(config, key)
key =
config
|> redis_key(key)
|> to_binary_redis_key()

case Redix.command(@redix_instance_name, ["GET", key]) do
{:ok, nil} -> :not_found
{:ok, value} -> :erlang.binary_to_term(value)
end
end

def keys(config) do
namespace = redis_key(config, "")
length = String.length(namespace)
@impl true
def all(config, match_spec) do
compiled_match_spec = :ets.match_spec_compile([{match_spec, [], [:"$_"]}])

Stream.resource(
fn -> do_scan(config, compiled_match_spec, "0") end,
&stream_scan(config, compiled_match_spec, &1),
fn _ -> :ok end)
|> Enum.to_list()
|> case do
[] -> []
keys -> fetch_values_for_keys(keys, config)
end
end

defp fetch_values_for_keys(keys, config) do
binary_keys = Enum.map(keys, &binary_redis_key(config, &1))

case Redix.command(@redix_instance_name, ["MGET"] ++ binary_keys) do
{:ok, values} ->
values = Enum.map(values, &:erlang.binary_to_term/1)

keys
|> Enum.zip(values)
|> Enum.reject(fn {_key, value} -> is_nil(value) end)
end
end

defp stream_scan(_config, _compiled_match_spec, {[], "0"}), do: {:halt, nil}
defp stream_scan(config, compiled_match_spec, {[], iterator}), do: do_scan(config, compiled_match_spec, iterator)
defp stream_scan(_config, _compiled_match_spec, {keys, iterator}), do: {keys, {[], iterator}}

defp do_scan(config, compiled_match_spec, iterator) do
prefix = to_binary_redis_key([namespace(config)]) <> ":*"

case Redix.command(@redix_instance_name, ["SCAN", iterator, "MATCH", prefix]) do
{:ok, [iterator, res]} -> {filter_or_load_value(compiled_match_spec, res), iterator}
end
end

defp filter_or_load_value(compiled_match_spec, keys) do
keys
|> Enum.map(&convert_key/1)
|> Enum.sort()
|> :ets.match_spec_run(compiled_match_spec)
end

defp convert_key(key) do
key
|> from_binary_redis_key()
|> unwrap()
end

{:ok, values} = Redix.command(@redix_instance_name, ["KEYS", "#{namespace}*"])
defp unwrap([_namespace, key]), do: key
defp unwrap([_namespace | key]), do: key

Enum.map(values, &String.slice(&1, length..-1))
defp binary_redis_key(config, key) do
config
|> redis_key(key)
|> to_binary_redis_key()
end

defp redis_key(config, key) do
namespace = Config.get(config, :namespace, "cache")
[namespace(config) | List.wrap(key)]
end

defp namespace(config), do: Config.get(config, :namespace, "cache")

defp to_binary_redis_key(key) do
key
|> Enum.map(fn part ->
part
|> :erlang.term_to_binary()
|> Base.url_encode64(padding: false)
end)
|> Enum.join(":")
end

"#{namespace}:#{key}"
defp from_binary_redis_key(key) do
key
|> String.split(":")
|> Enum.map(fn part ->
part
|> Base.url_decode64!(padding: false)
|> :erlang.binary_to_term()
end)
end

@spec raise_ttl_error :: no_return
defp raise_ttl_error,
do: Config.raise_error("`:ttl` configuration option is required for #{inspect(__MODULE__)}")
end

```

We are converting keys to binary keys since we can't directly use the Erlang terms as with ETS and Mnesia.

We'll need to start the Redix application on our app startup, so in `application.ex` add `{Redix, name: :redix}` to your supervision tree:

```elixir
Expand Down Expand Up @@ -107,10 +204,17 @@ defmodule MyAppWeb.PowRedisCacheTest do

@default_config [namespace: "test", ttl: :timer.hours(1)]

setup do
start_supervised!({Redix, host: "localhost", port: 6379, name: :redix})
Redix.command!(:redix, ["FLUSHALL"])

:ok
end

test "can put, get and delete records" do
assert PowRedisCache.get(@default_config, "key") == :not_found

PowRedisCache.put(@default_config, "key", "value")
PowRedisCache.put(@default_config, {"key", "value"})
:timer.sleep(100)
assert PowRedisCache.get(@default_config, "key") == "value"

Expand All @@ -119,22 +223,39 @@ defmodule MyAppWeb.PowRedisCacheTest do
assert PowRedisCache.get(@default_config, "key") == :not_found
end

test "fetch keys" do
PowRedisCache.put(@default_config, "key1", "value")
PowRedisCache.put(@default_config, "key2", "value")
test "can put multiple records at once" do
PowRedisCache.put(@default_config, [{"key1", "1"}, {"key2", "2"}])
:timer.sleep(100)
assert PowRedisCache.get(@default_config, "key1") == "1"
assert PowRedisCache.get(@default_config, "key2") == "2"
end

test "can match fetch all" do
PowRedisCache.put(@default_config, {"key1", "value"})
PowRedisCache.put(@default_config, {"key2", "value"})
:timer.sleep(100)

assert Enum.sort(PowRedisCache.keys(@default_config)) == ["key1", "key2"]
assert PowRedisCache.all(@default_config, :_) == [{"key1", "value"}, {"key2", "value"}]

PowRedisCache.put(@default_config, {["namespace", "key"], "value"})
:timer.sleep(100)

assert PowRedisCache.all(@default_config, ["namespace", :_]) == [{["namespace", "key"], "value"}]
end

test "records auto purge" do
config = Keyword.put(@default_config, :ttl, 100)

PowRedisCache.put(config, "key", "value")
PowRedisCache.put(config, {"key", "value"})
PowRedisCache.put(config, [{"key1", "1"}, {"key2", "2"}])
:timer.sleep(50)
assert PowRedisCache.get(config, "key") == "value"
assert PowRedisCache.get(config, "key1") == "1"
assert PowRedisCache.get(config, "key2") == "2"
:timer.sleep(100)
assert PowRedisCache.get(config, "key") == :not_found
assert PowRedisCache.get(config, "key1") == :not_found
assert PowRedisCache.get(config, "key2") == :not_found
end
end
```
```
12 changes: 8 additions & 4 deletions lib/pow/store/backend/base.ex
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,8 +12,12 @@ defmodule Pow.Store.Backend.Base do
"""
alias Pow.Config

@callback put(Config.t(), binary(), any()) :: :ok
@callback delete(Config.t(), binary()) :: :ok
@callback get(Config.t(), binary()) :: any() | :not_found
@callback keys(Config.t()) :: [any()]
@type key() :: [binary() | atom()] | binary()
@type record() :: {key(), any()}
@type key_match() :: [atom() | binary()]

@callback put(Config.t(), record() | [record()]) :: :ok
@callback delete(Config.t(), key()) :: :ok
@callback get(Config.t(), key()) :: any() | :not_found
@callback all(Config.t(), key_match()) :: [record()]
end
Loading

0 comments on commit c29269d

Please sign in to comment.