diff --git a/lib/plausible/application.ex b/lib/plausible/application.ex index 2d219ce54b609..5916c1220aab8 100644 --- a/lib/plausible/application.ex +++ b/lib/plausible/application.ex @@ -9,109 +9,84 @@ defmodule Plausible.Application do def start(_type, _args) do on_full_build(do: Plausible.License.ensure_valid_license()) - children = [ - Plausible.Cache.Stats, - Plausible.Repo, - Plausible.ClickhouseRepo, - Plausible.IngestRepo, - Plausible.AsyncInsertRepo, - Plausible.ImportDeletionRepo, - {Plausible.RateLimit, clean_period: :timer.minutes(10)}, - Plausible.Ingestion.Counters, - {Finch, name: Plausible.Finch, pools: finch_pool_config()}, - {Phoenix.PubSub, name: Plausible.PubSub}, - Plausible.Session.Salts, - Supervisor.child_spec(Plausible.Event.WriteBuffer, id: Plausible.Event.WriteBuffer), - Supervisor.child_spec(Plausible.Session.WriteBuffer, id: Plausible.Session.WriteBuffer), - ReferrerBlocklist, - Plausible.Cache.Adapter.child_spec(:user_agents, :cache_user_agents, - ttl_check_interval: :timer.seconds(5), - global_ttl: :timer.minutes(60) - ), - Plausible.Cache.Adapter.child_spec(:sessions, :cache_sessions, - ttl_check_interval: :timer.seconds(1), - global_ttl: :timer.minutes(30) - ), - {Plausible.Site.Cache, ttl_check_interval: false}, - {Plausible.Cache.Warmer, - [ - child_name: Plausible.Site.Cache.All, - cache_impl: Plausible.Site.Cache, - interval: :timer.minutes(15) + Enum.random(1..:timer.seconds(10)), - warmer_fn: :refresh_all - ]}, - {Plausible.Cache.Warmer, - [ - child_name: Plausible.Site.Cache.RecentlyUpdated, - cache_impl: Plausible.Site.Cache, - interval: :timer.seconds(30), - warmer_fn: :refresh_updated_recently - ]}, - {Plausible.Shield.IPRuleCache, ttl_check_interval: false}, - {Plausible.Cache.Warmer, - [ - child_name: Plausible.Shield.IPRuleCache.All, - cache_impl: Plausible.Shield.IPRuleCache, - interval: :timer.minutes(3) + Enum.random(1..:timer.seconds(10)), - warmer_fn: :refresh_all - ]}, - {Plausible.Cache.Warmer, - [ - child_name: Plausible.Shield.IPRuleCache.RecentlyUpdated, - cache_impl: Plausible.Shield.IPRuleCache, - interval: :timer.seconds(35), - warmer_fn: :refresh_updated_recently - ]}, - {Plausible.Shield.CountryRuleCache, ttl_check_interval: false}, - {Plausible.Cache.Warmer, - [ - child_name: Plausible.Shield.CountryRuleCache.All, - cache_impl: Plausible.Shield.CountryRuleCache, - interval: :timer.minutes(3) + Enum.random(1..:timer.seconds(10)), - warmer_fn: :refresh_all - ]}, - {Plausible.Cache.Warmer, - [ - child_name: Plausible.Shield.CountryRuleCache.RecentlyUpdated, - cache_impl: Plausible.Shield.CountryRuleCache, - interval: :timer.seconds(35), - warmer_fn: :refresh_updated_recently - ]}, - {Plausible.Shield.PageRuleCache, ttl_check_interval: false, ets_options: [:bag]}, - {Plausible.Cache.Warmer, - [ - child_name: Plausible.Shield.PageRuleCache.All, - cache_impl: Plausible.Shield.PageRuleCache, - interval: :timer.minutes(3) + Enum.random(1..:timer.seconds(10)), - warmer_fn: :refresh_all - ]}, - {Plausible.Cache.Warmer, - [ - child_name: Plausible.Shield.PageRuleCache.RecentlyUpdated, - cache_impl: Plausible.Shield.PageRuleCache, - interval: :timer.seconds(35), - warmer_fn: :refresh_updated_recently - ]}, - {Plausible.Shield.HostnameRuleCache, ttl_check_interval: false, ets_options: [:bag]}, - {Plausible.Cache.Warmer, - [ - child_name: Plausible.Shield.HostnameRuleCache.All, - cache_impl: Plausible.Shield.HostnameRuleCache, - interval: :timer.minutes(3) + Enum.random(1..:timer.seconds(10)), - warmer_fn: :refresh_all - ]}, - {Plausible.Cache.Warmer, - [ - child_name: Plausible.Shield.HostnameRuleCache.RecentlyUpdated, - cache_impl: Plausible.Shield.HostnameRuleCache, - interval: :timer.seconds(25), - warmer_fn: :refresh_updated_recently - ]}, - {Plausible.Auth.TOTP.Vault, key: totp_vault_key()}, - PlausibleWeb.Endpoint, - {Oban, Application.get_env(:plausible, Oban)}, - Plausible.PromEx - ] + children = + [ + Plausible.Cache.Stats, + Plausible.Repo, + Plausible.ClickhouseRepo, + Plausible.IngestRepo, + Plausible.AsyncInsertRepo, + Plausible.ImportDeletionRepo, + {Plausible.RateLimit, clean_period: :timer.minutes(10)}, + Plausible.Ingestion.Counters, + {Finch, name: Plausible.Finch, pools: finch_pool_config()}, + {Phoenix.PubSub, name: Plausible.PubSub}, + Plausible.Session.Salts, + Supervisor.child_spec(Plausible.Event.WriteBuffer, id: Plausible.Event.WriteBuffer), + Supervisor.child_spec(Plausible.Session.WriteBuffer, id: Plausible.Session.WriteBuffer), + ReferrerBlocklist, + cache(:user_agents, + adapter_opts: [ttl_check_interval: :timer.seconds(5), global_ttl: :timer.minutes(60)] + ), + cache(:sessions, + adapter_opts: [ttl_check_interval: :timer.seconds(1), global_ttl: :timer.minutes(30)] + ), + cache(Plausible.Site.Cache, + adapter_opts: [ttl_check_interval: false], + warmers: [ + refresh_all: + {Plausible.Site.Cache.All, + interval: :timer.minutes(15) + Enum.random(1..:timer.seconds(10))}, + refresh_updated_recently: + {Plausible.Site.Cache.ReventlyUpdated, interval: :timer.seconds(30)} + ] + ), + cache(Plausible.Shield.IPRuleCache, + adapter_opts: [ttl_check_interval: false], + warmers: [ + refresh_all: + {Plausible.Shield.IPRuleCache.All, + interval: :timer.minutes(3) + Enum.random(1..:timer.seconds(10))}, + refresh_updated_recently: + {Plausible.Shield.IPRuleCache.RecentlyUpdated, interval: :timer.seconds(35)} + ] + ), + cache(Plausible.Shield.CountryRuleCache, + adapter_opts: [ttl_check_interval: false], + warmers: [ + refresh_all: + {Plausible.Shield.CountryRuleCache.All, + interval: :timer.minutes(3) + Enum.random(1..:timer.seconds(10))}, + refresh_updated_recently: + {Plausible.Shield.CountryRuleCache.RecentlyUpdated, interval: :timer.seconds(35)} + ] + ), + cache(Plausible.Shield.PageRuleCache, + adapter_opts: [ttl_check_interval: false, ets_options: [:bag]], + warmers: [ + refresh_all: + {Plausible.Shield.PageRuleCache.All, + interval: :timer.minutes(3) + Enum.random(1..:timer.seconds(10))}, + refresh_updated_recently: + {Plausible.Shield.PageRuleCache.RecentlyUpdated, interval: :timer.seconds(35)} + ] + ), + cache(Plausible.Shield.HostnameRuleCache, + adapter_opts: [ttl_check_interval: false, ets_options: [:bag]], + warmers: [ + refresh_all: + {Plausible.Shield.HostnameRuleCache.All, + interval: :timer.minutes(3) + Enum.random(1..:timer.seconds(10))}, + refresh_updated_recently: + {Plausible.Shield.HostnameRuleCache.RecentlyUpdated, interval: :timer.seconds(25)} + ] + ), + {Plausible.Auth.TOTP.Vault, key: totp_vault_key()}, + PlausibleWeb.Endpoint, + {Oban, Application.get_env(:plausible, Oban)}, + Plausible.PromEx + ] + |> List.flatten() opts = [strategy: :one_for_one, name: Plausible.Supervisor] @@ -222,4 +197,31 @@ defmodule Plausible.Application do opts = Application.fetch_env!(:plausible, Plausible.Geo) :ok = Plausible.Geo.load_db(opts) end + + defp cache(id_or_module, opts) do + warmers = Keyword.get(opts, :warmers) + + if is_nil(warmers) do + Plausible.Cache.Adapter.child_spec( + id_or_module, + String.to_atom("cache_#{id_or_module}"), + Keyword.fetch!(opts, :adapter_opts) + ) + else + warmer_specs = + Enum.map(warmers, fn {warmer_fn, {warmer_id, warmer_opts}} -> + {Plausible.Cache.Warmer, + Keyword.merge( + [ + child_name: warmer_id, + cache_impl: id_or_module, + warmer_fn: warmer_fn + ], + warmer_opts + )} + end) + + [{id_or_module, Keyword.fetch!(opts, :adapter_opts)} | warmer_specs] + end + end end diff --git a/lib/plausible/cache/adapter.ex b/lib/plausible/cache/adapter.ex index 76a7e95b9dcae..100965165b6f0 100644 --- a/lib/plausible/cache/adapter.ex +++ b/lib/plausible/cache/adapter.ex @@ -36,9 +36,7 @@ defmodule Plausible.Cache.Adapter do @spec get(atom(), any()) :: any() def get(cache_name, key) do - result = ConCache.get(cache_name, key) - if is_nil(result), do: miss(cache_name), else: hit(cache_name) - result + ConCache.get(cache_name, key) catch :exit, _ -> Logger.error("Error retrieving key from '#{inspect(cache_name)}'") @@ -47,16 +45,7 @@ defmodule Plausible.Cache.Adapter do @spec get(atom(), any(), (-> any())) :: any() def get(cache_name, key, fallback_fn) do - cache = ConCache.Owner.cache(cache_name) - - case ConCache.Operations.get(cache, key) do - nil -> - get_or_store_isolated(cache, cache_name, key, fallback_fn) - - value -> - hit(cache_name) - value - end + ConCache.get_or_store(cache_name, key, fallback_fn) catch :exit, _ -> Logger.error("Error retrieving key from '#{inspect(cache_name)}'") @@ -109,28 +98,4 @@ defmodule Plausible.Cache.Adapter do Logger.error("Error retrieving key from '#{inspect(cache_name)}'") [] end - - defp hit(cache_name) do - Plausible.Cache.Stats.record_hit(cache_name) - end - - defp miss(cache_name) do - Plausible.Cache.Stats.record_miss(cache_name) - end - - defp get_or_store_isolated(cache, cache_name, key, fun) do - ConCache.isolated(cache_name, key, fn -> - case ConCache.Operations.get(cache, key) do - nil -> - new_value = fun.() - ConCache.Operations.dirty_put(cache, key, new_value) - miss(cache_name) - new_value - - existing -> - hit(cache_name) - existing - end - end) - end end diff --git a/lib/plausible/cache/stats.ex b/lib/plausible/cache/stats.ex index 2e1e8fbe7c68b..f88688bced274 100644 --- a/lib/plausible/cache/stats.ex +++ b/lib/plausible/cache/stats.ex @@ -7,22 +7,14 @@ defmodule Plausible.Cache.Stats do @hit :hit @miss :miss - @telemetry_hit [:plausible, :cache, :adapter, @hit] - @telemetry_miss [:plausible, :cache, :adapter, @miss] + @telemetry_hit ConCache.Operations.telemetry_hit() + @telemetry_miss ConCache.Operations.telemetry_miss() @telemetry_events [@telemetry_hit, @telemetry_miss] def start_link(_opts) do GenServer.start_link(__MODULE__, nil) end - def record_hit(cache_name) do - :telemetry.execute(@telemetry_hit, %{}, %{cache_name: cache_name}) - end - - def record_miss(cache_name) do - :telemetry.execute(@telemetry_miss, %{}, %{cache_name: cache_name}) - end - def init(nil) do __MODULE__ = :ets.new(__MODULE__, [ @@ -43,11 +35,11 @@ defmodule Plausible.Cache.Stats do {:ok, nil} end - def handle_telemetry_event(@telemetry_hit, _measurments, %{cache_name: cache_name}, _) do + def handle_telemetry_event(@telemetry_hit, _measurments, %{cache: %{name: cache_name}}, _) do bump(cache_name, @hit) end - def handle_telemetry_event(@telemetry_miss, _measurments, %{cache_name: cache_name}, _) do + def handle_telemetry_event(@telemetry_miss, _measurments, %{cache: %{name: cache_name}}, _) do bump(cache_name, @miss) end diff --git a/mix.exs b/mix.exs index 619e367a0d1e0..1b295ce84e601 100644 --- a/mix.exs +++ b/mix.exs @@ -142,7 +142,7 @@ defmodule Plausible.MixProject do {:ex_aws_s3, "~> 2.5"}, {:sweet_xml, "~> 0.7.4"}, {:zstream, "~> 0.6.4"}, - {:con_cache, "~> 1.0"} + {:con_cache, "~> 1.1.0"} ] end diff --git a/mix.lock b/mix.lock index a9ff9643bc2d1..240acdbc55453 100644 --- a/mix.lock +++ b/mix.lock @@ -18,7 +18,7 @@ "combination": {:hex, :combination, "0.0.3", "746aedca63d833293ec6e835aa1f34974868829b1486b1e1cb0685f0b2ae1f41", [:mix], [], "hexpm", "72b099f463df42ef7dc6371d250c7070b57b6c5902853f69deb894f79eda18ca"}, "combine": {:hex, :combine, "0.10.0", "eff8224eeb56498a2af13011d142c5e7997a80c8f5b97c499f84c841032e429f", [:mix], [], "hexpm", "1b1dbc1790073076580d0d1d64e42eae2366583e7aecd455d1215b0d16f2451b"}, "comeonin": {:hex, :comeonin, "5.4.0", "246a56ca3f41d404380fc6465650ddaa532c7f98be4bda1b4656b3a37cc13abe", [:mix], [], "hexpm", "796393a9e50d01999d56b7b8420ab0481a7538d0caf80919da493b4a6e51faf1"}, - "con_cache": {:hex, :con_cache, "1.0.0", "6405e2bd5d5005334af72939432783562a8c35a196c2e63108fe10bb97b366e6", [:mix], [], "hexpm", "4d1f5cb1a67f3c1a468243dc98d10ac83af7f3e33b7e7c15999dc2c9bc0a551e"}, + "con_cache": {:hex, :con_cache, "1.1.0", "45c7c6cd6dc216e47636232e8c683734b7fe293221fccd9454fa1757bc685044", [:mix], [{:telemetry, "~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "8655f2ae13a1e56c8aef304d250814c7ed929c12810f126fc423ecc8e871593b"}, "cors_plug": {:hex, :cors_plug, "3.0.3", "7c3ac52b39624bc616db2e937c282f3f623f25f8d550068b6710e58d04a0e330", [:mix], [{:plug, "~> 1.13", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm", "3f2d759e8c272ed3835fab2ef11b46bddab8c1ab9528167bd463b6452edf830d"}, "cowboy": {:hex, :cowboy, "2.10.0", "ff9ffeff91dae4ae270dd975642997afe2a1179d94b1887863e43f681a203e26", [:make, :rebar3], [{:cowlib, "2.12.1", [hex: :cowlib, repo: "hexpm", optional: false]}, {:ranch, "1.8.0", [hex: :ranch, repo: "hexpm", optional: false]}], "hexpm", "3afdccb7183cc6f143cb14d3cf51fa00e53db9ec80cdcd525482f5e99bc41d6b"}, "cowboy_telemetry": {:hex, :cowboy_telemetry, "0.4.0", "f239f68b588efa7707abce16a84d0d2acf3a0f50571f8bb7f56a15865aae820c", [:rebar3], [{:cowboy, "~> 2.7", [hex: :cowboy, repo: "hexpm", optional: false]}, {:telemetry, "~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "7d98bac1ee4565d31b62d59f8823dfd8356a169e7fcbb83831b8a5397404c9de"}, diff --git a/test/plausible/cache/stats_test.exs b/test/plausible/cache/stats_test.exs deleted file mode 100644 index c01bf3627073f..0000000000000 --- a/test/plausible/cache/stats_test.exs +++ /dev/null @@ -1,45 +0,0 @@ -defmodule Plausible.Cache.StatsTest do - use Plausible.DataCase, async: true - - alias Plausible.Cache.Stats - - test "when tracking is not initialized, stats are 0" do - assert {:ok, %{hit_rate: +0.0, count: 0}} = Stats.gather(:foo) - end - - test "when cache is started, stats are 0", %{test: test} do - assert {:ok, %{hit_rate: +0.0, count: 0}} = Stats.gather(test) - end - - test "tracking changes hit ratio", %{test: test} do - Stats.record_miss(test) - assert {:ok, %{hit_rate: +0.0, count: 0}} = Stats.gather(test) - - Stats.record_hit(test) - assert {:ok, %{hit_rate: 50.0, count: 0}} = Stats.gather(test) - - Stats.record_hit(test) - Stats.record_hit(test) - assert {:ok, %{hit_rate: 75.0, count: 0}} = Stats.gather(test) - - Stats.record_miss(test) - Stats.record_miss(test) - assert {:ok, %{hit_rate: 50.0, count: 0}} = Stats.gather(test) - end - - test "hit ratio goes up to 100", %{test: test} do - Stats.record_hit(test) - Stats.record_hit(test) - Stats.record_hit(test) - assert {:ok, %{hit_rate: 100.0, count: 0}} = Stats.gather(test) - end - - test "count comes from cache adapter", %{test: test} do - %{start: {m, f, a}} = Plausible.Cache.Adapter.child_spec(test, test) - {:ok, _} = apply(m, f, a) - - Plausible.Cache.Adapter.put(test, :key, :value) - assert Stats.size(test) == 1 - assert {:ok, %{hit_rate: +0.0, count: 1}} = Stats.gather(test) - end -end