chore: merge pull request #169 from coderplanets/cron-job

add cron job and expire option to cache

Author: mydearxym

config/config.exs

@@ -80,6 +80,13 @@ config :rihanna,
   jobs_table_name: "background_jobs",
   producer_postgres_connection: {Ecto, GroupherServer.Repo}
 
+# cron-like job scheduler
+config :groupher_server, Helper.Scheduler,
+  jobs: [
+    # Every midnight
+    {"@daily", {Helper.Scheduler, :clear_all_cache, []}}
+  ]
+
 import_config "#{Mix.env()}.exs"
 
 if File.exists?("config/#{Mix.env()}.secret.exs") do

lib/groupher_server/application.ex

@@ -32,6 +32,7 @@ defmodule GroupherServer.Application do
             )
         ]
       ]),
+      worker(Helper.Scheduler, []),
       {Rihanna.Supervisor, [postgrex: GroupherServer.Repo.config()]}
     ]
 

lib/groupher_server/statistics/delegates/contribute.ex

@@ -85,7 +85,7 @@ defmodule GroupherServer.Statistics.Delegate.Contribute do
     %Community{id: id}
     |> do_get_contributes()
     |> to_counts_digest(days: @community_contribute_days)
-    |> done_and_cache(scope)
+    |> done_and_cache(scope, expire: 60_000)
   end
 
   defp update_contribute_record(%UserContribute{} = contribute) do

lib/helper/cache.ex

@@ -27,6 +27,19 @@ defmodule Helper.Cache do
     Cachex.put(:site_cache, cache_key, cache_value)
   end
 
+  def put(cache_key, cache_value, expire: expire_time) do
+    Cachex.put(:site_cache, cache_key, cache_value)
+    Cachex.expire(:site_cache, cache_key, expire_time)
+  end
+
+  @doc """
+  clear all the cache
+  ## Example
+  iex> Helper.Cache.clear()
+  {:ok, 1}
+  """
+  def clear_all(), do: Cachex.clear(:site_cache)
+
   @doc """
   cache scope of community contributes digest
   """

lib/helper/scheduler.ex

@@ -0,0 +1,16 @@
+defmodule Helper.Scheduler do
+  @moduledoc """
+  cron-like job scheduler
+  """
+  use Quantum.Scheduler, otp_app: :groupher_server
+
+  alias Helper.Cache
+
+  @doc """
+  clear all the cache in Cachex
+  just in case the cache system broken
+  """
+  def clear_all_cache do
+    Cache.clear_all()
+  end
+end

lib/helper/utils.ex

@@ -52,6 +52,13 @@ defmodule Helper.Utils do
   # def done({:error, error}), do: {:error, error}
   def done(result), do: {:ok, result}
 
+  def done_and_cache(result, scope, expire: expire_time) do
+    with {:ok, res} <- done(result) do
+      Cache.put(scope, res, expire: expire_time)
+      {:ok, res}
+    end
+  end
+
   def done_and_cache(result, scope) do
     with {:ok, res} <- done(result) do
       Cache.put(scope, res)

mix.exs

@@ -93,9 +93,14 @@ defmodule GroupherServer.Mixfile do
       {:sentry, "~> 7.1"},
       {:recase, "~> 0.4.0"},
       {:nanoid, "~> 2.0.0"},
+      # mailer
       {:bamboo, "1.2.0"},
+      # mem cache
       {:cachex, "3.1.3"},
-      {:rihanna, "1.3.5"}
+      # postgres-backed job queue
+      {:rihanna, "1.3.5"},
+      # cron-like scheduler job
+      {:quantum, "~> 2.3"}
     ]
   end
 

mix.lock

@@ -17,6 +17,7 @@
   "cowboy": {:hex, :cowboy, "2.6.3", "99aa50e94e685557cad82e704457336a453d4abcb77839ad22dbe71f311fcc06", [:rebar3], [{:cowlib, "~> 2.7.3", [hex: :cowlib, repo: "hexpm", optional: false]}, {:ranch, "~> 1.7.1", [hex: :ranch, repo: "hexpm", optional: false]}], "hexpm"},
   "cowlib": {:hex, :cowlib, "2.7.3", "a7ffcd0917e6d50b4d5fb28e9e2085a0ceb3c97dea310505f7460ff5ed764ce9", [:rebar3], [], "hexpm"},
   "credo": {:hex, :credo, "1.1.0", "e0c07b2fd7e2109495f582430a1bc96b2c71b7d94c59dfad120529f65f19872f", [:mix], [{:bunt, "~> 0.2.0", [hex: :bunt, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm"},
+  "crontab": {:hex, :crontab, "1.1.7", "b9219f0bdc8678b94143655a8f229716c5810c0636a4489f98c0956137e53985", [:mix], [{:ecto, "~> 1.0 or ~> 2.0 or ~> 3.0", [hex: :ecto, repo: "hexpm", optional: true]}], "hexpm"},
   "dataloader": {:hex, :dataloader, "1.0.6", "fb724d6d3fb6acb87d27e3b32dea3a307936ad2d245faf9cf5221d1323d6a4ba", [:mix], [{:ecto, ">= 0.0.0", [hex: :ecto, repo: "hexpm", optional: true]}], "hexpm"},
   "db_connection": {:hex, :db_connection, "2.1.0", "122e2f62c4906bf2e49554f1e64db5030c19229aa40935f33088e7d543aa79d0", [:mix], [{:connection, "~> 1.0.2", [hex: :connection, repo: "hexpm", optional: false]}], "hexpm"},
   "decimal": {:hex, :decimal, "1.8.0", "ca462e0d885f09a1c5a342dbd7c1dcf27ea63548c65a65e67334f4b61803822e", [:mix], [], "hexpm"},
@@ -31,6 +32,8 @@
   "faker": {:hex, :faker, "0.10.0", "367c2ae47e7b4ac6410e1eaa880c07b5fe4194476f697d37ac1ce25f3058aae2", [:mix], [], "hexpm"},
   "file_system": {:hex, :file_system, "0.2.6", "fd4dc3af89b9ab1dc8ccbcc214a0e60c41f34be251d9307920748a14bf41f1d3", [:mix], [], "hexpm"},
   "fs": {:hex, :fs, "0.9.2", "ed17036c26c3f70ac49781ed9220a50c36775c6ca2cf8182d123b6566e49ec59", [:rebar], [], "hexpm"},
+  "gen_stage": {:hex, :gen_stage, "0.14.2", "6a2a578a510c5bfca8a45e6b27552f613b41cf584b58210f017088d3d17d0b14", [:mix], [], "hexpm"},
+  "gen_state_machine": {:hex, :gen_state_machine, "2.0.5", "9ac15ec6e66acac994cc442dcc2c6f9796cf380ec4b08267223014be1c728a95", [:mix], [], "hexpm"},
   "gettext": {:hex, :gettext, "0.17.0", "abe21542c831887a2b16f4c94556db9c421ab301aee417b7c4fbde7fbdbe01ec", [:mix], [], "hexpm"},
   "guardian": {:hex, :guardian, "1.2.1", "bdc8dd3dbf0fb7216cb6f91c11831faa1a64d39cdaed9a611e37f2413e584983", [:mix], [{:jose, "~> 1.8", [hex: :jose, repo: "hexpm", optional: false]}, {:phoenix, "~> 1.3", [hex: :phoenix, repo: "hexpm", optional: true]}, {:plug, "~> 1.3.3 or ~> 1.4", [hex: :plug, repo: "hexpm", optional: true]}], "hexpm"},
   "hackney": {:hex, :hackney, "1.15.1", "9f8f471c844b8ce395f7b6d8398139e26ddca9ebc171a8b91342ee15a19963f4", [:rebar3], [{:certifi, "2.5.1", [hex: :certifi, repo: "hexpm", optional: false]}, {:idna, "6.0.0", [hex: :idna, repo: "hexpm", optional: false]}, {:metrics, "1.0.1", [hex: :metrics, repo: "hexpm", optional: false]}, {:mimerl, "~>1.1", [hex: :mimerl, repo: "hexpm", optional: false]}, {:ssl_verify_fun, "1.1.4", [hex: :ssl_verify_fun, repo: "hexpm", optional: false]}], "hexpm"},
@@ -39,6 +42,7 @@
   "jason": {:hex, :jason, "1.1.2", "b03dedea67a99223a2eaf9f1264ce37154564de899fd3d8b9a21b1a6fd64afe7", [:mix], [{:decimal, "~> 1.0", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm"},
   "jose": {:hex, :jose, "1.9.0", "4167c5f6d06ffaebffd15cdb8da61a108445ef5e85ab8f5a7ad926fdf3ada154", [:mix, :rebar3], [{:base64url, "~> 0.0.1", [hex: :base64url, repo: "hexpm", optional: false]}], "hexpm"},
   "jumper": {:hex, :jumper, "1.0.1", "3c00542ef1a83532b72269fab9f0f0c82bf23a35e27d278bfd9ed0865cecabff", [:mix], [], "hexpm"},
+  "libring": {:hex, :libring, "1.4.0", "41246ba2f3fbc76b3971f6bce83119dfec1eee17e977a48d8a9cfaaf58c2a8d6", [:mix], [], "hexpm"},
   "metrics": {:hex, :metrics, "1.0.1", "25f094dea2cda98213cecc3aeff09e940299d950904393b2a29d191c346a8486", [:rebar3], [], "hexpm"},
   "mime": {:hex, :mime, "1.3.1", "30ce04ab3175b6ad0bdce0035cba77bba68b813d523d1aac73d9781b4d193cf8", [:mix], [], "hexpm"},
   "mimerl": {:hex, :mimerl, "1.2.0", "67e2d3f571088d5cfd3e550c383094b47159f3eee8ffa08e64106cdf5e981be3", [:rebar3], [], "hexpm"},
@@ -57,6 +61,7 @@
   "poolboy": {:hex, :poolboy, "1.5.1", "6b46163901cfd0a1b43d692657ed9d7e599853b3b21b95ae5ae0a777cf9b6ca8", [:rebar], [], "hexpm"},
   "postgrex": {:hex, :postgrex, "0.14.3", "5754dee2fdf6e9e508cbf49ab138df964278700b764177e8f3871e658b345a1e", [:mix], [{:connection, "~> 1.0", [hex: :connection, repo: "hexpm", optional: false]}, {:db_connection, "~> 2.0", [hex: :db_connection, repo: "hexpm", optional: false]}, {:decimal, "~> 1.5", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}], "hexpm"},
   "pre_commit": {:hex, :pre_commit, "0.3.4", "e2850f80be8090d50ad8019ef2426039307ff5dfbe70c736ad0d4d401facf304", [:mix], [], "hexpm"},
+  "quantum": {:hex, :quantum, "2.3.4", "72a0e8855e2adc101459eac8454787cb74ab4169de6ca50f670e72142d4960e9", [:mix], [{:calendar, "~> 0.17", [hex: :calendar, repo: "hexpm", optional: true]}, {:crontab, "~> 1.1", [hex: :crontab, repo: "hexpm", optional: false]}, {:gen_stage, "~> 0.12", [hex: :gen_stage, repo: "hexpm", optional: false]}, {:swarm, "~> 3.3", [hex: :swarm, repo: "hexpm", optional: false]}, {:timex, "~> 3.1", [hex: :timex, repo: "hexpm", optional: true]}], "hexpm"},
   "ranch": {:hex, :ranch, "1.7.1", "6b1fab51b49196860b733a49c07604465a47bdb78aa10c1c16a3d199f7f8c881", [:rebar3], [], "hexpm"},
   "recase": {:hex, :recase, "0.4.0", "8fb52846f75948156385af2dfdc12f69e5ce27b022a9d1682c70a2fb3ed149c7", [:mix], [], "hexpm"},
   "rihanna": {:hex, :rihanna, "1.3.5", "5f5e6c5b1e514978a29a6791f338f4bb963401959fc212bd18d4a2c92d79a7a4", [:mix], [{:ecto, ">= 2.0.0", [hex: :ecto, repo: "hexpm", optional: true]}, {:ecto_sql, ">= 3.0.0", [hex: :ecto_sql, repo: "hexpm", optional: true]}, {:postgrex, ">= 0.13.3", [hex: :postgrex, repo: "hexpm", optional: false]}], "hexpm"},
@@ -66,6 +71,7 @@
   "short_maps": {:hex, :short_maps, "0.1.2", "a7c2bfd91179cdbdfe90e74a023992335d116982fa672612c74776b2e9257a7b", [:mix], [], "hexpm"},
   "sleeplocks": {:hex, :sleeplocks, "1.1.1", "3d462a0639a6ef36cc75d6038b7393ae537ab394641beb59830a1b8271faeed3", [:rebar3], [], "hexpm"},
   "ssl_verify_fun": {:hex, :ssl_verify_fun, "1.1.4", "f0eafff810d2041e93f915ef59899c923f4568f4585904d010387ed74988e77b", [:make, :mix, :rebar3], [], "hexpm"},
+  "swarm": {:hex, :swarm, "3.4.0", "64f8b30055d74640d2186c66354b33b999438692a91be275bb89cdc7e401f448", [:mix], [{:gen_state_machine, "~> 2.0", [hex: :gen_state_machine, repo: "hexpm", optional: false]}, {:libring, "~> 1.0", [hex: :libring, repo: "hexpm", optional: false]}], "hexpm"},
   "telemetry": {:hex, :telemetry, "0.4.0", "8339bee3fa8b91cb84d14c2935f8ecf399ccd87301ad6da6b71c09553834b2ab", [:rebar3], [], "hexpm"},
   "tesla": {:hex, :tesla, "0.10.0", "e588c7e7f1c0866c81eeed5c38f02a4a94d6309eede336c1e6ca08b0a95abd3f", [:mix], [{:exjsx, ">= 0.1.0", [hex: :exjsx, repo: "hexpm", optional: true]}, {:fuse, "~> 2.4", [hex: :fuse, repo: "hexpm", optional: true]}, {:hackney, "~> 1.6", [hex: :hackney, repo: "hexpm", optional: true]}, {:ibrowse, "~> 4.2", [hex: :ibrowse, repo: "hexpm", optional: true]}, {:mime, "~> 1.0", [hex: :mime, repo: "hexpm", optional: false]}, {:poison, ">= 1.0.0", [hex: :poison, repo: "hexpm", optional: true]}], "hexpm"},
   "timex": {:hex, :timex, "3.6.1", "efdf56d0e67a6b956cc57774353b0329c8ab7726766a11547e529357ffdc1d56", [:mix], [{:combine, "~> 0.10", [hex: :combine, repo: "hexpm", optional: false]}, {:gettext, "~> 0.10", [hex: :gettext, repo: "hexpm", optional: false]}, {:tzdata, "~> 0.1.8 or ~> 0.5 or ~> 1.0.0", [hex: :tzdata, repo: "hexpm", optional: false]}], "hexpm"},

test/groupher_server/statistics/statistics_test.exs

@@ -121,7 +121,6 @@ defmodule GroupherServer.Test.Statistics do
       assert second.count == 2
     end
 
-    @tag :wip
     test "should return recent #{@community_contribute_days} days community contributes by default",
          ~m(community)a do
       days_ago = Timex.shift(Timex.today(), days: -@community_contribute_days)
@@ -148,7 +147,6 @@ defmodule GroupherServer.Test.Statistics do
       assert length(contributes) == @community_contribute_days + 1
     end
 
-    @tag :wip
     test "the contributes data should be cached after first query", ~m(community)a do
       scope = Cache.get_scope(:community_contributes, community.id)
       assert {:error, nil} = Cache.get(scope)
@@ -158,7 +156,6 @@ defmodule GroupherServer.Test.Statistics do
       assert {:ok, contributes} = Cache.get(scope)
     end
 
-    @tag :wip2
     test "Rihanna should work in test sandbox", ~m(community)a do
       res = Rihanna.enqueue({IO, :puts, ["Work, work, work, work, work."]})
       Process.sleep(1000)

test/helper/cache_test.exs

@@ -5,13 +5,11 @@ defmodule GroupherServer.Test.Helper.Cache do
   alias Helper.Cache
 
   describe "[cache test]" do
-    @tag :wip
     test "cache get unexsit key should get nil" do
       assert {:error, nil} = Cache.get("no exsit")
       assert {:error, nil} = Cache.get(:no_exsit)
     end
 
-    @tag :wip
     test "cache put should work" do
       assert {:error, nil} = Cache.get(:data)
 
@@ -26,5 +24,22 @@ defmodule GroupherServer.Test.Helper.Cache do
       assert {:ok, true} = Cache.put("namespace.aaa.bbb", [1, %{a: "2"}])
       assert {:ok, [1, %{a: "2"}]} = Cache.get("namespace.aaa.bbb")
     end
+
+    test "cache can be clear" do
+      assert {:ok, true} = Cache.put(:data, "value")
+      assert {:ok, "value"} = Cache.get(:data)
+
+      assert {:ok, _} = Cache.clear_all()
+      assert {:error, nil} = Cache.get(:data)
+    end
+
+    test "cache expire should work" do
+      assert {:ok, true} = Cache.put(:data, "value", expire: 1000)
+      assert {:ok, "value"} = Cache.get(:data)
+      Process.sleep(900)
+      assert {:ok, "value"} = Cache.get(:data)
+      Process.sleep(1200)
+      assert {:error, nil} = Cache.get(:data)
+    end
   end
 end