diff --git a/.formatter.exs b/.formatter.exs index 8a6391c..ad7e60a 100644 --- a/.formatter.exs +++ b/.formatter.exs @@ -1,4 +1,5 @@ [ + line_length: 200, import_deps: [:ecto, :phoenix], inputs: ["*.{ex,exs}", "priv/*/seeds.exs", "{config,lib,test}/**/*.{ex,exs}"], subdirectories: ["priv/*/migrations"] diff --git a/.gitignore b/.gitignore index 22fc8e8..afd8c77 100644 --- a/.gitignore +++ b/.gitignore @@ -20,10 +20,13 @@ erl_crash.dump *.ez # Ignore package tarball (built via "mix hex.build"). -noncegeek-*.tar +Noncegeek-*.tar # Ignore assets that are produced by build tools. -/priv/static/assets/ +# Ignore assets that are produced by build tools. +/priv/static/uploads/* +/priv/static/assets/* +/priv/static/images/* # Ignore digested assets cache. /priv/static/cache_manifest.json diff --git a/.iex.exs b/.iex.exs new file mode 100644 index 0000000..16563d3 --- /dev/null +++ b/.iex.exs @@ -0,0 +1,5 @@ +import Ecto.Query, warn: false + +alias Noncegeek.Repo +alias Noncegeek.Explorer +alias Explorer.Model.{Event, Token, Collection} diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..c657856 --- /dev/null +++ b/Makefile @@ -0,0 +1,4 @@ +iex: + iex --erl "-kernel shell_history enabled" -S mix +server: + iex --erl "-kernel shell_history enabled" -S mix phx.server \ No newline at end of file diff --git a/config/config.exs b/config/config.exs index 43a182b..37ed193 100644 --- a/config/config.exs +++ b/config/config.exs @@ -33,8 +33,7 @@ config :swoosh, :api_client, false config :esbuild, version: "0.14.29", default: [ - args: - ~w(js/app.js --bundle --target=es2017 --outdir=../priv/static/assets --external:/fonts/* --external:/images/*), + args: ~w(js/app.js --bundle --target=es2017 --outdir=../priv/static/assets --external:/fonts/* --external:/images/*), cd: Path.expand("../assets", __DIR__), env: %{"NODE_PATH" => Path.expand("../deps", __DIR__)} ] @@ -47,8 +46,18 @@ config :logger, :console, # Use Jason for JSON parsing in Phoenix config :phoenix, :json_library, Jason +config :noncegeek, Oban, + repo: Noncegeek.Repo, + plugins: [{Oban.Plugins.Pruner, max_age: 3 * 24 * 60 * 60}], + queues: [default: 10] + config :noncegeek, AptosEx, rpc_endpoint: "https://testnet.aptoslabs.com/v1" +config :noncegeek, + contract_address: "0xe698622471b41a92e13ae893ae4ff88b20c528f6da2bedcb24d74646bf972dc3", + contract_creator: "0xe10e40298c16778e71a03fa7e00e7d29e12a77b5e1797b799034551401cc0cc4", + collection_name: "NonceGeek Leaf" + # Import environment specific config. This must remain at the bottom # of this file so it overrides the configuration defined above. import_config "#{config_env()}.exs" diff --git a/lib/noncegeek/application.ex b/lib/noncegeek/application.ex index a099b70..11a6390 100644 --- a/lib/noncegeek/application.ex +++ b/lib/noncegeek/application.ex @@ -16,9 +16,8 @@ defmodule Noncegeek.Application do {Phoenix.PubSub, name: Noncegeek.PubSub}, # Start the Endpoint (http/https) NoncegeekWeb.Endpoint, + {Oban, Application.fetch_env!(:noncegeek, Oban)}, {AptosEx, Application.fetch_env!(:noncegeek, AptosEx)} - # Start a worker by calling: Noncegeek.Worker.start_link(arg) - # {Noncegeek.Worker, arg} ] # See https://hexdocs.pm/elixir/Supervisor.html diff --git a/lib/noncegeek/explorer.ex b/lib/noncegeek/explorer.ex new file mode 100644 index 0000000..44bacfc --- /dev/null +++ b/lib/noncegeek/explorer.ex @@ -0,0 +1,124 @@ +defmodule Noncegeek.Explorer do + @moduledoc """ + The Explorer context. + """ + import Ecto.Query, warn: false + + require Integer + + alias AptosEx + + alias Noncegeek.Fetcher + alias Noncegeek.Explorer.Model.{Token, Event} + alias Noncegeek.{Repo, Turbo} + + @doc """ + Paged tokens + """ + def paged_tokens() do + Turbo.all(Token) + end + + @doc """ + Get single token + """ + def get_token(clauses) when is_list(clauses) or is_map(clauses), + do: Turbo.get_by(Token, clauses) + + def get_token(clauses) when is_integer(clauses), do: Turbo.get(Token, clauses) + + @doc """ + fetch token data && store + + ## Exmaples + + iex> Noncegeek.Explorer.fetch_token_data("0xe19430a2498ff6800666d41cfd4b64d6d2a53574ef7457f700f96f4a61703d07", "DummyNames", "dummy1") + + """ + def fetch_token_data(creator, collection_name, token_name) do + with {:ok, token} <- + get_token(%{creator: creator, name: token_name, collection_name: collection_name}), + {:ok, data} <- AptosEx.get_token_data(creator, collection_name, token_name) do + Turbo.update(token, data) + end + end + + def get_account_events(account, type) do + sequence_number = get_sequence_number(account, type) + + case type do + "0x3::token::DepositEvent" -> + Fetcher.get_deposit_events(account, sequence_number) + + "0x3::token::WithdrawEvent" -> + Fetcher.get_withdraw_events(account, sequence_number) + end + end + + def get_sequence_number(account, type) do + from(e in Event, + where: e.type == ^type, + where: e.account_address == ^account, + order_by: [desc: e.sequence_number], + select: e.sequence_number, + limit: 1 + ) + |> Repo.one() + |> case do + nil -> 0 + value -> value + 1 + end + end + + @doc """ + fetch account events + + iex> Noncegeek.Explorer.refresh_account_events("0xe19430a2498ff6800666d41cfd4b64d6d2a53574ef7457f700f96f4a61703d07") + iex> Noncegeek.Explorer.refresh_account_events("e698622471b41a92e13ae893ae4ff88b20c528f6da2bedcb24d74646bf972dc3") + + """ + def refresh_account_events(account) do + get_account_events(account, "0x3::token::DepositEvent") + get_account_events(account, "0x3::token::WithdrawEvent") + + list_account_tokens(account) + end + + @doc """ + list account tokens + ## TODO + + - [ ] user_tokens + + """ + def list_account_tokens(account) do + from(e in Event, + where: + e.account_address == ^account and + e.type in ["0x3::token::DepositEvent", "0x3::token::WithdrawEvent"], + order_by: [desc: e.version], + preload: [:token] + ) + |> Repo.all() + |> case do + [] -> + [] + + value -> + value + |> Enum.group_by(& &1.token_id) + |> Enum.map(fn {_token_id, token_events} -> + case token_events |> length() |> Integer.is_even() do + true -> + nil + + false -> + List.first(token_events) + end + end) + |> Enum.reject(&is_nil/1) + |> List.flatten() + end + |> then(&{:ok, &1}) + end +end diff --git a/lib/noncegeek/explorer/jobs/fetch_token_data.ex b/lib/noncegeek/explorer/jobs/fetch_token_data.ex new file mode 100644 index 0000000..702abcc --- /dev/null +++ b/lib/noncegeek/explorer/jobs/fetch_token_data.ex @@ -0,0 +1,45 @@ +defmodule Noncegeek.Explorer.Job.FetchTokenData do + @moduledoc false + + require Logger + + use Oban.Worker, queue: :default, priority: 1, max_attempts: 20 + + alias Noncegeek.Explorer + + @contract_creator Application.get_env(:noncegeek, :contract_creator) + @collection_name Application.get_env(:noncegeek, :collection_name) + + @impl Oban.Worker + def perform(%Oban.Job{args: %{"token_id" => token_id} = _args}) do + %{"token_data_id" => %{"creator" => creator, "collection" => collection_name, "name" => name}} = token_id + + with {:ok, token} <- Explorer.fetch_token_data(creator, collection_name, name) do + IO.inspect(token, label: "token") + create_nft_image(token) + :ok + else + _ -> + {:error, :retry_fetcher_token_data} + end + end + + defp create_nft_image(%{collection_name: unquote(@collection_name), creator: unquote(@contract_creator), name: name} = _token) do + IO.inspect("aaaaaaaaaaaaaaaaaaaaaaaaaaaaa") + + unique_num = + name + |> String.split(":") + |> List.last() + |> String.trim() + + file_path = "priv/static/images/#{unique_num}.jpg" + + if !File.exists?(file_path) do + {:ok, %{body: body}} = Faker.Avatar.image_url() |> Tesla.get() + File.write!(file_path, body) + end + end + + defp create_nft_image(_), do: nil +end diff --git a/lib/noncegeek/explorer/models/event.ex b/lib/noncegeek/explorer/models/event.ex index 12e175a..606a805 100644 --- a/lib/noncegeek/explorer/models/event.ex +++ b/lib/noncegeek/explorer/models/event.ex @@ -19,7 +19,7 @@ defmodule Noncegeek.Explorer.Model.Event do field :amount, :decimal field :data, :map - belongs_to :token, Lotus.Explorer.Model.Token, + belongs_to :token, Noncegeek.Explorer.Model.Token, foreign_key: :token_id, references: :token_id, type: :map diff --git a/lib/noncegeek/explorer/models/token.ex b/lib/noncegeek/explorer/models/token.ex index 3a34373..1a855c6 100644 --- a/lib/noncegeek/explorer/models/token.ex +++ b/lib/noncegeek/explorer/models/token.ex @@ -9,7 +9,6 @@ defmodule Noncegeek.Explorer.Model.Token do schema "tokens" do # required field :token_id, :map - field :collection_id, :map field :collection_name, :string field :creator, :string field :name, :string @@ -25,9 +24,6 @@ defmodule Noncegeek.Explorer.Model.Token do field :uri, :string field :property_version, :integer - # fetcher - field :last_fetched_at, :utc_datetime_usec - timestamps() end @@ -42,7 +38,6 @@ defmodule Noncegeek.Explorer.Model.Token do )a optional_fields = ~w( - last_fetched_at maximum largest_property_version mutability_config @@ -69,7 +64,5 @@ defmodule Noncegeek.Explorer.Model.Token do token |> cast(attrs, required_fields) |> validate_required(required_fields) - |> NameSlug.maybe_generate_slug() - |> NameSlug.unique_constraint() end end diff --git a/lib/noncegeek/fetcher.ex b/lib/noncegeek/fetcher.ex new file mode 100644 index 0000000..39b935e --- /dev/null +++ b/lib/noncegeek/fetcher.ex @@ -0,0 +1,101 @@ +defmodule Noncegeek.Fetcher do + @moduledoc false + + alias AptosEx + + alias Noncegeek.Fetcher.Transform + alias Noncegeek.{Explorer, Import} + + defmodule TaskData do + @moduledoc """ + %Task{} with state && result && contract_name + """ + defstruct event_handle: nil, + account: nil, + field: nil, + type: nil, + sequence_number: 0, + ref: nil + + # def event_handle_id(%__MODULE__{event_handle: event_handle, field: field}), do: event_handle <> "::" <> field + def event_type(%__MODULE__{type: type}), do: type + end + + @doc """ + iex> Noncegeek.Fetcher.task(%{sequence_number: 0, account: "0xe698622471b41a92e13ae893ae4ff88b20c528f6da2bedcb24d74646bf972dc3", event_handle: "0xe698622471b41a92e13ae893ae4ff88b20c528f6da2bedcb24d74646bf972dc3::LEAF::MintData", field: "mint_events"}) + + """ + def task(%{sequence_number: sequence_number} = task_data) do + case fetch_and_import_events(task_data) do + {:ok, %{events: events}} when events != [] -> + events + |> Enum.max_by(&Map.get(&1, :sequence_number)) + |> Map.get(:sequence_number) + |> Kernel.+(1) + + {:ok, _} -> + sequence_number + + # {:error, _} -> + # sequence_number + end + end + + @doc """ + Noncegeek.Fetcher.get_withdraw_events("0xe19430a2498ff6800666d41cfd4b64d6d2a53574ef7457f700f96f4a61703d07", 0) + """ + def get_withdraw_events(account, sequence_number) do + task_data = %TaskData{ + sequence_number: sequence_number, + account: account, + event_handle: "0x3::token::TokenStore", + field: "withdraw_events" + } + + task(task_data) + end + + @doc """ + Noncegeek.Fetcher.get_deposit_events("0xe19430a2498ff6800666d41cfd4b64d6d2a53574ef7457f700f96f4a61703d07", 0) + """ + def get_deposit_events(account, sequence_number) do + task_data = %TaskData{ + sequence_number: sequence_number, + account: account, + event_handle: "0x3::token::TokenStore", + field: "deposit_events" + } + + task(task_data) + end + + defp fetch_and_import_events( + %{ + sequence_number: sequence_number, + account: account, + field: field, + event_handle: event_handle + } = _task_data + ) do + with {:ok, event_list} <- + AptosEx.get_events(account, event_handle, field, start: sequence_number), + {:ok, import_list} <- Transform.params_set(event_list) do + {:ok, result} = Import.run(import_list) + + async_fetcher(result) + + {:ok, result} + end + end + + defp async_fetcher(%{tokens: tokens}) do + tokens + |> Enum.each(fn item -> + %{token_id: item.token_id} + |> Explorer.Job.FetchTokenData.new() + |> Oban.insert() + end) + end + + defp async_fetcher(_), do: :ok +end diff --git a/lib/noncegeek/fetcher/event_handle/leader.ex b/lib/noncegeek/fetcher/event_handle/leader.ex new file mode 100644 index 0000000..c9c62d8 --- /dev/null +++ b/lib/noncegeek/fetcher/event_handle/leader.ex @@ -0,0 +1,184 @@ +defmodule Noncegeek.Fetcher.EventHandle.Leader do + @moduledoc """ + NOTE: [Refacoty] TaskSupervisor is not a good way, it's better to use DynamicSupervisor + """ + + use GenServer + + require Logger + + alias Noncegeek.{Fetcher, Explorer} + alias Fetcher.{EventHandle, TaskData} + + @contract Application.get_env(:noncegeek, :contract_address) + + # milliseconds + @block_interval 10_000 + + @watched_events [ + %{ + account: @contract, + event_handle: "#{@contract}::LEAF::MintData", + type: "#{@contract}::LEAF::MintEvent", + field: "mint_events" + } + ] + + defmodule State do + defstruct ~w( + client + tasks + )a + end + + def child_spec([named_arguments]) when is_map(named_arguments), + do: child_spec([named_arguments, []]) + + def child_spec([_named_arguments, gen_server_options] = start_link_arguments) + when is_list(gen_server_options) do + Supervisor.child_spec( + %{ + id: __MODULE__, + start: {__MODULE__, :start_link, start_link_arguments}, + type: :supervisor + }, + [] + ) + end + + def start_link(client, gen_server_options \\ []) do + GenServer.start_link(__MODULE__, client, gen_server_options) + end + + def get_entries() do + GenServer.call(__MODULE__, :get_entries) + end + + @impl true + def init(client) do + state = %State{ + client: client, + tasks: [] + } + + for item <- @watched_events do + Logger.info("start #{inspect(@watched_events)}") + + sequence_number = Explorer.get_sequence_number(item.account, item.type) + + Process.send_after( + self(), + {:fetcher, + %TaskData{ + event_handle: item.event_handle, + sequence_number: sequence_number, + field: item.field, + account: item.account + }}, + @block_interval + ) + end + + {:ok, state} + end + + @impl true + def handle_info({:fetcher, event}, %State{client: _client, tasks: tasks} = state) do + %{ref: ref} = Task.Supervisor.async_nolink(EventHandle.TaskSupervisor, Fetcher, :task, [event]) + + task_data = %TaskData{event | ref: ref} + + {:noreply, %State{state | tasks: [task_data | tasks]}} + end + + @impl true + def handle_info({ref, sequence_number}, %State{tasks: tasks, client: _client} = state) do + Process.demonitor(ref, [:flush]) + + case Enum.find_index(tasks, &(&1.ref == ref)) do + nil -> + # 异常处理, 建议重启程序 + Logger.error("Tried to update non-existing task: #{inspect(ref)}") + # Process.send_after(self(), :fetcher, @block_interval) + # retry + + {:noreply, %State{state | tasks: []}} + + index -> + Process.sleep(@block_interval) + + old_task_data = Enum.at(tasks, index) + + %{ref: new_task_ref} = + Task.Supervisor.async_nolink(EventHandle.TaskSupervisor, Fetcher, :task, [ + %TaskData{old_task_data | sequence_number: sequence_number} + ]) + + new_task_data = %TaskData{ + old_task_data + | ref: new_task_ref, + sequence_number: sequence_number + } + + Logger.info("Start new task") + + new_tasks = List.replace_at(tasks, index, new_task_data) + + {:noreply, %State{state | tasks: new_tasks}} + end + end + + @impl true + def handle_info({ref, {:error, :etimedout}}, %State{tasks: tasks} = state) do + case Enum.find_index(tasks, &(&1.ref == ref)) do + nil -> + Logger.error("Tried to update non-existing task: #{inspect(ref)}") + restart_all_tasks(state) + + index -> + Logger.warn("#{inspect(ref)} timeout, restaring task") + + start_new_task(index, state) + end + end + + @impl true + def handle_info({:DOWN, ref, :process, _pid, reason}, %State{tasks: tasks} = state) do + case Enum.find_index(tasks, &(&1.ref == ref)) do + nil -> + Logger.error("Tried to update non-existing task: #{inspect(ref)}") + restart_all_tasks(state) + + index -> + Logger.error("exited with reason (#{inspect(reason)})") + start_new_task(index, state) + end + end + + defp restart_all_tasks(%{tasks: tasks} = state) do + Process.sleep(@block_interval) + + for item <- tasks do + Process.send_after( + self(), + {:fetcher, %TaskData{event_handle: item.event_handle, field: item.field, account: item.account}}, + @block_interval + ) + end + + {:noreply, %State{state | tasks: []}} + end + + defp start_new_task(index, %State{tasks: tasks, client: _client} = state) do + Process.sleep(@block_interval) + + old_task_data = Enum.at(tasks, index) + + %{ref: new_task_ref} = Task.Supervisor.async_nolink(EventHandle.TaskSupervisor, Fetcher, :task, [old_task_data]) + + new_task_data = %TaskData{old_task_data | ref: new_task_ref} + new_tasks = List.replace_at(tasks, index, new_task_data) + + {:noreply, %State{state | tasks: new_tasks}} + end +end diff --git a/lib/noncegeek/fetcher/event_handle/supervisor.ex b/lib/noncegeek/fetcher/event_handle/supervisor.ex new file mode 100644 index 0000000..c481ced --- /dev/null +++ b/lib/noncegeek/fetcher/event_handle/supervisor.ex @@ -0,0 +1,40 @@ +defmodule Noncegeek.Fetcher.EventHandle.Supervisor do + @moduledoc false + + use Supervisor + + require Logger + + alias Noncegeek.Fetcher + + def child_spec([init_arguments]) do + child_spec([init_arguments, []]) + end + + def child_spec([_init_arguments, _gen_server_options] = start_link_arguments) do + default = %{ + id: __MODULE__, + start: {__MODULE__, :start_link, start_link_arguments}, + type: :supervisor + } + + Supervisor.child_spec(default, []) + end + + def start_link(arguments, gen_server_options \\ []) do + Supervisor.start_link(__MODULE__, arguments, gen_server_options) + end + + @impl Supervisor + def init(client) do + Logger.info("Started EventHandle") + + Supervisor.init( + [ + {Task.Supervisor, name: Fetcher.EventHandle.TaskSupervisor}, + {Fetcher.EventHandle.Leader, [client, [name: Fetcher.EventHandle.Leader]]} + ], + strategy: :one_for_one + ) + end +end diff --git a/lib/noncegeek/fetcher/supervisor.ex b/lib/noncegeek/fetcher/supervisor.ex new file mode 100644 index 0000000..7a047d3 --- /dev/null +++ b/lib/noncegeek/fetcher/supervisor.ex @@ -0,0 +1,34 @@ +defmodule Noncegeek.Fetcher.Supervisor do + @moduledoc """ + Supervisor of all indexer worker supervision trees + """ + + use Supervisor + + require Logger + + alias Noncegeek.Fetcher + + def start_link(arguments, gen_server_options \\ []) do + Supervisor.start_link( + __MODULE__, + arguments, + Keyword.put_new(gen_server_options, :name, __MODULE__) + ) + end + + @impl true + def init(_arg) do + Logger.info("Noncegeek.Fetcher.Supervisor Stared") + # FIXME support multi-chain client + + basic_fetchers = [ + {Fetcher.EventHandle.Supervisor, [nil, [name: Fetcher.EventHandle.Supervisor]]} + ] + + Supervisor.init( + basic_fetchers, + strategy: :one_for_one + ) + end +end diff --git a/lib/noncegeek/fetcher/transform.ex b/lib/noncegeek/fetcher/transform.ex new file mode 100644 index 0000000..948fd11 --- /dev/null +++ b/lib/noncegeek/fetcher/transform.ex @@ -0,0 +1,64 @@ +defmodule Noncegeek.Fetcher.Transform do + @moduledoc false + + def integerfy(id) when is_binary(id), do: String.to_integer(id) + def integerfy(id), do: id + + def stringfy(v) when is_binary(v), do: v + def stringfy(v) when is_integer(v), do: to_string(v) + def stringfy(v) when is_atom(v), do: to_string(v) + def stringfy(v), do: v + + def params_set(event_list) do + event_params = event_params_set(event_list) + token_params = token_params_set(event_list) + + {:ok, + %{ + events: %{params: event_params}, + tokens: %{params: token_params} + }} + end + + def token_params_set(event_list) do + event_list + |> Enum.group_by(&get_token_id(&1)) + |> Enum.map(fn {token_id, _} -> + %{ + token_id: token_id, + property_version: token_id.property_version, + collection_name: token_id.token_data_id.collection, + creator: token_id.token_data_id.creator, + name: token_id.token_data_id.name + } + end) + end + + def event_params_set(event_list) do + event_list + |> Enum.map(&to_elixir/1) + |> Enum.map(&event_to_params(&1)) + end + + defp to_elixir(event), do: Enum.into(event, %{}, &entry_to_elixir/1) + + defp entry_to_elixir({key, value}) when key in ~w(sequence_number version)a, + do: {key, integerfy(value)} + + defp entry_to_elixir(entry), do: entry + + defp event_to_params(event_data) do + %{ + sequence_number: event_data.sequence_number, + creation_number: event_data.guid.creation_number, + account_address: event_data.guid.account_address, + data: event_data.data, + token_id: get_token_id(event_data), + version: event_data.version, + type: event_data.type + } + end + + defp get_token_id(event_data), + do: get_in(event_data, [:data, :id]) || get_in(event_data, [:data, :token_id]) +end diff --git a/lib/noncegeek/import.ex b/lib/noncegeek/import.ex new file mode 100644 index 0000000..9caaca9 --- /dev/null +++ b/lib/noncegeek/import.ex @@ -0,0 +1,257 @@ +defmodule Noncegeek.Import do + @moduledoc """ + Bulk importing of data into `Noncegeek.Repo` + """ + + alias Ecto.Changeset + + alias Noncegeek.Repo + alias Noncegeek.Import + + @stages [ + Import.Stage.Explorer + ] + + @runners Enum.flat_map(@stages, fn stage -> stage.runners() end) + @transaction_timeout :timer.minutes(4) + @global_options ~w(broadcast timeout)a + @local_options ~w(on_conflict params with timeout)a + + @doc """ + import Ecto.Multi + + ## Examples + + iex> data = %{ + ...> tokens: %{params: []}, + ...> events: %{params: []}, + ...> collections: %{params: []}, + } + + iex> Noncegeek.Explorer.Import.all(data) + + """ + def run(options) when is_map(options) do + with {:ok, runner_options_pairs} <- validate_options(options), + {:ok, valid_runner_option_pairs} <- validate_runner_options_pairs(runner_options_pairs), + {:ok, runner_to_changes_list} <- runner_to_changes_list(valid_runner_option_pairs), + {:ok, data} <- insert_runner_to_changes_list(runner_to_changes_list, options) do + {:ok, data} + end + end + + defp runner_to_changes_list(runner_options_pairs) when is_list(runner_options_pairs) do + runner_options_pairs + |> Stream.map(fn {runner, options} -> runner_changes_list(runner, options) end) + |> Enum.reduce({:ok, %{}}, fn + {:ok, {runner, changes_list}}, {:ok, acc_runner_to_changes_list} -> + {:ok, Map.put(acc_runner_to_changes_list, runner, changes_list)} + + {:ok, _}, {:error, _} = error -> + error + + {:error, _} = error, {:ok, _} -> + error + + {:error, runner_changesets}, {:error, acc_changesets} -> + {:error, acc_changesets ++ runner_changesets} + end) + end + + defp runner_changes_list(runner, %{params: params} = options) do + ecto_schema_module = runner.ecto_schema_module() + changeset_function_name = Map.get(options, :with, :changeset) + struct = ecto_schema_module.__struct__() + + params + |> Stream.map(&apply(ecto_schema_module, changeset_function_name, [struct, &1])) + |> Enum.reduce({:ok, []}, fn + changeset = %Changeset{valid?: false}, {:ok, _} -> + {:error, [changeset]} + + changeset = %Changeset{valid?: false}, {:error, acc_changesets} -> + {:error, [changeset | acc_changesets]} + + %Changeset{changes: changes, valid?: true}, {:ok, acc_changes} -> + {:ok, [changes | acc_changes]} + + %Changeset{valid?: true}, {:error, _} = error -> + error + + :ignore, error -> + {:error, error} + end) + |> case do + {:ok, changes} -> {:ok, {runner, changes}} + {:error, _} = error -> error + end + end + + defp validate_options(options) when is_map(options) do + local_options = Map.drop(options, @global_options) + + {reverse_runner_options_pairs, unknown_options} = + Enum.reduce(@runners, {[], local_options}, fn runner, {acc_runner_options_pairs, unknown_options} = acc -> + option_key = runner.option_key() + + case local_options do + %{^option_key => option_value} -> + {[{runner, option_value} | acc_runner_options_pairs], Map.delete(unknown_options, option_key)} + + _ -> + acc + end + end) + + # {:ok, Enum.reverse(reverse_runner_options_pairs)} + case Enum.empty?(unknown_options) do + true -> {:ok, Enum.reverse(reverse_runner_options_pairs)} + false -> {:error, {:unknown_options, unknown_options}} + end + end + + defp validate_runner_options_pairs(runner_options_pairs) when is_list(runner_options_pairs) do + {status, reversed} = + runner_options_pairs + |> Stream.map(fn {runner, options} -> validate_runner_options(runner, options) end) + |> Enum.reduce({:ok, []}, fn + :ignore, acc -> + acc + + {:ok, valid_runner_option_pair}, {:ok, valid_runner_options_pairs} -> + {:ok, [valid_runner_option_pair | valid_runner_options_pairs]} + + {:ok, _}, {:error, _} = error -> + error + + {:error, reason}, {:ok, _} -> + {:error, [reason]} + + {:error, reason}, {:error, reasons} -> + {:error, [reason | reasons]} + end) + + {status, Enum.reverse(reversed)} + end + + defp validate_runner_options(runner, options) when is_map(options) do + option_key = runner.option_key() + + runner_specific_options = + if Map.has_key?(Enum.into(runner.__info__(:functions), %{}), :runner_specific_options) do + runner.runner_specific_options() + else + [] + end + + case {validate_runner_option_params_required(option_key, options), validate_runner_options_known(option_key, options, runner_specific_options)} do + {:ignore, :ok} -> :ignore + {:ignore, {:error, _} = error} -> error + {:ok, :ok} -> {:ok, {runner, options}} + {:ok, {:error, _} = error} -> error + {{:error, reason}, :ok} -> {:error, [reason]} + {{:error, reason}, {:error, reasons}} -> {:error, [reason | reasons]} + end + end + + defp validate_runner_option_params_required(_, %{params: params}) do + case Enum.empty?(params) do + false -> :ok + true -> :ignore + end + end + + # defp validate_runner_option_params_required(runner_option_key, _), + # do: {:error, {:required, [runner_option_key, :params]}} + + # @local_options ~w(on_conflict params with timeout)a + + defp validate_runner_options_known(runner_option_key, options, runner_specific_options) do + base_unknown_option_keys = Map.keys(options) -- @local_options + unknown_option_keys = base_unknown_option_keys -- runner_specific_options + + if Enum.empty?(unknown_option_keys) do + :ok + else + reasons = Enum.map(unknown_option_keys, &{:unknown, [runner_option_key, &1]}) + + {:error, reasons} + end + end + + defp runner_to_changes_list_to_multis(runner_to_changes_list, options) + when is_map(runner_to_changes_list) and is_map(options) do + timestamps = timestamps() + full_options = Map.put(options, :timestamps, timestamps) + + {multis, final_runner_to_changes_list} = + Enum.flat_map_reduce(@stages, runner_to_changes_list, fn stage, remaining_runner_to_changes_list -> + stage.multis(remaining_runner_to_changes_list, full_options) + end) + + unless Enum.empty?(final_runner_to_changes_list) do + raise ArgumentError, + "No stages consumed the following runners: #{final_runner_to_changes_list |> Map.keys() |> inspect()}" + end + + multis + end + + def insert_changes_list(repo, changes_list, options) + when is_atom(repo) and is_list(changes_list) do + ecto_schema_module = Keyword.fetch!(options, :for) + + timestamped_changes_list = timestamp_changes_list(changes_list, Keyword.fetch!(options, :timestamps)) + + {_, inserted} = + repo.safe_insert_all( + ecto_schema_module, + timestamped_changes_list, + Keyword.delete(options, :for) + ) + + {:ok, inserted} + end + + defp timestamp_changes_list(changes_list, timestamps) when is_list(changes_list) do + Enum.map(changes_list, ×tamp_params(&1, timestamps)) + end + + defp timestamp_params(changes, timestamps) when is_map(changes) do + Map.merge(changes, timestamps) + end + + defp insert_runner_to_changes_list(runner_to_changes_list, options) + when is_map(runner_to_changes_list) do + runner_to_changes_list + |> runner_to_changes_list_to_multis(options) + |> logged_import(options) + end + + defp logged_import(multis, options) when is_list(multis) and is_map(options), + do: import_transactions(multis, options) + + defp import_transactions(multis, options) when is_list(multis) and is_map(options) do + Enum.reduce_while(multis, {:ok, %{}}, fn multi, {:ok, acc_changes} -> + case import_transaction(multi, options) do + {:ok, changes} -> {:cont, {:ok, Map.merge(acc_changes, changes)}} + {:error, _, _, _} = error -> {:halt, error} + end + end) + rescue + exception in DBConnection.ConnectionError -> + case Exception.message(exception) do + "tcp recv: closed" <> _ -> {:error, :timeout} + _ -> reraise exception, __STACKTRACE__ + end + end + + defp import_transaction(multi, options) when is_map(options) do + Repo.logged_transaction(multi, timeout: Map.get(options, :timeout, @transaction_timeout)) + end + + def timestamps do + now = DateTime.utc_now() + %{inserted_at: now, updated_at: now} + end +end diff --git a/lib/noncegeek/import/runner.ex b/lib/noncegeek/import/runner.ex new file mode 100644 index 0000000..c7d867a --- /dev/null +++ b/lib/noncegeek/import/runner.ex @@ -0,0 +1,53 @@ +defmodule Noncegeek.Import.Runner do + @moduledoc false + + alias Ecto.Multi + + @typedoc """ + A callback module that implements this module's behaviour. + """ + @type t :: module + + @typedoc """ + consensus changes extracted from a valid `Ecto.Changeset` produced by the `t:changeset_function_name/0` in + `c:ecto_schemma_module/0`. + """ + @type changes :: %{optional(atom) => term()} + + @typedoc """ + A list of `t:changes/0` to be imported by `c:run/3`. + """ + @type changes_list :: [changes] + + @type changeset_function_name :: atom + @type on_conflict :: :nothing | :replace_all | Ecto.Query.t() + + @typedoc """ + Runner-specific options under `c:option_key/0` in all options passed to `c:run/3`. + """ + @type options :: %{ + required(:params) => [map()], + optional(:on_conflict) => on_conflict(), + optional(:timeout) => timeout, + optional(:with) => changeset_function_name() + } + + @doc """ + Key in `t:all_options` used by this `Noncegeek.Import` behaviour implementation. + """ + @callback option_key() :: atom() + + @doc """ + The `Ecto.Schema` module that contains the `:changeset` function for validating `options[options_key][:params]`. + """ + @callback ecto_schema_module() :: module() + @callback run(Multi.t(), changes_list, %{optional(atom()) => term()}) :: Multi.t() + @callback timeout() :: timeout() + + @doc """ + The optional list of runner-specific options. + """ + @callback runner_specific_options() :: [atom()] + + @optional_callbacks runner_specific_options: 0 +end diff --git a/lib/noncegeek/import/runner/events.ex b/lib/noncegeek/import/runner/events.ex new file mode 100644 index 0000000..cb30666 --- /dev/null +++ b/lib/noncegeek/import/runner/events.ex @@ -0,0 +1,82 @@ +defmodule Noncegeek.Import.Runner.Events do + @moduledoc false + + alias Ecto.Multi + + alias Noncegeek.Import + alias Noncegeek.Explorer.Model.Event + + @timeout 60_000 + + def timeout, do: @timeout + def option_key, do: :events + def ecto_schema_module, do: Event + + def run(multi, changes_list, %{timestamps: timestamps} = options) do + insert_options = + options + |> Map.get(option_key(), %{}) + |> Map.take(~w(on_conflict timeout)a) + |> Map.put_new(:timeout, @timeout) + |> Map.put(:timestamps, timestamps) + + Multi.run(multi, :events, fn repo, _ -> + insert(repo, changes_list, insert_options) + end) + end + + def insert(repo, changes_list, %{timeout: timeout, timestamps: timestamps} = _options) + when is_list(changes_list) do + {:ok, _} = + Import.insert_changes_list( + repo, + changes_list, + conflict_target: [:type, :sequence_number], + on_conflict: :nothing, + for: Event, + returning: true, + timeout: timeout, + timestamps: timestamps + ) + end +end + +# withdraw_event + +# { +# "version": "20194971", +# "key": "0x0800000000000000dc4e806913a006d86da8327a079d794435e2e3117fd418062ddf43943d663490", +# "sequence_number": "0", +# "type": "0x3::token::WithdrawEvent", +# "data": { +# "amount": "1", +# "id": { +# "property_version": "0", +# "token_id": { +# "collection": "DummyDog", +# "creator": "0xdc4e806913a006d86da8327a079d794435e2e3117fd418062ddf43943d663490", +# "name": "DummyDog 1" +# } +# } +# } +# } + +# deposit events + +# { +# "version": "20192337", +# "key": "0x0700000000000000dc4e806913a006d86da8327a079d794435e2e3117fd418062ddf43943d663490", +# "sequence_number": "4", +# "type": "0x3::token::DepositEvent", +# "data": { +# "amount": "1", +# "id": { +# "property_version": "0", +# "token_id": { +# "collection": "DummyDog", +# "creator": "0xdc4e806913a006d86da8327a079d794435e2e3117fd418062ddf43943d663490", +# "name": "DummyDog 1" +# } +# } +# } +# } diff --git a/lib/noncegeek/import/runner/tokens.ex b/lib/noncegeek/import/runner/tokens.ex new file mode 100644 index 0000000..cb22c0e --- /dev/null +++ b/lib/noncegeek/import/runner/tokens.ex @@ -0,0 +1,45 @@ +defmodule Noncegeek.Import.Runner.Tokens do + @moduledoc false + + # import Ecto.Query, only: [from: 2] + + alias Ecto.Multi + + alias Noncegeek.Import + alias Noncegeek.Explorer.Model.Token + + @timeout 60_000 + + def timeout, do: @timeout + def option_key, do: :tokens + def ecto_schema_module, do: Token + + def run(multi, changes_list, %{timestamps: timestamps} = options) do + insert_options = + options + |> Map.get(option_key(), %{}) + |> Map.take(~w(on_conflict timeout)a) + |> Map.put_new(:timeout, @timeout) + |> Map.put(:timestamps, timestamps) + + multi + |> Multi.run(:tokens, fn repo, _ -> + insert(repo, changes_list, insert_options) + end) + end + + def insert(repo, changes_list, %{timeout: timeout, timestamps: timestamps} = _options) + when is_list(changes_list) do + {:ok, _} = + Import.insert_changes_list( + repo, + changes_list, + conflict_target: :token_id, + on_conflict: :nothing, + for: Token, + returning: true, + timeout: timeout, + timestamps: timestamps + ) + end +end diff --git a/lib/noncegeek/import/stage.ex b/lib/noncegeek/import/stage.ex new file mode 100644 index 0000000..e8a14f7 --- /dev/null +++ b/lib/noncegeek/import/stage.ex @@ -0,0 +1,68 @@ +defmodule Noncegeek.Import.Stage do + @moduledoc """ + Behaviour used to chunk `changes_list` into multiple `t:Ecto.Multi.t/0`` that can run in separate transactions to + limit the time that transactions take and how long blocking locks are held in Postgres. + """ + + alias Ecto.Multi + alias Noncegeek.Explorer.Import.Runner + + @typedoc """ + Maps `t:Noncegeek.Import.Import.Runner.t/0` callback module to the `t:Noncegeek.Import.Import.Runner.changes_list/0` it + can import. + """ + @type runner_to_changes_list :: %{Runner.t() => Runner.changes_list()} + + @doc """ + The runners consumed by this stage in `c:multis/0`. The list should be in the order that the runners are executed. + """ + @callback runners() :: [Runner.t(), ...] + + @doc """ + Chunks `changes_list` into 1 or more `t:Ecto.Multi.t/0` that can be run in separate transactions. + + The runners used by the stage should be removed from the returned `runner_to_changes_list` map. + """ + @callback multis(runner_to_changes_list, %{optional(atom()) => term()}) :: + {[Multi.t()], runner_to_changes_list} + + @doc """ + Uses a single `t:Noncegeek.Import.Runner.t/0` and chunks the `changes_list` across multiple `t:Ecto.Multi.t/0` + """ + def chunk_every(runner_to_changes_list, runner, chunk_size, options) + when is_map(runner_to_changes_list) and is_atom(runner) and is_integer(chunk_size) and + is_map(options) do + {changes_list, unstaged_runner_to_changes_list} = Map.pop(runner_to_changes_list, runner) + multis = changes_list_chunk_every(changes_list, chunk_size, runner, options) + + {multis, unstaged_runner_to_changes_list} + end + + defp changes_list_chunk_every(nil, _, _, _), do: [] + + defp changes_list_chunk_every(changes_list, chunk_size, runner, options) do + changes_list + |> Stream.chunk_every(chunk_size) + |> Enum.map(fn changes_chunk -> + runner.run(Multi.new(), changes_chunk, options) + end) + end + + def single_multi(runners, runner_to_changes_list, options) do + runners + |> Enum.reduce({Multi.new(), runner_to_changes_list}, fn runner, {multi, remaining_runner_to_changes_list} -> + {changes_list, new_remaining_runner_to_changes_list} = Map.pop(remaining_runner_to_changes_list, runner) + + new_multi = + case changes_list do + nil -> + multi + + _ -> + runner.run(multi, changes_list, options) + end + + {new_multi, new_remaining_runner_to_changes_list} + end) + end +end diff --git a/lib/noncegeek/import/stage/explorer.ex b/lib/noncegeek/import/stage/explorer.ex new file mode 100644 index 0000000..8296eb5 --- /dev/null +++ b/lib/noncegeek/import/stage/explorer.ex @@ -0,0 +1,21 @@ +defmodule Noncegeek.Import.Stage.Explorer do + @moduledoc false + + alias Noncegeek.Import.{Runner, Stage} + + @behaviour Stage + + @impl Stage + def runners, + do: [ + Runner.Events, + Runner.Tokens + ] + + @impl Stage + def multis(runner_to_changes_list, options) do + {final_multi, final_remaining_runner_to_changes_list} = Stage.single_multi(runners(), runner_to_changes_list, options) + + {[final_multi], final_remaining_runner_to_changes_list} + end +end diff --git a/lib/noncegeek/oban_job.ex b/lib/noncegeek/oban_job.ex new file mode 100644 index 0000000..703ae56 --- /dev/null +++ b/lib/noncegeek/oban_job.ex @@ -0,0 +1,29 @@ +defmodule Noncegeek.ObanJob do + @moduledoc """ + Mainly for quick query and helpers + """ + + use Ecto.Schema + + # Copied from oban/job.ex + schema "oban_jobs" do + field :state, :string, default: "available" + field :queue, :string, default: "default" + field :worker, :string + field :args, :map + field :meta, :map, default: %{} + field :tags, {:array, :string}, default: [] + field :errors, {:array, :map}, default: [] + field :attempt, :integer, default: 0 + field :attempted_by, {:array, :string} + field :max_attempts, :integer, default: 20 + field :priority, :integer, default: 0 + + field :attempted_at, :utc_datetime_usec + field :cancelled_at, :utc_datetime_usec + field :completed_at, :utc_datetime_usec + field :discarded_at, :utc_datetime_usec + field :inserted_at, :utc_datetime_usec + field :scheduled_at, :utc_datetime_usec + end +end diff --git a/lib/noncegeek/repo.ex b/lib/noncegeek/repo.ex index 490cca0..739beca 100644 --- a/lib/noncegeek/repo.ex +++ b/lib/noncegeek/repo.ex @@ -1,5 +1,74 @@ defmodule Noncegeek.Repo do - use Ecto.Repo, - otp_app: :noncegeek, - adapter: Ecto.Adapters.Postgres + @moduledoc false + + use Ecto.Repo, otp_app: :noncegeek, adapter: Ecto.Adapters.Postgres + + require Logger + + def logged_transaction(fun_or_multi, opts \\ []) do + {microseconds, value} = :timer.tc(__MODULE__, :transaction, [fun_or_multi, opts]) + + milliseconds = div(microseconds, 100) / 10.0 + Logger.debug(["transaction_time=", :io_lib_format.fwrite_g(milliseconds), ?m, ?s]) + + value + end + + @doc """ + Chunks elements into multiple `insert_all`'s to avoid DB driver param limits. + + *Note:* Should always be run within a transaction as multiple inserts may occur. + """ + def safe_insert_all(kind, elements, opts) do + returning = opts[:returning] + + elements + |> Enum.chunk_every(500) + |> Enum.reduce({0, []}, fn chunk, {total_count, acc} -> + {count, inserted} = + try do + insert_all(kind, chunk, opts) + rescue + exception -> + old_truncate = Application.get_env(:logger, :truncate) + Logger.configure(truncate: :infinity) + + Logger.error(fn -> + [ + "Could not insert all of chunk into ", + to_string(kind), + " using options because of error.\n", + "\n", + "Chunk Size: ", + chunk |> length() |> to_string(), + "\n", + "Chunk:\n", + "\n", + inspect(chunk, limit: :infinity, printable_limit: :infinity), + "\n", + "\n", + "Options:\n", + "\n", + inspect(opts), + "\n", + "\n", + "Exception:\n", + "\n", + Exception.format(:error, exception, __STACKTRACE__) + ] + end) + + Logger.configure(truncate: old_truncate) + + # reraise to kill caller + reraise exception, __STACKTRACE__ + end + + if returning do + {count + total_count, acc ++ inserted} + else + {count + total_count, nil} + end + end) + end end diff --git a/lib/noncegeek/turbo.ex b/lib/noncegeek/turbo.ex new file mode 100644 index 0000000..a0f097d --- /dev/null +++ b/lib/noncegeek/turbo.ex @@ -0,0 +1,87 @@ +defmodule Noncegeek.Turbo do + @moduledoc """ + Ecto Enhance API + """ + + import Ecto.Query, warn: false + + alias Noncegeek.Repo + + def all(queryable) do + queryable + |> Repo.all() + |> done() + end + + def get(queryable, id, preload: preload) do + queryable + |> preload(^preload) + |> Repo.get(id) + |> done(queryable, id) + end + + def get(queryable, id) do + queryable + |> Repo.get(id) + |> done(queryable, id) + end + + @doc """ + simular to Repo.get_by/3, with standard result/error handle + """ + def get_by(queryable, clauses, preload: preload) do + queryable + |> preload(^preload) + |> Repo.get_by(clauses) + |> case do + nil -> + {:error, :not_found} + + result -> + {:ok, result} + end + end + + def get_by(queryable, clauses) do + queryable + |> Repo.get_by(clauses) + |> case do + nil -> + {:error, :not_found} + + result -> + {:ok, result} + end + end + + def create(schema, attrs) do + schema + |> struct + |> schema.changeset(attrs) + |> Repo.insert() + end + + def update(content, attrs) do + content + |> content.__struct__.changeset(attrs) + |> Repo.update() + end + + def delete(content), do: Repo.delete(content) + + def findby_or_insert(queryable, clauses, attrs) do + case queryable |> get_by(clauses) do + {:ok, content} -> + {:ok, content} + + {:error, _} -> + queryable |> create(attrs) + end + end + + def done(nil), do: {:error, "record not found."} + def done(result), do: {:ok, result} + def done(result, _, _), do: {:ok, result} + def done(nil, :boolean), do: {:ok, false} + def done(_, :boolean), do: {:ok, true} +end diff --git a/lib/noncegeek_web/controllers/page_controller.ex b/lib/noncegeek_web/controllers/page_controller.ex index eff204c..2bef6b4 100644 --- a/lib/noncegeek_web/controllers/page_controller.ex +++ b/lib/noncegeek_web/controllers/page_controller.ex @@ -5,3 +5,5 @@ defmodule NoncegeekWeb.PageController do render(conn, "index.html") end end + +# File.write!("priv/static/images/1.jpg", body) diff --git a/lib/noncegeek_web/telemetry.ex b/lib/noncegeek_web/telemetry.ex index 3ea37fe..b56d070 100644 --- a/lib/noncegeek_web/telemetry.ex +++ b/lib/noncegeek_web/telemetry.ex @@ -49,8 +49,7 @@ defmodule NoncegeekWeb.Telemetry do ), summary("noncegeek.repo.query.idle_time", unit: {:native, :millisecond}, - description: - "The time the connection spent waiting before being checked out for the query" + description: "The time the connection spent waiting before being checked out for the query" ), # VM Metrics diff --git a/lib/noncegeek_web/templates/page/index.html.heex b/lib/noncegeek_web/templates/page/index.html.heex index f844bd8..69a15bb 100644 --- a/lib/noncegeek_web/templates/page/index.html.heex +++ b/lib/noncegeek_web/templates/page/index.html.heex @@ -1,6 +1,7 @@ <%= gettext "Welcome to %{name}!", name: "Phoenix" %> Peace of mind from prototype to production + diff --git a/mix.exs b/mix.exs index 15d95f8..fc380ec 100644 --- a/mix.exs +++ b/mix.exs @@ -48,7 +48,13 @@ defmodule Noncegeek.MixProject do {:telemetry_poller, "~> 1.0"}, {:gettext, "~> 0.18"}, {:jason, "~> 1.2"}, - {:plug_cowboy, "~> 2.5"} + {:plug_cowboy, "~> 2.5"}, + # http client + {:tesla, "~> 1.4"}, + # faker data, + {:faker, "~> 0.17"}, + # job + {:oban, "~> 2.13"} ] end diff --git a/mix.lock b/mix.lock index 2938944..4104680 100644 --- a/mix.lock +++ b/mix.lock @@ -9,11 +9,13 @@ "ecto": {:hex, :ecto, "3.9.1", "67173b1687afeb68ce805ee7420b4261649d5e2deed8fe5550df23bab0bc4396", [:mix], [{:decimal, "~> 1.6 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "c80bb3d736648df790f7f92f81b36c922d9dd3203ca65be4ff01d067f54eb304"}, "ecto_sql": {:hex, :ecto_sql, "3.9.0", "2bb21210a2a13317e098a420a8c1cc58b0c3421ab8e3acfa96417dab7817918c", [:mix], [{:db_connection, "~> 2.5 or ~> 2.4.1", [hex: :db_connection, repo: "hexpm", optional: false]}, {:ecto, "~> 3.9.0", [hex: :ecto, repo: "hexpm", optional: false]}, {:myxql, "~> 0.6.0", [hex: :myxql, repo: "hexpm", optional: true]}, {:postgrex, "~> 0.16.0 or ~> 1.0", [hex: :postgrex, repo: "hexpm", optional: true]}, {:tds, "~> 2.1.1 or ~> 2.2", [hex: :tds, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4.0 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "a8f3f720073b8b1ac4c978be25fa7960ed7fd44997420c304a4a2e200b596453"}, "esbuild": {:hex, :esbuild, "0.5.0", "d5bb08ff049d7880ee3609ed5c4b864bd2f46445ea40b16b4acead724fb4c4a3", [:mix], [{:castore, ">= 0.0.0", [hex: :castore, repo: "hexpm", optional: false]}], "hexpm", "f183a0b332d963c4cfaf585477695ea59eef9a6f2204fdd0efa00e099694ffe5"}, + "faker": {:hex, :faker, "0.17.0", "671019d0652f63aefd8723b72167ecdb284baf7d47ad3a82a15e9b8a6df5d1fa", [:mix], [], "hexpm", "a7d4ad84a93fd25c5f5303510753789fc2433ff241bf3b4144d3f6f291658a6a"}, "file_system": {:hex, :file_system, "0.2.10", "fb082005a9cd1711c05b5248710f8826b02d7d1784e7c3451f9c1231d4fc162d", [:mix], [], "hexpm", "41195edbfb562a593726eda3b3e8b103a309b733ad25f3d642ba49696bf715dc"}, "floki": {:hex, :floki, "0.34.0", "002d0cc194b48794d74711731db004fafeb328fe676976f160685262d43706a8", [:mix], [], "hexpm", "9c3a9f43f40dde00332a589bd9d389b90c1f518aef500364d00636acc5ebc99c"}, "gettext": {:hex, :gettext, "0.20.0", "75ad71de05f2ef56991dbae224d35c68b098dd0e26918def5bb45591d5c8d429", [:mix], [], "hexpm", "1c03b177435e93a47441d7f681a7040bd2a816ece9e2666d1c9001035121eb3d"}, "jason": {:hex, :jason, "1.4.0", "e855647bc964a44e2f67df589ccf49105ae039d4179db7f6271dfd3843dc27e6", [:mix], [{:decimal, "~> 1.0 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm", "79a3791085b2a0f743ca04cec0f7be26443738779d09302e01318f97bdb82121"}, "mime": {:hex, :mime, "2.0.3", "3676436d3d1f7b81b5a2d2bd8405f412c677558c81b1c92be58c00562bb59095", [:mix], [], "hexpm", "27a30bf0db44d25eecba73755acf4068cbfe26a4372f9eb3e4ea3a45956bff6b"}, + "oban": {:hex, :oban, "2.13.4", "b4c4f48f4c89cc01036670eefa28aa9c03d09aadd402655475b936983d597006", [:mix], [{:ecto_sql, "~> 3.6", [hex: :ecto_sql, repo: "hexpm", optional: false]}, {:jason, "~> 1.1", [hex: :jason, repo: "hexpm", optional: false]}, {:postgrex, "~> 0.16", [hex: :postgrex, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "a7d26f82b409e2d7928fbb75a17716e06ad3f783ebe9af260e3dd23abed7f124"}, "phoenix": {:hex, :phoenix, "1.6.15", "0a1d96bbc10747fd83525370d691953cdb6f3ccbac61aa01b4acb012474b047d", [:mix], [{:castore, ">= 0.0.0", [hex: :castore, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:phoenix_pubsub, "~> 2.0", [hex: :phoenix_pubsub, repo: "hexpm", optional: false]}, {:phoenix_view, "~> 1.0 or ~> 2.0", [hex: :phoenix_view, repo: "hexpm", optional: false]}, {:plug, "~> 1.10", [hex: :plug, repo: "hexpm", optional: false]}, {:plug_cowboy, "~> 2.2", [hex: :plug_cowboy, repo: "hexpm", optional: true]}, {:plug_crypto, "~> 1.2", [hex: :plug_crypto, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "d70ab9fbf6b394755ea88b644d34d79d8b146e490973151f248cacd122d20672"}, "phoenix_ecto": {:hex, :phoenix_ecto, "4.4.0", "0672ed4e4808b3fbed494dded89958e22fb882de47a97634c0b13e7b0b5f7720", [:mix], [{:ecto, "~> 3.3", [hex: :ecto, repo: "hexpm", optional: false]}, {:phoenix_html, "~> 2.14.2 or ~> 3.0", [hex: :phoenix_html, repo: "hexpm", optional: true]}, {:plug, "~> 1.9", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm", "09864e558ed31ee00bd48fcc1d4fc58ae9678c9e81649075431e69dbabb43cc1"}, "phoenix_html": {:hex, :phoenix_html, "3.2.0", "1c1219d4b6cb22ac72f12f73dc5fad6c7563104d083f711c3fcd8551a1f4ae11", [:mix], [{:plug, "~> 1.5", [hex: :plug, repo: "hexpm", optional: true]}], "hexpm", "36ec97ba56d25c0136ef1992c37957e4246b649d620958a1f9fa86165f8bc54f"}, @@ -32,4 +34,5 @@ "telemetry": {:hex, :telemetry, "1.1.0", "a589817034a27eab11144ad24d5c0f9fab1f58173274b1e9bae7074af9cbee51", [:rebar3], [], "hexpm", "b727b2a1f75614774cff2d7565b64d0dfa5bd52ba517f16543e6fc7efcc0df48"}, "telemetry_metrics": {:hex, :telemetry_metrics, "0.6.1", "315d9163a1d4660aedc3fee73f33f1d355dcc76c5c3ab3d59e76e3edf80eef1f", [:mix], [{:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "7be9e0871c41732c233be71e4be11b96e56177bf15dde64a8ac9ce72ac9834c6"}, "telemetry_poller": {:hex, :telemetry_poller, "1.0.0", "db91bb424e07f2bb6e73926fcafbfcbcb295f0193e0a00e825e589a0a47e8453", [:rebar3], [{:telemetry, "~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "b3a24eafd66c3f42da30fc3ca7dda1e9d546c12250a2d60d7b81d264fbec4f6e"}, + "tesla": {:hex, :tesla, "1.4.4", "bb89aa0c9745190930366f6a2ac612cdf2d0e4d7fff449861baa7875afd797b2", [:mix], [{:castore, "~> 0.1", [hex: :castore, repo: "hexpm", optional: true]}, {:exjsx, ">= 3.0.0", [hex: :exjsx, repo: "hexpm", optional: true]}, {:finch, "~> 0.3", [hex: :finch, repo: "hexpm", optional: true]}, {:fuse, "~> 2.4", [hex: :fuse, repo: "hexpm", optional: true]}, {:gun, "~> 1.3", [hex: :gun, repo: "hexpm", optional: true]}, {:hackney, "~> 1.6", [hex: :hackney, repo: "hexpm", optional: true]}, {:ibrowse, "4.4.0", [hex: :ibrowse, repo: "hexpm", optional: true]}, {:jason, ">= 1.0.0", [hex: :jason, repo: "hexpm", optional: true]}, {:mime, "~> 1.0 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:mint, "~> 1.0", [hex: :mint, repo: "hexpm", optional: true]}, {:poison, ">= 1.0.0", [hex: :poison, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: true]}], "hexpm", "d5503a49f9dec1b287567ea8712d085947e247cb11b06bc54adb05bfde466457"}, } diff --git a/priv/repo/migrations/20221107131140_add_oban_jobs_table.exs b/priv/repo/migrations/20221107131140_add_oban_jobs_table.exs new file mode 100644 index 0000000..2e555a5 --- /dev/null +++ b/priv/repo/migrations/20221107131140_add_oban_jobs_table.exs @@ -0,0 +1,13 @@ +defmodule Noncegeek.Repo.Migrations.AddObanJobsTable do + use Ecto.Migration + + def up do + Oban.Migrations.up(version: 11) + end + + # We specify `version: 1` in `down`, ensuring that we'll roll all the way back down if + # necessary, regardless of which version we've migrated `up` to. + def down do + Oban.Migrations.down(version: 1) + end +end
Peace of mind from prototype to production