Merge pull request #2 from myastrallabs/feat/fetcher

update explorer

.formatter.exs

@@ -1,4 +1,5 @@
 [
+  line_length: 200,
   import_deps: [:ecto, :phoenix],
   inputs: ["*.{ex,exs}", "priv/*/seeds.exs", "{config,lib,test}/**/*.{ex,exs}"],
   subdirectories: ["priv/*/migrations"]

.gitignore

@@ -20,10 +20,13 @@ erl_crash.dump
 *.ez
 
 # Ignore package tarball (built via "mix hex.build").
-noncegeek-*.tar
+Noncegeek-*.tar
 
 # Ignore assets that are produced by build tools.
-/priv/static/assets/
+# Ignore assets that are produced by build tools.
+/priv/static/uploads/*
+/priv/static/assets/*
+/priv/static/images/*
 
 # Ignore digested assets cache.
 /priv/static/cache_manifest.json

.iex.exs

@@ -0,0 +1,5 @@
+import Ecto.Query, warn: false
+
+alias Noncegeek.Repo
+alias Noncegeek.Explorer
+alias Explorer.Model.{Event, Token, Collection}

Makefile

@@ -0,0 +1,4 @@
+iex: 
+	iex --erl "-kernel shell_history enabled" -S mix
+server:
+	iex --erl "-kernel shell_history enabled" -S mix phx.server

config/config.exs

@@ -33,8 +33,7 @@ config :swoosh, :api_client, false
 config :esbuild,
   version: "0.14.29",
   default: [
-    args:
-      ~w(js/app.js --bundle --target=es2017 --outdir=../priv/static/assets --external:/fonts/* --external:/images/*),
+    args: ~w(js/app.js --bundle --target=es2017 --outdir=../priv/static/assets --external:/fonts/* --external:/images/*),
     cd: Path.expand("../assets", __DIR__),
     env: %{"NODE_PATH" => Path.expand("../deps", __DIR__)}
   ]
@@ -47,8 +46,18 @@ config :logger, :console,
 # Use Jason for JSON parsing in Phoenix
 config :phoenix, :json_library, Jason
 
+config :noncegeek, Oban,
+  repo: Noncegeek.Repo,
+  plugins: [{Oban.Plugins.Pruner, max_age: 3 * 24 * 60 * 60}],
+  queues: [default: 10]
+
 config :noncegeek, AptosEx, rpc_endpoint: "https://testnet.aptoslabs.com/v1"
 
+config :noncegeek,
+  contract_address: "0xe698622471b41a92e13ae893ae4ff88b20c528f6da2bedcb24d74646bf972dc3",
+  contract_creator: "0xe10e40298c16778e71a03fa7e00e7d29e12a77b5e1797b799034551401cc0cc4",
+  collection_name: "NonceGeek Leaf"
+
 # Import environment specific config. This must remain at the bottom
 # of this file so it overrides the configuration defined above.
 import_config "#{config_env()}.exs"

lib/noncegeek/application.ex

@@ -16,9 +16,8 @@ defmodule Noncegeek.Application do
       {Phoenix.PubSub, name: Noncegeek.PubSub},
       # Start the Endpoint (http/https)
       NoncegeekWeb.Endpoint,
+      {Oban, Application.fetch_env!(:noncegeek, Oban)},
       {AptosEx, Application.fetch_env!(:noncegeek, AptosEx)}
-      # Start a worker by calling: Noncegeek.Worker.start_link(arg)
-      # {Noncegeek.Worker, arg}
     ]
 
     # See https://hexdocs.pm/elixir/Supervisor.html

lib/noncegeek/explorer.ex

@@ -0,0 +1,124 @@
+defmodule Noncegeek.Explorer do
+  @moduledoc """
+  The Explorer context.
+  """
+  import Ecto.Query, warn: false
+
+  require Integer
+
+  alias AptosEx
+
+  alias Noncegeek.Fetcher
+  alias Noncegeek.Explorer.Model.{Token, Event}
+  alias Noncegeek.{Repo, Turbo}
+
+  @doc """
+  Paged tokens
+  """
+  def paged_tokens() do
+    Turbo.all(Token)
+  end
+
+  @doc """
+  Get single token
+  """
+  def get_token(clauses) when is_list(clauses) or is_map(clauses),
+    do: Turbo.get_by(Token, clauses)
+
+  def get_token(clauses) when is_integer(clauses), do: Turbo.get(Token, clauses)
+
+  @doc """
+  fetch token data && store
+
+  ## Exmaples
+
+    iex> Noncegeek.Explorer.fetch_token_data("0xe19430a2498ff6800666d41cfd4b64d6d2a53574ef7457f700f96f4a61703d07", "DummyNames", "dummy1")
+
+  """
+  def fetch_token_data(creator, collection_name, token_name) do
+    with {:ok, token} <-
+           get_token(%{creator: creator, name: token_name, collection_name: collection_name}),
+         {:ok, data} <- AptosEx.get_token_data(creator, collection_name, token_name) do
+      Turbo.update(token, data)
+    end
+  end
+
+  def get_account_events(account, type) do
+    sequence_number = get_sequence_number(account, type)
+
+    case type do
+      "0x3::token::DepositEvent" ->
+        Fetcher.get_deposit_events(account, sequence_number)
+
+      "0x3::token::WithdrawEvent" ->
+        Fetcher.get_withdraw_events(account, sequence_number)
+    end
+  end
+
+  def get_sequence_number(account, type) do
+    from(e in Event,
+      where: e.type == ^type,
+      where: e.account_address == ^account,
+      order_by: [desc: e.sequence_number],
+      select: e.sequence_number,
+      limit: 1
+    )
+    |> Repo.one()
+    |> case do
+      nil -> 0
+      value -> value + 1
+    end
+  end
+
+  @doc """
+  fetch account events
+
+  iex> Noncegeek.Explorer.refresh_account_events("0xe19430a2498ff6800666d41cfd4b64d6d2a53574ef7457f700f96f4a61703d07")
+  iex> Noncegeek.Explorer.refresh_account_events("e698622471b41a92e13ae893ae4ff88b20c528f6da2bedcb24d74646bf972dc3")
+
+  """
+  def refresh_account_events(account) do
+    get_account_events(account, "0x3::token::DepositEvent")
+    get_account_events(account, "0x3::token::WithdrawEvent")
+
+    list_account_tokens(account)
+  end
+
+  @doc """
+  list account tokens
+  ## TODO
+
+  - [ ] user_tokens
+
+  """
+  def list_account_tokens(account) do
+    from(e in Event,
+      where:
+        e.account_address == ^account and
+          e.type in ["0x3::token::DepositEvent", "0x3::token::WithdrawEvent"],
+      order_by: [desc: e.version],
+      preload: [:token]
+    )
+    |> Repo.all()
+    |> case do
+      [] ->
+        []
+
+      value ->
+        value
+        |> Enum.group_by(& &1.token_id)
+        |> Enum.map(fn {_token_id, token_events} ->
+          case token_events |> length() |> Integer.is_even() do
+            true ->
+              nil
+
+            false ->
+              List.first(token_events)
+          end
+        end)
+        |> Enum.reject(&is_nil/1)
+        |> List.flatten()
+    end
+    |> then(&{:ok, &1})
+  end
+end

lib/noncegeek/explorer/jobs/fetch_token_data.ex

@@ -0,0 +1,45 @@
+defmodule Noncegeek.Explorer.Job.FetchTokenData do
+  @moduledoc false
+
+  require Logger
+
+  use Oban.Worker, queue: :default, priority: 1, max_attempts: 20
+
+  alias Noncegeek.Explorer
+
+  @contract_creator Application.get_env(:noncegeek, :contract_creator)
+  @collection_name Application.get_env(:noncegeek, :collection_name)
+
+  @impl Oban.Worker
+  def perform(%Oban.Job{args: %{"token_id" => token_id} = _args}) do
+    %{"token_data_id" => %{"creator" => creator, "collection" => collection_name, "name" => name}} = token_id
+
+    with {:ok, token} <- Explorer.fetch_token_data(creator, collection_name, name) do
+      IO.inspect(token, label: "token")
+      create_nft_image(token)
+      :ok
+    else
+      _ ->
+        {:error, :retry_fetcher_token_data}
+    end
+  end
+
+  defp create_nft_image(%{collection_name: unquote(@collection_name), creator: unquote(@contract_creator), name: name} = _token) do
+    IO.inspect("aaaaaaaaaaaaaaaaaaaaaaaaaaaaa")
+
+    unique_num =
+      name
+      |> String.split(":")
+      |> List.last()
+      |> String.trim()
+
+    file_path = "priv/static/images/#{unique_num}.jpg"
+
+    if !File.exists?(file_path) do
+      {:ok, %{body: body}} = Faker.Avatar.image_url() |> Tesla.get()
+      File.write!(file_path, body)
+    end
+  end
+
+  defp create_nft_image(_), do: nil
+end

lib/noncegeek/explorer/models/event.ex

@@ -19,7 +19,7 @@ defmodule Noncegeek.Explorer.Model.Event do
     field :amount, :decimal
     field :data, :map
 
-    belongs_to :token, Lotus.Explorer.Model.Token,
+    belongs_to :token, Noncegeek.Explorer.Model.Token,
       foreign_key: :token_id,
       references: :token_id,
       type: :map

lib/noncegeek/explorer/models/token.ex

@@ -9,7 +9,6 @@ defmodule Noncegeek.Explorer.Model.Token do
   schema "tokens" do
     # required
     field :token_id, :map
-    field :collection_id, :map
     field :collection_name, :string
     field :creator, :string
     field :name, :string
@@ -25,9 +24,6 @@ defmodule Noncegeek.Explorer.Model.Token do
     field :uri, :string
     field :property_version, :integer
 
-    # fetcher
-    field :last_fetched_at, :utc_datetime_usec
-
     timestamps()
   end
 
@@ -42,7 +38,6 @@ defmodule Noncegeek.Explorer.Model.Token do
     )a
 
     optional_fields = ~w(
-      last_fetched_at
       maximum
       largest_property_version
       mutability_config
@@ -69,7 +64,5 @@ defmodule Noncegeek.Explorer.Model.Token do
     token
     |> cast(attrs, required_fields)
     |> validate_required(required_fields)
-    |> NameSlug.maybe_generate_slug()
-    |> NameSlug.unique_constraint()
   end
 end

lib/noncegeek/fetcher.ex

@@ -0,0 +1,101 @@
+defmodule Noncegeek.Fetcher do
+  @moduledoc false
+
+  alias AptosEx
+
+  alias Noncegeek.Fetcher.Transform
+  alias Noncegeek.{Explorer, Import}
+
+  defmodule TaskData do
+    @moduledoc """
+    %Task{} with state && result && contract_name
+    """
+    defstruct event_handle: nil,
+              account: nil,
+              field: nil,
+              type: nil,
+              sequence_number: 0,
+              ref: nil
+
+    # def event_handle_id(%__MODULE__{event_handle: event_handle, field: field}), do: event_handle <> "::" <> field
+    def event_type(%__MODULE__{type: type}), do: type
+  end
+
+  @doc """
+  iex> Noncegeek.Fetcher.task(%{sequence_number: 0, account: "0xe698622471b41a92e13ae893ae4ff88b20c528f6da2bedcb24d74646bf972dc3", event_handle: "0xe698622471b41a92e13ae893ae4ff88b20c528f6da2bedcb24d74646bf972dc3::LEAF::MintData", field: "mint_events"})
+
+  """
+  def task(%{sequence_number: sequence_number} = task_data) do
+    case fetch_and_import_events(task_data) do
+      {:ok, %{events: events}} when events != [] ->
+        events
+        |> Enum.max_by(&Map.get(&1, :sequence_number))
+        |> Map.get(:sequence_number)
+        |> Kernel.+(1)
+
+      {:ok, _} ->
+        sequence_number
+
+        # {:error, _} ->
+        #   sequence_number
+    end
+  end
+
+  @doc """
+  Noncegeek.Fetcher.get_withdraw_events("0xe19430a2498ff6800666d41cfd4b64d6d2a53574ef7457f700f96f4a61703d07", 0)
+  """
+  def get_withdraw_events(account, sequence_number) do
+    task_data = %TaskData{
+      sequence_number: sequence_number,
+      account: account,
+      event_handle: "0x3::token::TokenStore",
+      field: "withdraw_events"
+    }
+
+    task(task_data)
+  end
+
+  @doc """
+  Noncegeek.Fetcher.get_deposit_events("0xe19430a2498ff6800666d41cfd4b64d6d2a53574ef7457f700f96f4a61703d07", 0)
+  """
+  def get_deposit_events(account, sequence_number) do
+    task_data = %TaskData{
+      sequence_number: sequence_number,
+      account: account,
+      event_handle: "0x3::token::TokenStore",
+      field: "deposit_events"
+    }
+
+    task(task_data)
+  end
+
+  defp fetch_and_import_events(
+         %{
+           sequence_number: sequence_number,
+           account: account,
+           field: field,
+           event_handle: event_handle
+         } = _task_data
+       ) do
+    with {:ok, event_list} <-
+           AptosEx.get_events(account, event_handle, field, start: sequence_number),
+         {:ok, import_list} <- Transform.params_set(event_list) do
+      {:ok, result} = Import.run(import_list)
+
+      async_fetcher(result)
+
+      {:ok, result}
+    end
+  end
+
+  defp async_fetcher(%{tokens: tokens}) do
+    tokens
+    |> Enum.each(fn item ->
+      %{token_id: item.token_id}
+      |> Explorer.Job.FetchTokenData.new()
+      |> Oban.insert()
+    end)
+  end
+
+  defp async_fetcher(_), do: :ok
+end