diff --git a/.github/workflows/publish-docker-image-for-redstone.yml b/.github/workflows/publish-docker-image-for-redstone.yml new file mode 100644 index 0000000000..029c42bc69 --- /dev/null +++ b/.github/workflows/publish-docker-image-for-redstone.yml @@ -0,0 +1,44 @@ +name: Redstone Publish Docker image + +on: + workflow_dispatch: + push: + branches: + - production-redstone +jobs: + push_to_registry: + name: Push Docker image to Docker Hub + runs-on: ubuntu-latest + env: + RELEASE_VERSION: ${{ vars.RELEASE_VERSION }} + DOCKER_CHAIN_NAME: redstone + steps: + - uses: actions/checkout@v4 + - name: Setup repo + uses: ./.github/actions/setup-repo-and-short-sha + with: + docker-username: ${{ secrets.DOCKER_USERNAME }} + docker-password: ${{ secrets.DOCKER_PASSWORD }} + + - name: Build and push Docker image + uses: docker/build-push-action@v5 + with: + context: . + file: ./docker/Dockerfile + push: true + tags: blockscout/blockscout-${{ env.DOCKER_CHAIN_NAME }}:latest, blockscout/blockscout-${{ env.DOCKER_CHAIN_NAME }}:${{ env.RELEASE_VERSION }}-postrelease-${{ env.SHORT_SHA }} + platforms: | + linux/amd64 + linux/arm64/v8 + build-args: | + CACHE_EXCHANGE_RATES_PERIOD= + API_V1_READ_METHODS_DISABLED=false + DISABLE_WEBAPP=false + API_V1_WRITE_METHODS_DISABLED=false + CACHE_TOTAL_GAS_USAGE_COUNTER_ENABLED= + ADMIN_PANEL_ENABLED=false + CACHE_ADDRESS_WITH_BALANCES_UPDATE_INTERVAL= + BLOCKSCOUT_VERSION=v${{ env.RELEASE_VERSION }}-beta.+commit.${{ env.SHORT_SHA }} + RELEASE_VERSION=${{ env.RELEASE_VERSION }} + CHAIN_TYPE=optimism + MUD_INDEXER_ENABLED=true \ No newline at end of file diff --git a/.github/workflows/release-redstone.yml b/.github/workflows/release-redstone.yml new file mode 100644 index 0000000000..9207dd195a --- /dev/null +++ b/.github/workflows/release-redstone.yml @@ -0,0 +1,46 @@ +name: Release for Redstone + +on: + release: + types: [published] + +env: + OTP_VERSION: ${{ vars.OTP_VERSION }} + ELIXIR_VERSION: ${{ vars.ELIXIR_VERSION }} + +jobs: + push_to_registry: + name: Push Docker image to Docker Hub + runs-on: ubuntu-latest + env: + RELEASE_VERSION: ${{ vars.RELEASE_VERSION }} + steps: + - uses: actions/checkout@v4 + - name: Setup repo + uses: ./.github/actions/setup-repo + with: + docker-username: ${{ secrets.DOCKER_USERNAME }} + docker-password: ${{ secrets.DOCKER_PASSWORD }} + + - name: Build and push Docker image for Redstone + uses: docker/build-push-action@v5 + with: + context: . + file: ./docker/Dockerfile + push: true + tags: blockscout/blockscout-redstone:latest, blockscout/blockscout-redstone:${{ env.RELEASE_VERSION }} + platforms: | + linux/amd64 + linux/arm64/v8 + build-args: | + CACHE_EXCHANGE_RATES_PERIOD= + API_V1_READ_METHODS_DISABLED=false + DISABLE_WEBAPP=false + API_V1_WRITE_METHODS_DISABLED=false + CACHE_TOTAL_GAS_USAGE_COUNTER_ENABLED= + ADMIN_PANEL_ENABLED=false + CACHE_ADDRESS_WITH_BALANCES_UPDATE_INTERVAL= + BLOCKSCOUT_VERSION=v${{ env.RELEASE_VERSION }}-beta + RELEASE_VERSION=${{ env.RELEASE_VERSION }} + CHAIN_TYPE=optimism + MUD_INDEXER_ENABLED=true \ No newline at end of file diff --git a/apps/block_scout_web/lib/block_scout_web/api_router.ex b/apps/block_scout_web/lib/block_scout_web/api_router.ex index 939e55297c..475d896416 100644 --- a/apps/block_scout_web/lib/block_scout_web/api_router.ex +++ b/apps/block_scout_web/lib/block_scout_web/api_router.ex @@ -346,6 +346,18 @@ defmodule BlockScoutWeb.ApiRouter do get("/batches/:batch_number", V2.ZkSyncController, :batch) end end + + scope "/mud" do + if Application.compile_env(:explorer, Explorer.Chain.Mud)[:enabled] do + get("/worlds", V2.MudController, :worlds) + get("/worlds/count", V2.MudController, :worlds_count) + get("/worlds/:world/tables", V2.MudController, :world_tables) + get("/worlds/:world/tables/count", V2.MudController, :world_tables_count) + get("/worlds/:world/tables/:table_id/records", V2.MudController, :world_table_records) + get("/worlds/:world/tables/:table_id/records/count", V2.MudController, :world_table_records_count) + get("/worlds/:world/tables/:table_id/records/:record_id", V2.MudController, :world_table_record) + end + end end scope "/v1/graphql" do diff --git a/apps/block_scout_web/lib/block_scout_web/controllers/api/v2/mud_controller.ex b/apps/block_scout_web/lib/block_scout_web/controllers/api/v2/mud_controller.ex new file mode 100644 index 0000000000..8f9d081084 --- /dev/null +++ b/apps/block_scout_web/lib/block_scout_web/controllers/api/v2/mud_controller.ex @@ -0,0 +1,261 @@ +defmodule BlockScoutWeb.API.V2.MudController do + use BlockScoutWeb, :controller + + import BlockScoutWeb.Chain, + only: [ + next_page_params: 4, + split_list_by_page: 1, + default_paging_options: 0 + ] + + import BlockScoutWeb.PagingHelper, only: [mud_records_sorting: 1] + + alias Explorer.Chain.{Data, Hash, Mud, Mud.Schema.FieldSchema, Mud.Table} + + action_fallback(BlockScoutWeb.API.V2.FallbackController) + + @doc """ + Function to handle GET requests to `/api/v2/mud/worlds` endpoint. + """ + @spec worlds(Plug.Conn.t(), map()) :: Plug.Conn.t() + def worlds(conn, params) do + {worlds, next_page} = + params + |> mud_paging_options(["world"], [Hash.Address]) + |> Mud.worlds_list() + |> split_list_by_page() + + next_page_params = + next_page_params(next_page, worlds, conn.query_params, fn item -> + %{"world" => item} + end) + + conn + |> put_status(200) + |> render(:worlds, %{worlds: worlds, next_page_params: next_page_params}) + end + + @doc """ + Function to handle GET requests to `/api/v2/mud/worlds/count` endpoint. + """ + @spec worlds_count(Plug.Conn.t(), map()) :: Plug.Conn.t() + def worlds_count(conn, _params) do + count = Mud.worlds_count() + + conn + |> put_status(200) + |> render(:count, %{count: count}) + end + + @doc """ + Function to handle GET requests to `/api/v2/mud/worlds/:world/tables` endpoint. + """ + @spec world_tables(Plug.Conn.t(), map()) :: Plug.Conn.t() + def world_tables(conn, %{"world" => world_param} = params) do + with {:format, {:ok, world}} <- {:format, Hash.Address.cast(world_param)} do + options = params |> mud_paging_options(["table_id"], [Hash.Full]) |> Keyword.merge(mud_tables_filter(params)) + + {tables, next_page} = + world + |> Mud.world_tables(options) + |> split_list_by_page() + + next_page_params = + next_page_params(next_page, tables, conn.query_params, fn item -> + %{"table_id" => item |> elem(0)} + end) + + conn + |> put_status(200) + |> render(:tables, %{tables: tables, next_page_params: next_page_params}) + end + end + + @doc """ + Function to handle GET requests to `/api/v2/mud/worlds/:world/tables/count` endpoint. + """ + @spec world_tables_count(Plug.Conn.t(), map()) :: Plug.Conn.t() + def world_tables_count(conn, %{"world" => world_param} = params) do + with {:format, {:ok, world}} <- {:format, Hash.Address.cast(world_param)} do + options = params |> mud_tables_filter() + + count = Mud.world_tables_count(world, options) + + conn + |> put_status(200) + |> render(:count, %{count: count}) + end + end + + @doc """ + Function to handle GET requests to `/api/v2/mud/worlds/:world/tables/:table_id/records` endpoint. + """ + @spec world_table_records(Plug.Conn.t(), map()) :: Plug.Conn.t() + def world_table_records(conn, %{"world" => world_param, "table_id" => table_id_param} = params) do + with {:format, {:ok, world}} <- {:format, Hash.Address.cast(world_param)}, + {:format, {:ok, table_id}} <- {:format, Hash.Full.cast(table_id_param)}, + {:ok, schema} <- Mud.world_table_schema(world, table_id) do + options = + params + |> mud_paging_options(["key_bytes", "key0", "key1"], [Data, Hash.Full, Hash.Full]) + |> Keyword.merge(mud_records_filter(params, schema)) + |> Keyword.merge(mud_records_sorting(params)) + + {records, next_page} = world |> Mud.world_table_records(table_id, options) |> split_list_by_page() + + blocks = Mud.preload_records_timestamps(records) + + next_page_params = + next_page_params(next_page, records, conn.query_params, fn item -> + keys = [item.key_bytes, item.key0, item.key1] |> Enum.filter(&(!is_nil(&1))) + ["key_bytes", "key0", "key1"] |> Enum.zip(keys) |> Enum.into(%{}) + end) + + conn + |> put_status(200) + |> render(:records, %{ + records: records, + table_id: table_id, + schema: schema, + blocks: blocks, + next_page_params: next_page_params + }) + end + end + + @doc """ + Function to handle GET requests to `/api/v2/mud/worlds/:world/tables/:table_id/records/count` endpoint. + """ + @spec world_table_records_count(Plug.Conn.t(), map()) :: Plug.Conn.t() + def world_table_records_count(conn, %{"world" => world_param, "table_id" => table_id_param} = params) do + with {:format, {:ok, world}} <- {:format, Hash.Address.cast(world_param)}, + {:format, {:ok, table_id}} <- {:format, Hash.Full.cast(table_id_param)}, + {:ok, schema} <- Mud.world_table_schema(world, table_id) do + options = params |> mud_records_filter(schema) + + count = Mud.world_table_records_count(world, table_id, options) + + conn + |> put_status(200) + |> render(:count, %{count: count}) + end + end + + @doc """ + Function to handle GET requests to `/api/v2/mud/worlds/:world/tables/:table_id/records/:record_id` endpoint. + """ + @spec world_table_record(Plug.Conn.t(), map()) :: Plug.Conn.t() + def world_table_record( + conn, + %{"world" => world_param, "table_id" => table_id_param, "record_id" => record_id_param} = _params + ) do + with {:format, {:ok, world}} <- {:format, Hash.Address.cast(world_param)}, + {:format, {:ok, table_id}} <- {:format, Hash.Full.cast(table_id_param)}, + {:format, {:ok, record_id}} <- {:format, Data.cast(record_id_param)}, + {:ok, schema} <- Mud.world_table_schema(world, table_id), + {:ok, record} <- Mud.world_table_record(world, table_id, record_id) do + blocks = Mud.preload_records_timestamps([record]) + + conn + |> put_status(200) + |> render(:record, %{record: record, table_id: table_id, schema: schema, blocks: blocks}) + end + end + + defp mud_tables_filter(params) do + Enum.reduce(params, [], fn {key, value}, acc -> + case key do + "filter_namespace" -> + Keyword.put(acc, :filter_namespace, parse_namespace_string(value)) + + "q" -> + Keyword.put(acc, :filter_search, parse_search_string(value)) + + _ -> + acc + end + end) + end + + defp parse_namespace_string(namespace) do + filter = + case namespace do + nil -> {:ok, nil} + "0x" <> hex -> Base.decode16(hex, case: :mixed) + str -> {:ok, str} + end + + case filter do + {:ok, ns} when is_binary(ns) and byte_size(ns) <= 14 -> + ns |> String.pad_trailing(14, <<0>>) + + _ -> + nil + end + end + + defp parse_search_string(q) do + # If the search string looks like hex-encoded table id or table full name, + # we try to parse and filter by that table id directly. + # Otherwise we do a full-text search of given string inside table id. + with :error <- Hash.Full.cast(q), + :error <- Table.table_full_name_to_table_id(q) do + q + else + {:ok, table_id} -> table_id + end + end + + defp mud_records_filter(params, schema) do + Enum.reduce(params, [], fn {key, value}, acc -> + case key do + "filter_key0" -> Keyword.put(acc, :filter_key0, encode_filter(value, schema, 0)) + "filter_key1" -> Keyword.put(acc, :filter_key1, encode_filter(value, schema, 1)) + _ -> acc + end + end) + end + + defp encode_filter(value, schema, field_idx) do + case value do + "false" -> + <<0::256>> + + "true" -> + <<1::256>> + + "0x" <> hex -> + bin = Base.decode16!(hex, case: :mixed) + # addresses are padded to 32 bytes with zeros on the right + if FieldSchema.type_of(schema.key_schema, field_idx) == 97 do + <<0::size(256 - byte_size(bin) * 8), bin::binary>> + else + <> + end + + dec -> + num = dec |> Integer.parse() |> elem(0) + <> + end + end + + defp mud_paging_options(params, keys, types) do + page_key = + keys + |> Enum.zip(types) + |> Enum.reduce(%{}, fn {key, type}, acc -> + with param when param != nil <- Map.get(params, key), + {:ok, val} <- type.cast(param) do + acc |> Map.put(String.to_existing_atom(key), val) + else + _ -> acc + end + end) + + if page_key == %{} do + [paging_options: default_paging_options()] + else + [paging_options: %{default_paging_options() | key: page_key}] + end + end +end diff --git a/apps/block_scout_web/lib/block_scout_web/paging_helper.ex b/apps/block_scout_web/lib/block_scout_web/paging_helper.ex index 4bd8be1d40..bea0045e3a 100644 --- a/apps/block_scout_web/lib/block_scout_web/paging_helper.ex +++ b/apps/block_scout_web/lib/block_scout_web/paging_helper.ex @@ -303,4 +303,21 @@ defmodule BlockScoutWeb.PagingHelper do do: [{:dynamic, :blocks_validated, :desc_nulls_last, ValidatorStability.dynamic_validated_blocks()}] defp do_validators_stability_sorting(_, _), do: [] + + @spec mud_records_sorting(%{required(String.t()) => String.t()}) :: [ + {:sorting, SortingHelper.sorting_params()} + ] + def mud_records_sorting(%{"sort" => sort_field, "order" => order}) do + [sorting: do_mud_records_sorting(sort_field, order)] + end + + def mud_records_sorting(_), do: [] + + defp do_mud_records_sorting("key_bytes", "asc"), do: [asc_nulls_first: :key_bytes] + defp do_mud_records_sorting("key_bytes", "desc"), do: [desc_nulls_last: :key_bytes] + defp do_mud_records_sorting("key0", "asc"), do: [asc_nulls_first: :key0] + defp do_mud_records_sorting("key0", "desc"), do: [desc_nulls_last: :key0] + defp do_mud_records_sorting("key1", "asc"), do: [asc_nulls_first: :key1] + defp do_mud_records_sorting("key1", "desc"), do: [desc_nulls_last: :key1] + defp do_mud_records_sorting(_, _), do: [] end diff --git a/apps/block_scout_web/lib/block_scout_web/views/api/v2/mud_view.ex b/apps/block_scout_web/lib/block_scout_web/views/api/v2/mud_view.ex new file mode 100644 index 0000000000..128c748bb9 --- /dev/null +++ b/apps/block_scout_web/lib/block_scout_web/views/api/v2/mud_view.ex @@ -0,0 +1,85 @@ +defmodule BlockScoutWeb.API.V2.MudView do + use BlockScoutWeb, :view + + alias Explorer.Chain.{Mud, Mud.Table} + + @doc """ + Function to render GET requests to `/api/v2/mud/worlds` endpoint. + """ + @spec render(String.t(), map()) :: map() + def render("worlds.json", %{worlds: worlds, next_page_params: next_page_params}) do + %{ + items: worlds, + next_page_params: next_page_params + } + end + + @doc """ + Function to render GET requests to `/api/v2/mud/worlds/count` endpoint. + """ + def render("count.json", %{count: count}) do + %{ + count: count + } + end + + @doc """ + Function to render GET requests to `/api/v2/mud/worlds/:world/tables` endpoint. + """ + def render("tables.json", %{tables: tables, next_page_params: next_page_params}) do + %{ + items: tables |> Enum.map(&%{table: Table.from(&1 |> elem(0)), schema: &1 |> elem(1)}), + next_page_params: next_page_params + } + end + + @doc """ + Function to render GET requests to `/api/v2/mud/worlds/:world/tables/:table_id/records` endpoint. + """ + def render("records.json", %{ + records: records, + table_id: table_id, + schema: schema, + blocks: blocks, + next_page_params: next_page_params + }) do + %{ + items: records |> Enum.map(&format_record(&1, schema, blocks)), + table: table_id |> Table.from(), + schema: schema, + next_page_params: next_page_params + } + end + + @doc """ + Function to render GET requests to `/api/v2/mud/worlds/:world/tables/:table_id/records/:record_id` endpoint. + """ + def render("record.json", %{record: record, table_id: table_id, blocks: blocks, schema: schema}) do + %{ + record: record |> format_record(schema, blocks), + table: table_id |> Table.from(), + schema: schema + } + end + + defp format_record(nil, _schema, _blocks), do: nil + + defp format_record(record, schema, blocks) do + %{ + id: record.key_bytes, + raw: %{ + key_bytes: record.key_bytes, + key0: record.key0, + key1: record.key1, + static_data: record.static_data, + encoded_lengths: record.encoded_lengths, + dynamic_data: record.dynamic_data, + block_number: record.block_number, + log_index: record.log_index + }, + is_deleted: record.is_deleted, + decoded: Mud.decode_record(record, schema), + timestamp: blocks |> Map.get(Decimal.to_integer(record.block_number), nil) + } + end +end diff --git a/apps/block_scout_web/test/test_helper.exs b/apps/block_scout_web/test/test_helper.exs index 11003096da..ee03ef1827 100644 --- a/apps/block_scout_web/test/test_helper.exs +++ b/apps/block_scout_web/test/test_helper.exs @@ -35,6 +35,7 @@ Ecto.Adapters.SQL.Sandbox.mode(Explorer.Repo.Beacon, :manual) Ecto.Adapters.SQL.Sandbox.mode(Explorer.Repo.Stability, :manual) Ecto.Adapters.SQL.Sandbox.mode(Explorer.Repo.BridgedTokens, :manual) Ecto.Adapters.SQL.Sandbox.mode(Explorer.Repo.Filecoin, :manual) +Ecto.Adapters.SQL.Sandbox.mode(Explorer.Repo.Mud, :manual) Absinthe.Test.prime(BlockScoutWeb.GraphQL.Schema) diff --git a/apps/explorer/config/config.exs b/apps/explorer/config/config.exs index ca7095c15b..78aded683b 100644 --- a/apps/explorer/config/config.exs +++ b/apps/explorer/config/config.exs @@ -150,6 +150,8 @@ config :explorer, :http_adapter, HTTPoison config :explorer, Explorer.Chain.BridgedToken, enabled: ConfigHelper.parse_bool_env_var("BRIDGED_TOKENS_ENABLED") +config :explorer, Explorer.Chain.Mud, enabled: ConfigHelper.parse_bool_env_var("MUD_INDEXER_ENABLED") + config :logger, :explorer, # keep synced with `config/config.exs` format: "$dateT$time $metadata[$level] $message\n", diff --git a/apps/explorer/config/dev.exs b/apps/explorer/config/dev.exs index a387ee2422..36ba586294 100644 --- a/apps/explorer/config/dev.exs +++ b/apps/explorer/config/dev.exs @@ -37,6 +37,8 @@ config :explorer, Explorer.Repo.Filecoin, timeout: :timer.seconds(80) config :explorer, Explorer.Repo.Stability, timeout: :timer.seconds(80) +config :explorer, Explorer.Repo.Mud, timeout: :timer.seconds(80) + config :explorer, Explorer.Tracer, env: "dev", disabled?: true config :logger, :explorer, diff --git a/apps/explorer/config/prod.exs b/apps/explorer/config/prod.exs index 27fa8cad95..f8337d04ca 100644 --- a/apps/explorer/config/prod.exs +++ b/apps/explorer/config/prod.exs @@ -60,6 +60,10 @@ config :explorer, Explorer.Repo.Stability, prepare: :unnamed, timeout: :timer.seconds(60) +config :explorer, Explorer.Repo.Mud, + prepare: :unnamed, + timeout: :timer.seconds(60) + config :explorer, Explorer.Tracer, env: "production", disabled?: true config :logger, :explorer, diff --git a/apps/explorer/config/test.exs b/apps/explorer/config/test.exs index 9ace0f12c2..64d37ba266 100644 --- a/apps/explorer/config/test.exs +++ b/apps/explorer/config/test.exs @@ -54,7 +54,8 @@ for repo <- [ Explorer.Repo.Suave, Explorer.Repo.BridgedTokens, Explorer.Repo.Filecoin, - Explorer.Repo.Stability + Explorer.Repo.Stability, + Explorer.Repo.Mud ] do config :explorer, repo, database: "explorer_test", diff --git a/apps/explorer/lib/explorer/application.ex b/apps/explorer/lib/explorer/application.ex index 3b1c32cfeb..72c1296a5e 100644 --- a/apps/explorer/lib/explorer/application.ex +++ b/apps/explorer/lib/explorer/application.ex @@ -141,7 +141,7 @@ defmodule Explorer.Application do ] |> List.flatten() - repos_by_chain_type() ++ account_repo() ++ configurable_children_set + repos_by_chain_type() ++ account_repo() ++ mud_repo() ++ configurable_children_set end defp repos_by_chain_type do @@ -172,6 +172,14 @@ defmodule Explorer.Application do end end + defp mud_repo do + if Application.get_env(:explorer, Explorer.Chain.Mud)[:enabled] || Mix.env() == :test do + [Explorer.Repo.Mud] + else + [] + end + end + defp should_start?(process) do Application.get_env(:explorer, process, [])[:enabled] == true end diff --git a/apps/explorer/lib/explorer/chain/mud.ex b/apps/explorer/lib/explorer/chain/mud.ex new file mode 100644 index 0000000000..4f8f2895d9 --- /dev/null +++ b/apps/explorer/lib/explorer/chain/mud.ex @@ -0,0 +1,395 @@ +defmodule Explorer.Chain.Mud do + @moduledoc """ + Represents a MUD framework database record. + """ + use Explorer.Schema + + import Ecto.Query, + only: [ + distinct: 2, + order_by: 3, + select: 3, + where: 3, + limit: 2 + ] + + alias ABI.TypeDecoder + alias Explorer.{Chain, PagingOptions, Repo, SortingHelper} + + alias Explorer.Chain.{ + Address, + Block, + Data, + Hash, + Mud, + Mud.Schema, + Mud.Schema.FieldSchema + } + + require Logger + + @schema_prefix "mud" + + @store_tables_table_id Base.decode16!("746273746f72650000000000000000005461626c657300000000000000000000", case: :lower) + + # https://github.com/latticexyz/mud/blob/cc4f4246e52982354e398113c46442910f9b04bb/packages/store/src/codegen/tables/Tables.sol#L34-L42 + @store_tables_table_schema %Schema{ + key_schema: FieldSchema.from("0x002001005f000000000000000000000000000000000000000000000000000000"), + value_schema: FieldSchema.from("0x006003025f5f5fc4c40000000000000000000000000000000000000000000000"), + key_names: ["tableId"], + value_names: ["fieldLayout", "keySchema", "valueSchema", "abiEncodedKeyNames", "abiEncodedValueNames"] + } + + @primary_key false + typed_schema "records" do + field(:address, Hash.Address, null: false) + field(:table_id, Hash.Full, null: false) + field(:key_bytes, Data) + field(:key0, Hash.Full) + field(:key1, Hash.Full) + field(:static_data, Data) + field(:encoded_lengths, Data) + field(:dynamic_data, Data) + field(:is_deleted, :boolean, null: false) + field(:block_number, :decimal, null: false) + field(:log_index, :decimal, null: false) + end + + def enabled? do + Application.get_env(:explorer, __MODULE__)[:enabled] + end + + @doc """ + Returns the paginated list of registered MUD world addresses. + """ + @spec worlds_list(Keyword.t()) :: [Mud.t()] + def worlds_list(options \\ []) do + paging_options = Keyword.get(options, :paging_options, Chain.default_paging_options()) + + Mud + |> select([r], r.address) + |> distinct(true) + |> page_worlds(paging_options) + |> limit(^paging_options.page_size) + |> Repo.Mud.all() + end + + defp page_worlds(query, %PagingOptions{key: %{world: world}}) do + query |> where([item], item.address > ^world) + end + + defp page_worlds(query, _), do: query + + @doc """ + Returns the total number of registered MUD worlds. + """ + @spec worlds_count() :: non_neg_integer() + def worlds_count do + Mud + |> select([r], r.address) + |> distinct(true) + |> Repo.Mud.aggregate(:count) + end + + @doc """ + Returns the decoded MUD table schema by world address and table ID. + """ + @spec world_table_schema(Hash.Address.t(), Hash.Full.t()) :: {:ok, Schema.t()} | {:error, :not_found} + def world_table_schema(world, table_id) do + Mud + |> where([r], r.address == ^world and r.table_id == ^@store_tables_table_id and r.key0 == ^table_id) + |> Repo.Mud.one() + |> case do + nil -> + {:error, :not_found} + + r -> + {:ok, decode_schema(r)} + end + end + + @doc """ + Returns the paginated list of registered MUD tables in the given world, optionally filtered by namespace or table name. + Each returned table in the resulting list is represented as a tuple of its ID and decoded schema. + """ + @spec world_tables(Hash.Address.t(), Keyword.t()) :: [{Hash.Full.t(), Schema.t()}] + def world_tables(world, options \\ []) do + paging_options = Keyword.get(options, :paging_options, Chain.default_paging_options()) + filter_namespace = Keyword.get(options, :filter_namespace, nil) + filter_search = Keyword.get(options, :filter_search, nil) + + Mud + |> where([r], r.address == ^world and r.table_id == ^@store_tables_table_id) + |> filter_tables_by_namespace(filter_namespace) + |> filter_tables_by_search(filter_search) + |> page_tables(paging_options) + |> order_by([r], asc: r.key0) + |> limit(^paging_options.page_size) + |> Repo.Mud.all() + |> Enum.map(&{&1.key0, decode_schema(&1)}) + end + + defp page_tables(query, %PagingOptions{key: %{table_id: table_id}}) do + query |> where([item], item.key0 > ^table_id) + end + + defp page_tables(query, _), do: query + + @doc """ + Returns the number of registered MUD tables in the given world. + """ + @spec world_tables_count(Hash.Address.t(), Keyword.t()) :: non_neg_integer() + def world_tables_count(world, options \\ []) do + filter_namespace = Keyword.get(options, :filter_namespace, nil) + filter_search = Keyword.get(options, :filter_search, nil) + + Mud + |> where([r], r.address == ^world and r.table_id == ^@store_tables_table_id) + |> filter_tables_by_namespace(filter_namespace) + |> filter_tables_by_search(filter_search) + |> Repo.Mud.aggregate(:count) + end + + defp filter_tables_by_namespace(query, nil), do: query + + defp filter_tables_by_namespace(query, namespace) do + query |> where([tb], fragment("substring(? FROM 3 FOR 14)", tb.key0) == ^namespace) + end + + defp filter_tables_by_search(query, %Hash{} = table_id) do + query |> where([tb], tb.key0 == ^table_id) + end + + defp filter_tables_by_search(query, search_string) when is_binary(search_string) do + query |> where([tb], ilike(fragment("encode(?, 'escape')", tb.key0), ^"%#{search_string}%")) + end + + defp filter_tables_by_search(query, _), do: query + + @default_sorting [ + asc: :key_bytes + ] + + @doc """ + Returns the paginated list of raw MUD records in the given world table. + Resulting records can be sorted or filtered by any of the first 2 key columns. + """ + @spec world_table_records(Hash.Address.t(), Hash.Full.t(), Keyword.t()) :: [Mud.t()] + def world_table_records(world, table_id, options \\ []) do + paging_options = Keyword.get(options, :paging_options, Chain.default_paging_options()) + sorting = Keyword.get(options, :sorting, []) + + Mud + |> where([r], r.address == ^world and r.table_id == ^table_id and r.is_deleted == false) + |> filter_records(:key0, Keyword.get(options, :filter_key0)) + |> filter_records(:key1, Keyword.get(options, :filter_key1)) + |> SortingHelper.apply_sorting(sorting, @default_sorting) + |> SortingHelper.page_with_sorting(paging_options, sorting, @default_sorting) + |> Repo.Mud.all() + end + + @doc """ + Preloads last modification timestamps for the list of raw MUD records. + + Returns a map of block numbers to timestamps. + """ + @spec preload_records_timestamps([Mud.t()]) :: %{non_neg_integer() => DateTime.t()} + def preload_records_timestamps(records) do + block_numbers = records |> Enum.map(&(&1.block_number |> Decimal.to_integer())) |> Enum.uniq() + + Block + |> where([b], b.number in ^block_numbers) + |> select([b], {b.number, b.timestamp}) + |> Repo.all() + |> Enum.into(%{}) + end + + @doc """ + Returns the number of MUD records in the given world table. + """ + @spec world_table_records_count(Hash.Address.t(), Hash.Full.t(), Keyword.t()) :: non_neg_integer() + def world_table_records_count(world, table_id, options \\ []) do + Mud + |> where([r], r.address == ^world and r.table_id == ^table_id and r.is_deleted == false) + |> filter_records(:key0, Keyword.get(options, :filter_key0)) + |> filter_records(:key1, Keyword.get(options, :filter_key1)) + |> Repo.Mud.aggregate(:count) + end + + defp filter_records(query, _key_name, nil), do: query + + defp filter_records(query, :key0, key), do: query |> where([r], r.key0 == ^key) + + defp filter_records(query, :key1, key), do: query |> where([r], r.key1 == ^key) + + @doc """ + Returns the raw MUD record from the given world table by its ID. + """ + @spec world_table_record(Hash.Address.t(), Hash.Full.t(), Data.t()) :: {:ok, Mud.t()} | {:error, :not_found} + def world_table_record(world, table_id, record_id) do + Mud + |> where([r], r.address == ^world and r.table_id == ^table_id and r.key_bytes == ^record_id) + |> Repo.Mud.one() + |> case do + nil -> + {:error, :not_found} + + r -> + {:ok, r} + end + end + + defp decode_schema(nil), do: nil + + defp decode_schema(record) do + schema_record = decode_record(record, @store_tables_table_schema) + + %Schema{ + key_schema: schema_record["keySchema"] |> FieldSchema.from(), + value_schema: schema_record["valueSchema"] |> FieldSchema.from(), + key_names: schema_record["abiEncodedKeyNames"] |> decode_abi_encoded_strings(), + value_names: schema_record["abiEncodedValueNames"] |> decode_abi_encoded_strings() + } + end + + defp decode_abi_encoded_strings("0x" <> hex_encoded) do + hex_encoded + |> Base.decode16!(case: :mixed) + |> TypeDecoder.decode_raw([{:array, :string}]) + |> Enum.at(0) + end + + @doc """ + Decodes a given raw MUD record according to table schema. + + Returns a JSON-like map with decoded field names and values. + """ + @spec decode_record(Mud.t() | nil, Schema.t() | nil) :: map() | nil + def decode_record(nil, _schema), do: nil + + def decode_record(_record, nil), do: nil + + def decode_record(record, schema) do + key = decode_key_tuple(record.key_bytes.bytes, schema.key_names, schema.key_schema) + + value = + if record.is_deleted do + schema.value_names |> Enum.into(%{}, &{&1, nil}) + else + decode_fields( + record.static_data, + record.encoded_lengths, + record.dynamic_data, + schema.value_names, + schema.value_schema + ) + end + + key |> Map.merge(value) + end + + defp decode_key_tuple(key_bytes, fields, layout_schema) do + {_, types} = Schema.decode_types(layout_schema) + + fields + |> Enum.zip(types) + |> Enum.reduce({%{}, key_bytes}, fn {field, type}, {acc, data} -> + type_size = static_type_size(type) + <> = data + + enc = + if type < 64 or type >= 96 do + :binary.part(word, 32 - type_size, type_size) + else + :binary.part(word, 0, type_size) + end + + decoded = decode_type(type, enc) + + {Map.put(acc, field, decoded), rest} + end) + |> elem(0) + end + + defp decode_fields(static_data, encoded_lengths, dynamic_data, fields, layout_schema) do + {static_fields_count, types} = Schema.decode_types(layout_schema) + + {static_types, dynamic_types} = Enum.split(types, static_fields_count) + + {static_fields, dynamic_fields} = Enum.split(fields, static_fields_count) + + res = + static_fields + |> Enum.zip(static_types) + |> Enum.reduce({%{}, (static_data && static_data.bytes) || <<>>}, fn {field, type}, {acc, data} -> + type_size = static_type_size(type) + <> = data + decoded = decode_type(type, enc) + {Map.put(acc, field, decoded), rest} + end) + |> elem(0) + + if encoded_lengths == nil or byte_size(encoded_lengths.bytes) == 0 do + res + else + dynamic_type_lengths = + encoded_lengths.bytes + |> :binary.bin_to_list(0, 25) + |> Enum.chunk_every(5) + |> Enum.reverse() + |> Enum.map(&(&1 |> :binary.list_to_bin() |> :binary.decode_unsigned())) + + [dynamic_fields, dynamic_types, dynamic_type_lengths] + |> Enum.zip() + |> Enum.reduce({res, (dynamic_data && dynamic_data.bytes) || <<>>}, fn {field, type, length}, {acc, data} -> + <> = data + decoded = decode_type(type, enc) + + {Map.put(acc, field, decoded), rest} + end) + |> elem(0) + end + end + + defp static_type_size(type) do + case type do + _ when type < 97 -> rem(type, 32) + 1 + 97 -> 20 + _ -> 0 + end + end + + # credo:disable-for-next-line Credo.Check.Refactor.CyclomaticComplexity + defp decode_type(type, raw) do + case type do + _ when type < 32 -> + raw |> :binary.decode_unsigned() |> Integer.to_string() + + _ when type < 64 -> + size = static_type_size(type) + <> = raw + int |> Integer.to_string() + + _ when type < 96 or type == 196 -> + "0x" <> Base.encode16(raw, case: :lower) + + 96 -> + raw == <<1>> + + 97 -> + Address.checksum(raw) + + _ when type < 196 -> + raw + |> :binary.bin_to_list() + |> Enum.chunk_every(static_type_size(type - 98)) + |> Enum.map(&decode_type(type - 98, :binary.list_to_bin(&1))) + + 197 -> + raw + + _ -> + raise "Unknown type: #{type}" + end + end +end diff --git a/apps/explorer/lib/explorer/chain/mud/schema.ex b/apps/explorer/lib/explorer/chain/mud/schema.ex new file mode 100644 index 0000000000..42c22359f9 --- /dev/null +++ b/apps/explorer/lib/explorer/chain/mud/schema.ex @@ -0,0 +1,110 @@ +defmodule Explorer.Chain.Mud.Schema do + @moduledoc """ + Represents a MUD framework database record schema. + """ + + defmodule FieldSchema do + @moduledoc """ + Represents a MUD framework database record field schema. Describes number of columns and their types. + """ + + defstruct [:word] + + @typedoc """ + The MUD field schema. + * `word` - The field schema as 32-byte value. + """ + @type t :: %__MODULE__{ + word: <<_::256>> + } + + @doc """ + Decodes field schema type from raw binary or hex-encoded string. + """ + @spec from(binary()) :: t() | :error + def from(<>), do: %__MODULE__{word: bin} + + def from("0x" <> <>) do + with {:ok, bin} <- Base.decode16(hex, case: :mixed) do + %__MODULE__{word: bin} + end + end + + def from(_), do: :error + + @doc """ + Tells the type of the field at index `index` in the field schema. + """ + @spec type_of(t(), non_neg_integer()) :: non_neg_integer() + def type_of(%FieldSchema{word: word}, index), do: :binary.at(word, index + 4) + end + + @enforce_keys [:key_schema, :value_schema, :key_names, :value_names] + defstruct [:key_schema, :value_schema, :key_names, :value_names] + + @typedoc """ + The MUD table schema. Describe column types and names for the given MUD table. + * `key_schema` - The field schema for the key columns. + * `value_schema` - The field schema for the value columns. + * `key_names` - The names of the key columns. + * `value_names` - The names of the value columns. + """ + @type t :: %__MODULE__{ + key_schema: FieldSchema.t(), + value_schema: FieldSchema.t(), + key_names: [String.t()], + value_names: [String.t()] + } + + defimpl Jason.Encoder, for: Explorer.Chain.Mud.Schema do + alias Explorer.Chain.Mud.Schema + alias Jason.Encode + + def encode(data, opts) do + Encode.map( + %{ + "key_types" => data.key_schema |> Schema.decode_type_names(), + "value_types" => data.value_schema |> Schema.decode_type_names(), + "key_names" => data.key_names, + "value_names" => data.value_names + }, + opts + ) + end + end + + @doc """ + Tells the number of static fields in the schema and the list of raw type IDs of all fields in the schema. + """ + @spec decode_types(FieldSchema.t()) :: {non_neg_integer(), [non_neg_integer()]} + def decode_types(layout_schema) do + static_fields_count = :binary.at(layout_schema.word, 2) + dynamic_fields_count = :binary.at(layout_schema.word, 3) + + {static_fields_count, :binary.bin_to_list(layout_schema.word, 4, static_fields_count + dynamic_fields_count)} + end + + @doc """ + Tells the list of decoded type names for all fields in the schema. + """ + @spec decode_type_names(FieldSchema.t()) :: [String.t()] + def decode_type_names(layout_schema) do + {_, types} = decode_types(layout_schema) + types |> Enum.map(&encode_type_name/1) + end + + # credo:disable-for-next-line Credo.Check.Refactor.CyclomaticComplexity + defp encode_type_name(type) do + case type do + _ when type < 32 -> "uint" <> Integer.to_string((type + 1) * 8) + _ when type < 64 -> "int" <> Integer.to_string((type - 31) * 8) + _ when type < 96 -> "bytes" <> Integer.to_string(type - 63) + 96 -> "bool" + 97 -> "address" + _ when type < 196 -> encode_type_name(type - 98) <> "[]" + 196 -> "bytes" + 197 -> "string" + _ -> "unknown_type_" <> Integer.to_string(type) + end + end +end diff --git a/apps/explorer/lib/explorer/chain/mud/table.ex b/apps/explorer/lib/explorer/chain/mud/table.ex new file mode 100644 index 0000000000..3adfc39a65 --- /dev/null +++ b/apps/explorer/lib/explorer/chain/mud/table.ex @@ -0,0 +1,88 @@ +defmodule Explorer.Chain.Mud.Table do + @moduledoc """ + Represents a decoded MUD framework database table ID. + """ + + alias Explorer.Chain.Hash + + @enforce_keys [:table_id, :table_full_name, :table_type, :table_namespace, :table_name] + @derive Jason.Encoder + defstruct [:table_id, :table_full_name, :table_type, :table_namespace, :table_name] + + @typedoc """ + Decoded MUD table name struct. + * `table_id` - The 32-bytes raw MUD table ID. + * `table_full_name` - The decoded table full name. + * `table_type` - The decoded table type: "offchain" or "onchain". + * `table_namespace` - The decoded table namespace. + * `table_name` - The decoded table name. + """ + @type t :: %__MODULE__{ + table_id: Hash.Full.t(), + table_full_name: String.t(), + table_type: String.t(), + table_namespace: String.t(), + table_name: String.t() + } + + @doc """ + Decodes table type, namespace and name information from raw MUD table ID. + """ + @spec from(Hash.Full.t()) :: t() + def from(%Hash{byte_count: 32, bytes: raw} = table_id) do + <> = raw + + trimmed_namespace = String.trim_trailing(namespace, <<0>>) + trimmed_table_name = String.trim_trailing(table_name, <<0>>) + + table_full_name = + if String.length(trimmed_namespace) > 0 do + prefix <> "." <> trimmed_namespace <> "." <> trimmed_table_name + else + prefix <> "." <> trimmed_table_name + end + + table_type = + case prefix do + "ot" -> "offchain" + "tb" -> "onchain" + _ -> "unknown" + end + + %__MODULE__{ + table_id: table_id, + table_full_name: table_full_name, + table_type: table_type, + table_namespace: trimmed_namespace, + table_name: trimmed_table_name + } + end + + @doc """ + Encodes table full name as a raw MUD table ID. + """ + @spec table_full_name_to_table_id(String.t()) :: {:ok, Hash.Full.t()} | :error + def table_full_name_to_table_id(full_name) do + parts = + case String.split(full_name, ".") do + [prefix, name] -> [prefix, "", name] + [prefix, namespace, name] -> [prefix, namespace, name] + _ -> :error + end + + with [prefix, namespace, name] <- parts, + {:ok, prefix} <- normalize_length(prefix, 2), + {:ok, namespace} <- normalize_length(namespace, 14), + {:ok, name} <- normalize_length(name, 16) do + Hash.Full.cast(prefix <> namespace <> name) + end + end + + defp normalize_length(str, len) do + if String.length(str) <= len do + {:ok, String.pad_trailing(str, len, <<0>>)} + else + :error + end + end +end diff --git a/apps/explorer/lib/explorer/repo.ex b/apps/explorer/lib/explorer/repo.ex index a1d4f35ad7..845faa2fc2 100644 --- a/apps/explorer/lib/explorer/repo.ex +++ b/apps/explorer/lib/explorer/repo.ex @@ -274,4 +274,14 @@ defmodule Explorer.Repo do ConfigHelper.init_repo_module(__MODULE__, opts) end end + + defmodule Mud do + use Ecto.Repo, + otp_app: :explorer, + adapter: Ecto.Adapters.Postgres + + def init(_, opts) do + ConfigHelper.init_repo_module(__MODULE__, opts) + end + end end diff --git a/apps/explorer/lib/explorer/repo/config_helper.ex b/apps/explorer/lib/explorer/repo/config_helper.ex index e1edad2bc2..d1544aa801 100644 --- a/apps/explorer/lib/explorer/repo/config_helper.ex +++ b/apps/explorer/lib/explorer/repo/config_helper.ex @@ -32,6 +32,8 @@ defmodule Explorer.Repo.ConfigHelper do def get_api_db_url, do: System.get_env("DATABASE_READ_ONLY_API_URL") || System.get_env("DATABASE_URL") + def get_mud_db_url, do: System.get_env("MUD_DATABASE_URL") || System.get_env("DATABASE_URL") + def init_repo_module(module, opts) do db_url = Application.get_env(:explorer, module)[:url] repo_conf = Application.get_env(:explorer, module) diff --git a/apps/explorer/test/test_helper.exs b/apps/explorer/test/test_helper.exs index 90d21435e5..c5eda8f3e4 100644 --- a/apps/explorer/test/test_helper.exs +++ b/apps/explorer/test/test_helper.exs @@ -22,6 +22,7 @@ Ecto.Adapters.SQL.Sandbox.mode(Explorer.Repo.Beacon, :auto) Ecto.Adapters.SQL.Sandbox.mode(Explorer.Repo.BridgedTokens, :auto) Ecto.Adapters.SQL.Sandbox.mode(Explorer.Repo.Filecoin, :auto) Ecto.Adapters.SQL.Sandbox.mode(Explorer.Repo.Stability, :auto) +Ecto.Adapters.SQL.Sandbox.mode(Explorer.Repo.Mud, :auto) Mox.defmock(Explorer.ExchangeRates.Source.TestSource, for: Explorer.ExchangeRates.Source) Mox.defmock(Explorer.Market.History.Source.Price.TestSource, for: Explorer.Market.History.Source.Price) diff --git a/config/config_helper.exs b/config/config_helper.exs index bde67152dd..46f37b5ee7 100644 --- a/config/config_helper.exs +++ b/config/config_helper.exs @@ -24,11 +24,15 @@ defmodule ConfigHelper do _ -> base_repos end - if System.get_env("BRIDGED_TOKENS_ENABLED") do - repos ++ [Explorer.Repo.BridgedTokens] - else - repos - end + ext_repos = + [ + {parse_bool_env_var("BRIDGED_TOKENS_ENABLED"), Explorer.Repo.BridgedTokens}, + {parse_bool_env_var("MUD_INDEXER_ENABLED"), Explorer.Repo.Mud} + ] + |> Enum.filter(&elem(&1, 0)) + |> Enum.map(&elem(&1, 1)) + + repos ++ ext_repos end @spec hackney_options() :: any() diff --git a/config/runtime/dev.exs b/config/runtime/dev.exs index a8f21fdbbb..cb1aa7d3cb 100644 --- a/config/runtime/dev.exs +++ b/config/runtime/dev.exs @@ -163,6 +163,17 @@ config :explorer, Explorer.Repo.Stability, url: System.get_env("DATABASE_URL"), pool_size: 1 +database_mud = if System.get_env("MUD_DATABASE_URL"), do: nil, else: database +hostname_mud = if System.get_env("MUD_DATABASE_URL"), do: nil, else: hostname + +# Configure MUD indexer database +config :explorer, Explorer.Repo.Mud, + database: database_mud, + hostname: hostname_mud, + url: ExplorerConfigHelper.get_mud_db_url(), + pool_size: ConfigHelper.parse_integer_env_var("MUD_POOL_SIZE", 10), + queue_target: queue_target + variant = Variant.get() Code.require_file("#{variant}.exs", "apps/explorer/config/dev") diff --git a/config/runtime/prod.exs b/config/runtime/prod.exs index cabdab7b94..899a772783 100644 --- a/config/runtime/prod.exs +++ b/config/runtime/prod.exs @@ -127,6 +127,13 @@ config :explorer, Explorer.Repo.Stability, pool_size: 1, ssl: ExplorerConfigHelper.ssl_enabled?() +# Configures Mud database +config :explorer, Explorer.Repo.Mud, + url: ExplorerConfigHelper.get_mud_db_url(), + pool_size: ConfigHelper.parse_integer_env_var("MUD_POOL_SIZE", 50), + ssl: ExplorerConfigHelper.ssl_enabled?(), + queue_target: queue_target + variant = Variant.get() Code.require_file("#{variant}.exs", "apps/explorer/config/prod") diff --git a/cspell.json b/cspell.json index 065e1e2d80..0fa574b82c 100644 --- a/cspell.json +++ b/cspell.json @@ -391,6 +391,7 @@ "nowrap", "ntoa", "nxdomain", + "offchain", "omni", "onclick", "onconnect", diff --git a/docker-compose/envs/common-blockscout.env b/docker-compose/envs/common-blockscout.env index dab55ec0e6..f421e2b6fe 100644 --- a/docker-compose/envs/common-blockscout.env +++ b/docker-compose/envs/common-blockscout.env @@ -370,4 +370,7 @@ TENDERLY_CHAIN_PATH= # BRIDGED_TOKENS_BSC_OMNI_BRIDGE_MEDIATOR= # BRIDGED_TOKENS_POA_OMNI_BRIDGE_MEDIATOR= # BRIDGED_TOKENS_AMB_BRIDGE_MEDIATORS -# BRIDGED_TOKENS_FOREIGN_JSON_RPC \ No newline at end of file +# BRIDGED_TOKENS_FOREIGN_JSON_RPC +# MUD_INDEXER_ENABLED= +# MUD_DATABASE_URL= +# MUD_POOL_SIZE=50 \ No newline at end of file diff --git a/docker/Dockerfile b/docker/Dockerfile index 779985c1ab..ee2b67d289 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -25,6 +25,8 @@ ARG CHAIN_TYPE ENV CHAIN_TYPE=${CHAIN_TYPE} ARG BRIDGED_TOKENS_ENABLED ENV BRIDGED_TOKENS_ENABLED=${BRIDGED_TOKENS_ENABLED} +ARG MUD_INDEXER_ENABLED +ENV MUD_INDEXER_ENABLED=${MUD_INDEXER_ENABLED} # Cache elixir deps ADD mix.exs mix.lock ./