feat: MUD API support (#9869)
* feat: mud support * chore: fix ci warnings * feat: skip missing schemas * ci: build redstone image * chore: fix dialyzer * chore: remove noop migration * feat: full-text table search * fix: don't show deleted records * fix: type specs and dializer fixes * feat: checksum addresses * fix: handle invalid params * chore: add missing envsvb-graphql-req-size-limit
Kirill Fedoseev
6 months ago
committed by
GitHub
parent
18ef2c0ef5
commit
fb4fde678d
@ -0,0 +1,44 @@ |
||||
name: Redstone Publish Docker image |
||||
|
||||
on: |
||||
workflow_dispatch: |
||||
push: |
||||
branches: |
||||
- production-redstone |
||||
jobs: |
||||
push_to_registry: |
||||
name: Push Docker image to Docker Hub |
||||
runs-on: ubuntu-latest |
||||
env: |
||||
RELEASE_VERSION: ${{ vars.RELEASE_VERSION }} |
||||
DOCKER_CHAIN_NAME: redstone |
||||
steps: |
||||
- uses: actions/checkout@v4 |
||||
- name: Setup repo |
||||
uses: ./.github/actions/setup-repo-and-short-sha |
||||
with: |
||||
docker-username: ${{ secrets.DOCKER_USERNAME }} |
||||
docker-password: ${{ secrets.DOCKER_PASSWORD }} |
||||
|
||||
- name: Build and push Docker image |
||||
uses: docker/build-push-action@v5 |
||||
with: |
||||
context: . |
||||
file: ./docker/Dockerfile |
||||
push: true |
||||
tags: blockscout/blockscout-${{ env.DOCKER_CHAIN_NAME }}:latest, blockscout/blockscout-${{ env.DOCKER_CHAIN_NAME }}:${{ env.RELEASE_VERSION }}-postrelease-${{ env.SHORT_SHA }} |
||||
platforms: | |
||||
linux/amd64 |
||||
linux/arm64/v8 |
||||
build-args: | |
||||
CACHE_EXCHANGE_RATES_PERIOD= |
||||
API_V1_READ_METHODS_DISABLED=false |
||||
DISABLE_WEBAPP=false |
||||
API_V1_WRITE_METHODS_DISABLED=false |
||||
CACHE_TOTAL_GAS_USAGE_COUNTER_ENABLED= |
||||
ADMIN_PANEL_ENABLED=false |
||||
CACHE_ADDRESS_WITH_BALANCES_UPDATE_INTERVAL= |
||||
BLOCKSCOUT_VERSION=v${{ env.RELEASE_VERSION }}-beta.+commit.${{ env.SHORT_SHA }} |
||||
RELEASE_VERSION=${{ env.RELEASE_VERSION }} |
||||
CHAIN_TYPE=optimism |
||||
MUD_INDEXER_ENABLED=true |
||||
@ -0,0 +1,46 @@ |
||||
name: Release for Redstone |
||||
|
||||
on: |
||||
release: |
||||
types: [published] |
||||
|
||||
env: |
||||
OTP_VERSION: ${{ vars.OTP_VERSION }} |
||||
ELIXIR_VERSION: ${{ vars.ELIXIR_VERSION }} |
||||
|
||||
jobs: |
||||
push_to_registry: |
||||
name: Push Docker image to Docker Hub |
||||
runs-on: ubuntu-latest |
||||
env: |
||||
RELEASE_VERSION: ${{ vars.RELEASE_VERSION }} |
||||
steps: |
||||
- uses: actions/checkout@v4 |
||||
- name: Setup repo |
||||
uses: ./.github/actions/setup-repo |
||||
with: |
||||
docker-username: ${{ secrets.DOCKER_USERNAME }} |
||||
docker-password: ${{ secrets.DOCKER_PASSWORD }} |
||||
|
||||
- name: Build and push Docker image for Redstone |
||||
uses: docker/build-push-action@v5 |
||||
with: |
||||
context: . |
||||
file: ./docker/Dockerfile |
||||
push: true |
||||
tags: blockscout/blockscout-redstone:latest, blockscout/blockscout-redstone:${{ env.RELEASE_VERSION }} |
||||
platforms: | |
||||
linux/amd64 |
||||
linux/arm64/v8 |
||||
build-args: | |
||||
CACHE_EXCHANGE_RATES_PERIOD= |
||||
API_V1_READ_METHODS_DISABLED=false |
||||
DISABLE_WEBAPP=false |
||||
API_V1_WRITE_METHODS_DISABLED=false |
||||
CACHE_TOTAL_GAS_USAGE_COUNTER_ENABLED= |
||||
ADMIN_PANEL_ENABLED=false |
||||
CACHE_ADDRESS_WITH_BALANCES_UPDATE_INTERVAL= |
||||
BLOCKSCOUT_VERSION=v${{ env.RELEASE_VERSION }}-beta |
||||
RELEASE_VERSION=${{ env.RELEASE_VERSION }} |
||||
CHAIN_TYPE=optimism |
||||
MUD_INDEXER_ENABLED=true |
||||
@ -0,0 +1,261 @@ |
||||
defmodule BlockScoutWeb.API.V2.MudController do |
||||
use BlockScoutWeb, :controller |
||||
|
||||
import BlockScoutWeb.Chain, |
||||
only: [ |
||||
next_page_params: 4, |
||||
split_list_by_page: 1, |
||||
default_paging_options: 0 |
||||
] |
||||
|
||||
import BlockScoutWeb.PagingHelper, only: [mud_records_sorting: 1] |
||||
|
||||
alias Explorer.Chain.{Data, Hash, Mud, Mud.Schema.FieldSchema, Mud.Table} |
||||
|
||||
action_fallback(BlockScoutWeb.API.V2.FallbackController) |
||||
|
||||
@doc """ |
||||
Function to handle GET requests to `/api/v2/mud/worlds` endpoint. |
||||
""" |
||||
@spec worlds(Plug.Conn.t(), map()) :: Plug.Conn.t() |
||||
def worlds(conn, params) do |
||||
{worlds, next_page} = |
||||
params |
||||
|> mud_paging_options(["world"], [Hash.Address]) |
||||
|> Mud.worlds_list() |
||||
|> split_list_by_page() |
||||
|
||||
next_page_params = |
||||
next_page_params(next_page, worlds, conn.query_params, fn item -> |
||||
%{"world" => item} |
||||
end) |
||||
|
||||
conn |
||||
|> put_status(200) |
||||
|> render(:worlds, %{worlds: worlds, next_page_params: next_page_params}) |
||||
end |
||||
|
||||
@doc """ |
||||
Function to handle GET requests to `/api/v2/mud/worlds/count` endpoint. |
||||
""" |
||||
@spec worlds_count(Plug.Conn.t(), map()) :: Plug.Conn.t() |
||||
def worlds_count(conn, _params) do |
||||
count = Mud.worlds_count() |
||||
|
||||
conn |
||||
|> put_status(200) |
||||
|> render(:count, %{count: count}) |
||||
end |
||||
|
||||
@doc """ |
||||
Function to handle GET requests to `/api/v2/mud/worlds/:world/tables` endpoint. |
||||
""" |
||||
@spec world_tables(Plug.Conn.t(), map()) :: Plug.Conn.t() |
||||
def world_tables(conn, %{"world" => world_param} = params) do |
||||
with {:format, {:ok, world}} <- {:format, Hash.Address.cast(world_param)} do |
||||
options = params |> mud_paging_options(["table_id"], [Hash.Full]) |> Keyword.merge(mud_tables_filter(params)) |
||||
|
||||
{tables, next_page} = |
||||
world |
||||
|> Mud.world_tables(options) |
||||
|> split_list_by_page() |
||||
|
||||
next_page_params = |
||||
next_page_params(next_page, tables, conn.query_params, fn item -> |
||||
%{"table_id" => item |> elem(0)} |
||||
end) |
||||
|
||||
conn |
||||
|> put_status(200) |
||||
|> render(:tables, %{tables: tables, next_page_params: next_page_params}) |
||||
end |
||||
end |
||||
|
||||
@doc """ |
||||
Function to handle GET requests to `/api/v2/mud/worlds/:world/tables/count` endpoint. |
||||
""" |
||||
@spec world_tables_count(Plug.Conn.t(), map()) :: Plug.Conn.t() |
||||
def world_tables_count(conn, %{"world" => world_param} = params) do |
||||
with {:format, {:ok, world}} <- {:format, Hash.Address.cast(world_param)} do |
||||
options = params |> mud_tables_filter() |
||||
|
||||
count = Mud.world_tables_count(world, options) |
||||
|
||||
conn |
||||
|> put_status(200) |
||||
|> render(:count, %{count: count}) |
||||
end |
||||
end |
||||
|
||||
@doc """ |
||||
Function to handle GET requests to `/api/v2/mud/worlds/:world/tables/:table_id/records` endpoint. |
||||
""" |
||||
@spec world_table_records(Plug.Conn.t(), map()) :: Plug.Conn.t() |
||||
def world_table_records(conn, %{"world" => world_param, "table_id" => table_id_param} = params) do |
||||
with {:format, {:ok, world}} <- {:format, Hash.Address.cast(world_param)}, |
||||
{:format, {:ok, table_id}} <- {:format, Hash.Full.cast(table_id_param)}, |
||||
{:ok, schema} <- Mud.world_table_schema(world, table_id) do |
||||
options = |
||||
params |
||||
|> mud_paging_options(["key_bytes", "key0", "key1"], [Data, Hash.Full, Hash.Full]) |
||||
|> Keyword.merge(mud_records_filter(params, schema)) |
||||
|> Keyword.merge(mud_records_sorting(params)) |
||||
|
||||
{records, next_page} = world |> Mud.world_table_records(table_id, options) |> split_list_by_page() |
||||
|
||||
blocks = Mud.preload_records_timestamps(records) |
||||
|
||||
next_page_params = |
||||
next_page_params(next_page, records, conn.query_params, fn item -> |
||||
keys = [item.key_bytes, item.key0, item.key1] |> Enum.filter(&(!is_nil(&1))) |
||||
["key_bytes", "key0", "key1"] |> Enum.zip(keys) |> Enum.into(%{}) |
||||
end) |
||||
|
||||
conn |
||||
|> put_status(200) |
||||
|> render(:records, %{ |
||||
records: records, |
||||
table_id: table_id, |
||||
schema: schema, |
||||
blocks: blocks, |
||||
next_page_params: next_page_params |
||||
}) |
||||
end |
||||
end |
||||
|
||||
@doc """ |
||||
Function to handle GET requests to `/api/v2/mud/worlds/:world/tables/:table_id/records/count` endpoint. |
||||
""" |
||||
@spec world_table_records_count(Plug.Conn.t(), map()) :: Plug.Conn.t() |
||||
def world_table_records_count(conn, %{"world" => world_param, "table_id" => table_id_param} = params) do |
||||
with {:format, {:ok, world}} <- {:format, Hash.Address.cast(world_param)}, |
||||
{:format, {:ok, table_id}} <- {:format, Hash.Full.cast(table_id_param)}, |
||||
{:ok, schema} <- Mud.world_table_schema(world, table_id) do |
||||
options = params |> mud_records_filter(schema) |
||||
|
||||
count = Mud.world_table_records_count(world, table_id, options) |
||||
|
||||
conn |
||||
|> put_status(200) |
||||
|> render(:count, %{count: count}) |
||||
end |
||||
end |
||||
|
||||
@doc """ |
||||
Function to handle GET requests to `/api/v2/mud/worlds/:world/tables/:table_id/records/:record_id` endpoint. |
||||
""" |
||||
@spec world_table_record(Plug.Conn.t(), map()) :: Plug.Conn.t() |
||||
def world_table_record( |
||||
conn, |
||||
%{"world" => world_param, "table_id" => table_id_param, "record_id" => record_id_param} = _params |
||||
) do |
||||
with {:format, {:ok, world}} <- {:format, Hash.Address.cast(world_param)}, |
||||
{:format, {:ok, table_id}} <- {:format, Hash.Full.cast(table_id_param)}, |
||||
{:format, {:ok, record_id}} <- {:format, Data.cast(record_id_param)}, |
||||
{:ok, schema} <- Mud.world_table_schema(world, table_id), |
||||
{:ok, record} <- Mud.world_table_record(world, table_id, record_id) do |
||||
blocks = Mud.preload_records_timestamps([record]) |
||||
|
||||
conn |
||||
|> put_status(200) |
||||
|> render(:record, %{record: record, table_id: table_id, schema: schema, blocks: blocks}) |
||||
end |
||||
end |
||||
|
||||
defp mud_tables_filter(params) do |
||||
Enum.reduce(params, [], fn {key, value}, acc -> |
||||
case key do |
||||
"filter_namespace" -> |
||||
Keyword.put(acc, :filter_namespace, parse_namespace_string(value)) |
||||
|
||||
"q" -> |
||||
Keyword.put(acc, :filter_search, parse_search_string(value)) |
||||
|
||||
_ -> |
||||
acc |
||||
end |
||||
end) |
||||
end |
||||
|
||||
defp parse_namespace_string(namespace) do |
||||
filter = |
||||
case namespace do |
||||
nil -> {:ok, nil} |
||||
"0x" <> hex -> Base.decode16(hex, case: :mixed) |
||||
str -> {:ok, str} |
||||
end |
||||
|
||||
case filter do |
||||
{:ok, ns} when is_binary(ns) and byte_size(ns) <= 14 -> |
||||
ns |> String.pad_trailing(14, <<0>>) |
||||
|
||||
_ -> |
||||
nil |
||||
end |
||||
end |
||||
|
||||
defp parse_search_string(q) do |
||||
# If the search string looks like hex-encoded table id or table full name, |
||||
# we try to parse and filter by that table id directly. |
||||
# Otherwise we do a full-text search of given string inside table id. |
||||
with :error <- Hash.Full.cast(q), |
||||
:error <- Table.table_full_name_to_table_id(q) do |
||||
q |
||||
else |
||||
{:ok, table_id} -> table_id |
||||
end |
||||
end |
||||
|
||||
defp mud_records_filter(params, schema) do |
||||
Enum.reduce(params, [], fn {key, value}, acc -> |
||||
case key do |
||||
"filter_key0" -> Keyword.put(acc, :filter_key0, encode_filter(value, schema, 0)) |
||||
"filter_key1" -> Keyword.put(acc, :filter_key1, encode_filter(value, schema, 1)) |
||||
_ -> acc |
||||
end |
||||
end) |
||||
end |
||||
|
||||
defp encode_filter(value, schema, field_idx) do |
||||
case value do |
||||
"false" -> |
||||
<<0::256>> |
||||
|
||||
"true" -> |
||||
<<1::256>> |
||||
|
||||
"0x" <> hex -> |
||||
bin = Base.decode16!(hex, case: :mixed) |
||||
# addresses are padded to 32 bytes with zeros on the right |
||||
if FieldSchema.type_of(schema.key_schema, field_idx) == 97 do |
||||
<<0::size(256 - byte_size(bin) * 8), bin::binary>> |
||||
else |
||||
<<bin::binary, 0::size(256 - byte_size(bin) * 8)>> |
||||
end |
||||
|
||||
dec -> |
||||
num = dec |> Integer.parse() |> elem(0) |
||||
<<num::256>> |
||||
end |
||||
end |
||||
|
||||
defp mud_paging_options(params, keys, types) do |
||||
page_key = |
||||
keys |
||||
|> Enum.zip(types) |
||||
|> Enum.reduce(%{}, fn {key, type}, acc -> |
||||
with param when param != nil <- Map.get(params, key), |
||||
{:ok, val} <- type.cast(param) do |
||||
acc |> Map.put(String.to_existing_atom(key), val) |
||||
else |
||||
_ -> acc |
||||
end |
||||
end) |
||||
|
||||
if page_key == %{} do |
||||
[paging_options: default_paging_options()] |
||||
else |
||||
[paging_options: %{default_paging_options() | key: page_key}] |
||||
end |
||||
end |
||||
end |
||||
@ -0,0 +1,85 @@ |
||||
defmodule BlockScoutWeb.API.V2.MudView do |
||||
use BlockScoutWeb, :view |
||||
|
||||
alias Explorer.Chain.{Mud, Mud.Table} |
||||
|
||||
@doc """ |
||||
Function to render GET requests to `/api/v2/mud/worlds` endpoint. |
||||
""" |
||||
@spec render(String.t(), map()) :: map() |
||||
def render("worlds.json", %{worlds: worlds, next_page_params: next_page_params}) do |
||||
%{ |
||||
items: worlds, |
||||
next_page_params: next_page_params |
||||
} |
||||
end |
||||
|
||||
@doc """ |
||||
Function to render GET requests to `/api/v2/mud/worlds/count` endpoint. |
||||
""" |
||||
def render("count.json", %{count: count}) do |
||||
%{ |
||||
count: count |
||||
} |
||||
end |
||||
|
||||
@doc """ |
||||
Function to render GET requests to `/api/v2/mud/worlds/:world/tables` endpoint. |
||||
""" |
||||
def render("tables.json", %{tables: tables, next_page_params: next_page_params}) do |
||||
%{ |
||||
items: tables |> Enum.map(&%{table: Table.from(&1 |> elem(0)), schema: &1 |> elem(1)}), |
||||
next_page_params: next_page_params |
||||
} |
||||
end |
||||
|
||||
@doc """ |
||||
Function to render GET requests to `/api/v2/mud/worlds/:world/tables/:table_id/records` endpoint. |
||||
""" |
||||
def render("records.json", %{ |
||||
records: records, |
||||
table_id: table_id, |
||||
schema: schema, |
||||
blocks: blocks, |
||||
next_page_params: next_page_params |
||||
}) do |
||||
%{ |
||||
items: records |> Enum.map(&format_record(&1, schema, blocks)), |
||||
table: table_id |> Table.from(), |
||||
schema: schema, |
||||
next_page_params: next_page_params |
||||
} |
||||
end |
||||
|
||||
@doc """ |
||||
Function to render GET requests to `/api/v2/mud/worlds/:world/tables/:table_id/records/:record_id` endpoint. |
||||
""" |
||||
def render("record.json", %{record: record, table_id: table_id, blocks: blocks, schema: schema}) do |
||||
%{ |
||||
record: record |> format_record(schema, blocks), |
||||
table: table_id |> Table.from(), |
||||
schema: schema |
||||
} |
||||
end |
||||
|
||||
defp format_record(nil, _schema, _blocks), do: nil |
||||
|
||||
defp format_record(record, schema, blocks) do |
||||
%{ |
||||
id: record.key_bytes, |
||||
raw: %{ |
||||
key_bytes: record.key_bytes, |
||||
key0: record.key0, |
||||
key1: record.key1, |
||||
static_data: record.static_data, |
||||
encoded_lengths: record.encoded_lengths, |
||||
dynamic_data: record.dynamic_data, |
||||
block_number: record.block_number, |
||||
log_index: record.log_index |
||||
}, |
||||
is_deleted: record.is_deleted, |
||||
decoded: Mud.decode_record(record, schema), |
||||
timestamp: blocks |> Map.get(Decimal.to_integer(record.block_number), nil) |
||||
} |
||||
end |
||||
end |
||||
@ -0,0 +1,395 @@ |
||||
defmodule Explorer.Chain.Mud do |
||||
@moduledoc """ |
||||
Represents a MUD framework database record. |
||||
""" |
||||
use Explorer.Schema |
||||
|
||||
import Ecto.Query, |
||||
only: [ |
||||
distinct: 2, |
||||
order_by: 3, |
||||
select: 3, |
||||
where: 3, |
||||
limit: 2 |
||||
] |
||||
|
||||
alias ABI.TypeDecoder |
||||
alias Explorer.{Chain, PagingOptions, Repo, SortingHelper} |
||||
|
||||
alias Explorer.Chain.{ |
||||
Address, |
||||
Block, |
||||
Data, |
||||
Hash, |
||||
Mud, |
||||
Mud.Schema, |
||||
Mud.Schema.FieldSchema |
||||
} |
||||
|
||||
require Logger |
||||
|
||||
@schema_prefix "mud" |
||||
|
||||
@store_tables_table_id Base.decode16!("746273746f72650000000000000000005461626c657300000000000000000000", case: :lower) |
||||
|
||||
# https://github.com/latticexyz/mud/blob/cc4f4246e52982354e398113c46442910f9b04bb/packages/store/src/codegen/tables/Tables.sol#L34-L42 |
||||
@store_tables_table_schema %Schema{ |
||||
key_schema: FieldSchema.from("0x002001005f000000000000000000000000000000000000000000000000000000"), |
||||
value_schema: FieldSchema.from("0x006003025f5f5fc4c40000000000000000000000000000000000000000000000"), |
||||
key_names: ["tableId"], |
||||
value_names: ["fieldLayout", "keySchema", "valueSchema", "abiEncodedKeyNames", "abiEncodedValueNames"] |
||||
} |
||||
|
||||
@primary_key false |
||||
typed_schema "records" do |
||||
field(:address, Hash.Address, null: false) |
||||
field(:table_id, Hash.Full, null: false) |
||||
field(:key_bytes, Data) |
||||
field(:key0, Hash.Full) |
||||
field(:key1, Hash.Full) |
||||
field(:static_data, Data) |
||||
field(:encoded_lengths, Data) |
||||
field(:dynamic_data, Data) |
||||
field(:is_deleted, :boolean, null: false) |
||||
field(:block_number, :decimal, null: false) |
||||
field(:log_index, :decimal, null: false) |
||||
end |
||||
|
||||
def enabled? do |
||||
Application.get_env(:explorer, __MODULE__)[:enabled] |
||||
end |
||||
|
||||
@doc """ |
||||
Returns the paginated list of registered MUD world addresses. |
||||
""" |
||||
@spec worlds_list(Keyword.t()) :: [Mud.t()] |
||||
def worlds_list(options \\ []) do |
||||
paging_options = Keyword.get(options, :paging_options, Chain.default_paging_options()) |
||||
|
||||
Mud |
||||
|> select([r], r.address) |
||||
|> distinct(true) |
||||
|> page_worlds(paging_options) |
||||
|> limit(^paging_options.page_size) |
||||
|> Repo.Mud.all() |
||||
end |
||||
|
||||
defp page_worlds(query, %PagingOptions{key: %{world: world}}) do |
||||
query |> where([item], item.address > ^world) |
||||
end |
||||
|
||||
defp page_worlds(query, _), do: query |
||||
|
||||
@doc """ |
||||
Returns the total number of registered MUD worlds. |
||||
""" |
||||
@spec worlds_count() :: non_neg_integer() |
||||
def worlds_count do |
||||
Mud |
||||
|> select([r], r.address) |
||||
|> distinct(true) |
||||
|> Repo.Mud.aggregate(:count) |
||||
end |
||||
|
||||
@doc """ |
||||
Returns the decoded MUD table schema by world address and table ID. |
||||
""" |
||||
@spec world_table_schema(Hash.Address.t(), Hash.Full.t()) :: {:ok, Schema.t()} | {:error, :not_found} |
||||
def world_table_schema(world, table_id) do |
||||
Mud |
||||
|> where([r], r.address == ^world and r.table_id == ^@store_tables_table_id and r.key0 == ^table_id) |
||||
|> Repo.Mud.one() |
||||
|> case do |
||||
nil -> |
||||
{:error, :not_found} |
||||
|
||||
r -> |
||||
{:ok, decode_schema(r)} |
||||
end |
||||
end |
||||
|
||||
@doc """ |
||||
Returns the paginated list of registered MUD tables in the given world, optionally filtered by namespace or table name. |
||||
Each returned table in the resulting list is represented as a tuple of its ID and decoded schema. |
||||
""" |
||||
@spec world_tables(Hash.Address.t(), Keyword.t()) :: [{Hash.Full.t(), Schema.t()}] |
||||
def world_tables(world, options \\ []) do |
||||
paging_options = Keyword.get(options, :paging_options, Chain.default_paging_options()) |
||||
filter_namespace = Keyword.get(options, :filter_namespace, nil) |
||||
filter_search = Keyword.get(options, :filter_search, nil) |
||||
|
||||
Mud |
||||
|> where([r], r.address == ^world and r.table_id == ^@store_tables_table_id) |
||||
|> filter_tables_by_namespace(filter_namespace) |
||||
|> filter_tables_by_search(filter_search) |
||||
|> page_tables(paging_options) |
||||
|> order_by([r], asc: r.key0) |
||||
|> limit(^paging_options.page_size) |
||||
|> Repo.Mud.all() |
||||
|> Enum.map(&{&1.key0, decode_schema(&1)}) |
||||
end |
||||
|
||||
defp page_tables(query, %PagingOptions{key: %{table_id: table_id}}) do |
||||
query |> where([item], item.key0 > ^table_id) |
||||
end |
||||
|
||||
defp page_tables(query, _), do: query |
||||
|
||||
@doc """ |
||||
Returns the number of registered MUD tables in the given world. |
||||
""" |
||||
@spec world_tables_count(Hash.Address.t(), Keyword.t()) :: non_neg_integer() |
||||
def world_tables_count(world, options \\ []) do |
||||
filter_namespace = Keyword.get(options, :filter_namespace, nil) |
||||
filter_search = Keyword.get(options, :filter_search, nil) |
||||
|
||||
Mud |
||||
|> where([r], r.address == ^world and r.table_id == ^@store_tables_table_id) |
||||
|> filter_tables_by_namespace(filter_namespace) |
||||
|> filter_tables_by_search(filter_search) |
||||
|> Repo.Mud.aggregate(:count) |
||||
end |
||||
|
||||
defp filter_tables_by_namespace(query, nil), do: query |
||||
|
||||
defp filter_tables_by_namespace(query, namespace) do |
||||
query |> where([tb], fragment("substring(? FROM 3 FOR 14)", tb.key0) == ^namespace) |
||||
end |
||||
|
||||
defp filter_tables_by_search(query, %Hash{} = table_id) do |
||||
query |> where([tb], tb.key0 == ^table_id) |
||||
end |
||||
|
||||
defp filter_tables_by_search(query, search_string) when is_binary(search_string) do |
||||
query |> where([tb], ilike(fragment("encode(?, 'escape')", tb.key0), ^"%#{search_string}%")) |
||||
end |
||||
|
||||
defp filter_tables_by_search(query, _), do: query |
||||
|
||||
@default_sorting [ |
||||
asc: :key_bytes |
||||
] |
||||
|
||||
@doc """ |
||||
Returns the paginated list of raw MUD records in the given world table. |
||||
Resulting records can be sorted or filtered by any of the first 2 key columns. |
||||
""" |
||||
@spec world_table_records(Hash.Address.t(), Hash.Full.t(), Keyword.t()) :: [Mud.t()] |
||||
def world_table_records(world, table_id, options \\ []) do |
||||
paging_options = Keyword.get(options, :paging_options, Chain.default_paging_options()) |
||||
sorting = Keyword.get(options, :sorting, []) |
||||
|
||||
Mud |
||||
|> where([r], r.address == ^world and r.table_id == ^table_id and r.is_deleted == false) |
||||
|> filter_records(:key0, Keyword.get(options, :filter_key0)) |
||||
|> filter_records(:key1, Keyword.get(options, :filter_key1)) |
||||
|> SortingHelper.apply_sorting(sorting, @default_sorting) |
||||
|> SortingHelper.page_with_sorting(paging_options, sorting, @default_sorting) |
||||
|> Repo.Mud.all() |
||||
end |
||||
|
||||
@doc """ |
||||
Preloads last modification timestamps for the list of raw MUD records. |
||||
|
||||
Returns a map of block numbers to timestamps. |
||||
""" |
||||
@spec preload_records_timestamps([Mud.t()]) :: %{non_neg_integer() => DateTime.t()} |
||||
def preload_records_timestamps(records) do |
||||
block_numbers = records |> Enum.map(&(&1.block_number |> Decimal.to_integer())) |> Enum.uniq() |
||||
|
||||
Block |
||||
|> where([b], b.number in ^block_numbers) |
||||
|> select([b], {b.number, b.timestamp}) |
||||
|> Repo.all() |
||||
|> Enum.into(%{}) |
||||
end |
||||
|
||||
@doc """ |
||||
Returns the number of MUD records in the given world table. |
||||
""" |
||||
@spec world_table_records_count(Hash.Address.t(), Hash.Full.t(), Keyword.t()) :: non_neg_integer() |
||||
def world_table_records_count(world, table_id, options \\ []) do |
||||
Mud |
||||
|> where([r], r.address == ^world and r.table_id == ^table_id and r.is_deleted == false) |
||||
|> filter_records(:key0, Keyword.get(options, :filter_key0)) |
||||
|> filter_records(:key1, Keyword.get(options, :filter_key1)) |
||||
|> Repo.Mud.aggregate(:count) |
||||
end |
||||
|
||||
defp filter_records(query, _key_name, nil), do: query |
||||
|
||||
defp filter_records(query, :key0, key), do: query |> where([r], r.key0 == ^key) |
||||
|
||||
defp filter_records(query, :key1, key), do: query |> where([r], r.key1 == ^key) |
||||
|
||||
@doc """ |
||||
Returns the raw MUD record from the given world table by its ID. |
||||
""" |
||||
@spec world_table_record(Hash.Address.t(), Hash.Full.t(), Data.t()) :: {:ok, Mud.t()} | {:error, :not_found} |
||||
def world_table_record(world, table_id, record_id) do |
||||
Mud |
||||
|> where([r], r.address == ^world and r.table_id == ^table_id and r.key_bytes == ^record_id) |
||||
|> Repo.Mud.one() |
||||
|> case do |
||||
nil -> |
||||
{:error, :not_found} |
||||
|
||||
r -> |
||||
{:ok, r} |
||||
end |
||||
end |
||||
|
||||
defp decode_schema(nil), do: nil |
||||
|
||||
defp decode_schema(record) do |
||||
schema_record = decode_record(record, @store_tables_table_schema) |
||||
|
||||
%Schema{ |
||||
key_schema: schema_record["keySchema"] |> FieldSchema.from(), |
||||
value_schema: schema_record["valueSchema"] |> FieldSchema.from(), |
||||
key_names: schema_record["abiEncodedKeyNames"] |> decode_abi_encoded_strings(), |
||||
value_names: schema_record["abiEncodedValueNames"] |> decode_abi_encoded_strings() |
||||
} |
||||
end |
||||
|
||||
defp decode_abi_encoded_strings("0x" <> hex_encoded) do |
||||
hex_encoded |
||||
|> Base.decode16!(case: :mixed) |
||||
|> TypeDecoder.decode_raw([{:array, :string}]) |
||||
|> Enum.at(0) |
||||
end |
||||
|
||||
@doc """ |
||||
Decodes a given raw MUD record according to table schema. |
||||
|
||||
Returns a JSON-like map with decoded field names and values. |
||||
""" |
||||
@spec decode_record(Mud.t() | nil, Schema.t() | nil) :: map() | nil |
||||
def decode_record(nil, _schema), do: nil |
||||
|
||||
def decode_record(_record, nil), do: nil |
||||
|
||||
def decode_record(record, schema) do |
||||
key = decode_key_tuple(record.key_bytes.bytes, schema.key_names, schema.key_schema) |
||||
|
||||
value = |
||||
if record.is_deleted do |
||||
schema.value_names |> Enum.into(%{}, &{&1, nil}) |
||||
else |
||||
decode_fields( |
||||
record.static_data, |
||||
record.encoded_lengths, |
||||
record.dynamic_data, |
||||
schema.value_names, |
||||
schema.value_schema |
||||
) |
||||
end |
||||
|
||||
key |> Map.merge(value) |
||||
end |
||||
|
||||
defp decode_key_tuple(key_bytes, fields, layout_schema) do |
||||
{_, types} = Schema.decode_types(layout_schema) |
||||
|
||||
fields |
||||
|> Enum.zip(types) |
||||
|> Enum.reduce({%{}, key_bytes}, fn {field, type}, {acc, data} -> |
||||
type_size = static_type_size(type) |
||||
<<word::binary-size(32), rest::binary>> = data |
||||
|
||||
enc = |
||||
if type < 64 or type >= 96 do |
||||
:binary.part(word, 32 - type_size, type_size) |
||||
else |
||||
:binary.part(word, 0, type_size) |
||||
end |
||||
|
||||
decoded = decode_type(type, enc) |
||||
|
||||
{Map.put(acc, field, decoded), rest} |
||||
end) |
||||
|> elem(0) |
||||
end |
||||
|
||||
defp decode_fields(static_data, encoded_lengths, dynamic_data, fields, layout_schema) do |
||||
{static_fields_count, types} = Schema.decode_types(layout_schema) |
||||
|
||||
{static_types, dynamic_types} = Enum.split(types, static_fields_count) |
||||
|
||||
{static_fields, dynamic_fields} = Enum.split(fields, static_fields_count) |
||||
|
||||
res = |
||||
static_fields |
||||
|> Enum.zip(static_types) |
||||
|> Enum.reduce({%{}, (static_data && static_data.bytes) || <<>>}, fn {field, type}, {acc, data} -> |
||||
type_size = static_type_size(type) |
||||
<<enc::binary-size(type_size), rest::binary>> = data |
||||
decoded = decode_type(type, enc) |
||||
{Map.put(acc, field, decoded), rest} |
||||
end) |
||||
|> elem(0) |
||||
|
||||
if encoded_lengths == nil or byte_size(encoded_lengths.bytes) == 0 do |
||||
res |
||||
else |
||||
dynamic_type_lengths = |
||||
encoded_lengths.bytes |
||||
|> :binary.bin_to_list(0, 25) |
||||
|> Enum.chunk_every(5) |
||||
|> Enum.reverse() |
||||
|> Enum.map(&(&1 |> :binary.list_to_bin() |> :binary.decode_unsigned())) |
||||
|
||||
[dynamic_fields, dynamic_types, dynamic_type_lengths] |
||||
|> Enum.zip() |
||||
|> Enum.reduce({res, (dynamic_data && dynamic_data.bytes) || <<>>}, fn {field, type, length}, {acc, data} -> |
||||
<<enc::binary-size(length), rest::binary>> = data |
||||
decoded = decode_type(type, enc) |
||||
|
||||
{Map.put(acc, field, decoded), rest} |
||||
end) |
||||
|> elem(0) |
||||
end |
||||
end |
||||
|
||||
defp static_type_size(type) do |
||||
case type do |
||||
_ when type < 97 -> rem(type, 32) + 1 |
||||
97 -> 20 |
||||
_ -> 0 |
||||
end |
||||
end |
||||
|
||||
# credo:disable-for-next-line Credo.Check.Refactor.CyclomaticComplexity |
||||
defp decode_type(type, raw) do |
||||
case type do |
||||
_ when type < 32 -> |
||||
raw |> :binary.decode_unsigned() |> Integer.to_string() |
||||
|
||||
_ when type < 64 -> |
||||
size = static_type_size(type) |
||||
<<int::signed-integer-size(size * 8)>> = raw |
||||
int |> Integer.to_string() |
||||
|
||||
_ when type < 96 or type == 196 -> |
||||
"0x" <> Base.encode16(raw, case: :lower) |
||||
|
||||
96 -> |
||||
raw == <<1>> |
||||
|
||||
97 -> |
||||
Address.checksum(raw) |
||||
|
||||
_ when type < 196 -> |
||||
raw |
||||
|> :binary.bin_to_list() |
||||
|> Enum.chunk_every(static_type_size(type - 98)) |
||||
|> Enum.map(&decode_type(type - 98, :binary.list_to_bin(&1))) |
||||
|
||||
197 -> |
||||
raw |
||||
|
||||
_ -> |
||||
raise "Unknown type: #{type}" |
||||
end |
||||
end |
||||
end |
||||
@ -0,0 +1,110 @@ |
||||
defmodule Explorer.Chain.Mud.Schema do |
||||
@moduledoc """ |
||||
Represents a MUD framework database record schema. |
||||
""" |
||||
|
||||
defmodule FieldSchema do |
||||
@moduledoc """ |
||||
Represents a MUD framework database record field schema. Describes number of columns and their types. |
||||
""" |
||||
|
||||
defstruct [:word] |
||||
|
||||
@typedoc """ |
||||
The MUD field schema. |
||||
* `word` - The field schema as 32-byte value. |
||||
""" |
||||
@type t :: %__MODULE__{ |
||||
word: <<_::256>> |
||||
} |
||||
|
||||
@doc """ |
||||
Decodes field schema type from raw binary or hex-encoded string. |
||||
""" |
||||
@spec from(binary()) :: t() | :error |
||||
def from(<<bin::binary-size(32)>>), do: %__MODULE__{word: bin} |
||||
|
||||
def from("0x" <> <<hex::binary-size(64)>>) do |
||||
with {:ok, bin} <- Base.decode16(hex, case: :mixed) do |
||||
%__MODULE__{word: bin} |
||||
end |
||||
end |
||||
|
||||
def from(_), do: :error |
||||
|
||||
@doc """ |
||||
Tells the type of the field at index `index` in the field schema. |
||||
""" |
||||
@spec type_of(t(), non_neg_integer()) :: non_neg_integer() |
||||
def type_of(%FieldSchema{word: word}, index), do: :binary.at(word, index + 4) |
||||
end |
||||
|
||||
@enforce_keys [:key_schema, :value_schema, :key_names, :value_names] |
||||
defstruct [:key_schema, :value_schema, :key_names, :value_names] |
||||
|
||||
@typedoc """ |
||||
The MUD table schema. Describe column types and names for the given MUD table. |
||||
* `key_schema` - The field schema for the key columns. |
||||
* `value_schema` - The field schema for the value columns. |
||||
* `key_names` - The names of the key columns. |
||||
* `value_names` - The names of the value columns. |
||||
""" |
||||
@type t :: %__MODULE__{ |
||||
key_schema: FieldSchema.t(), |
||||
value_schema: FieldSchema.t(), |
||||
key_names: [String.t()], |
||||
value_names: [String.t()] |
||||
} |
||||
|
||||
defimpl Jason.Encoder, for: Explorer.Chain.Mud.Schema do |
||||
alias Explorer.Chain.Mud.Schema |
||||
alias Jason.Encode |
||||
|
||||
def encode(data, opts) do |
||||
Encode.map( |
||||
%{ |
||||
"key_types" => data.key_schema |> Schema.decode_type_names(), |
||||
"value_types" => data.value_schema |> Schema.decode_type_names(), |
||||
"key_names" => data.key_names, |
||||
"value_names" => data.value_names |
||||
}, |
||||
opts |
||||
) |
||||
end |
||||
end |
||||
|
||||
@doc """ |
||||
Tells the number of static fields in the schema and the list of raw type IDs of all fields in the schema. |
||||
""" |
||||
@spec decode_types(FieldSchema.t()) :: {non_neg_integer(), [non_neg_integer()]} |
||||
def decode_types(layout_schema) do |
||||
static_fields_count = :binary.at(layout_schema.word, 2) |
||||
dynamic_fields_count = :binary.at(layout_schema.word, 3) |
||||
|
||||
{static_fields_count, :binary.bin_to_list(layout_schema.word, 4, static_fields_count + dynamic_fields_count)} |
||||
end |
||||
|
||||
@doc """ |
||||
Tells the list of decoded type names for all fields in the schema. |
||||
""" |
||||
@spec decode_type_names(FieldSchema.t()) :: [String.t()] |
||||
def decode_type_names(layout_schema) do |
||||
{_, types} = decode_types(layout_schema) |
||||
types |> Enum.map(&encode_type_name/1) |
||||
end |
||||
|
||||
# credo:disable-for-next-line Credo.Check.Refactor.CyclomaticComplexity |
||||
defp encode_type_name(type) do |
||||
case type do |
||||
_ when type < 32 -> "uint" <> Integer.to_string((type + 1) * 8) |
||||
_ when type < 64 -> "int" <> Integer.to_string((type - 31) * 8) |
||||
_ when type < 96 -> "bytes" <> Integer.to_string(type - 63) |
||||
96 -> "bool" |
||||
97 -> "address" |
||||
_ when type < 196 -> encode_type_name(type - 98) <> "[]" |
||||
196 -> "bytes" |
||||
197 -> "string" |
||||
_ -> "unknown_type_" <> Integer.to_string(type) |
||||
end |
||||
end |
||||
end |
||||
@ -0,0 +1,88 @@ |
||||
defmodule Explorer.Chain.Mud.Table do |
||||
@moduledoc """ |
||||
Represents a decoded MUD framework database table ID. |
||||
""" |
||||
|
||||
alias Explorer.Chain.Hash |
||||
|
||||
@enforce_keys [:table_id, :table_full_name, :table_type, :table_namespace, :table_name] |
||||
@derive Jason.Encoder |
||||
defstruct [:table_id, :table_full_name, :table_type, :table_namespace, :table_name] |
||||
|
||||
@typedoc """ |
||||
Decoded MUD table name struct. |
||||
* `table_id` - The 32-bytes raw MUD table ID. |
||||
* `table_full_name` - The decoded table full name. |
||||
* `table_type` - The decoded table type: "offchain" or "onchain". |
||||
* `table_namespace` - The decoded table namespace. |
||||
* `table_name` - The decoded table name. |
||||
""" |
||||
@type t :: %__MODULE__{ |
||||
table_id: Hash.Full.t(), |
||||
table_full_name: String.t(), |
||||
table_type: String.t(), |
||||
table_namespace: String.t(), |
||||
table_name: String.t() |
||||
} |
||||
|
||||
@doc """ |
||||
Decodes table type, namespace and name information from raw MUD table ID. |
||||
""" |
||||
@spec from(Hash.Full.t()) :: t() |
||||
def from(%Hash{byte_count: 32, bytes: raw} = table_id) do |
||||
<<prefix::binary-size(2), namespace::binary-size(14), table_name::binary-size(16)>> = raw |
||||
|
||||
trimmed_namespace = String.trim_trailing(namespace, <<0>>) |
||||
trimmed_table_name = String.trim_trailing(table_name, <<0>>) |
||||
|
||||
table_full_name = |
||||
if String.length(trimmed_namespace) > 0 do |
||||
prefix <> "." <> trimmed_namespace <> "." <> trimmed_table_name |
||||
else |
||||
prefix <> "." <> trimmed_table_name |
||||
end |
||||
|
||||
table_type = |
||||
case prefix do |
||||
"ot" -> "offchain" |
||||
"tb" -> "onchain" |
||||
_ -> "unknown" |
||||
end |
||||
|
||||
%__MODULE__{ |
||||
table_id: table_id, |
||||
table_full_name: table_full_name, |
||||
table_type: table_type, |
||||
table_namespace: trimmed_namespace, |
||||
table_name: trimmed_table_name |
||||
} |
||||
end |
||||
|
||||
@doc """ |
||||
Encodes table full name as a raw MUD table ID. |
||||
""" |
||||
@spec table_full_name_to_table_id(String.t()) :: {:ok, Hash.Full.t()} | :error |
||||
def table_full_name_to_table_id(full_name) do |
||||
parts = |
||||
case String.split(full_name, ".") do |
||||
[prefix, name] -> [prefix, "", name] |
||||
[prefix, namespace, name] -> [prefix, namespace, name] |
||||
_ -> :error |
||||
end |
||||
|
||||
with [prefix, namespace, name] <- parts, |
||||
{:ok, prefix} <- normalize_length(prefix, 2), |
||||
{:ok, namespace} <- normalize_length(namespace, 14), |
||||
{:ok, name} <- normalize_length(name, 16) do |
||||
Hash.Full.cast(prefix <> namespace <> name) |
||||
end |
||||
end |
||||
|
||||
defp normalize_length(str, len) do |
||||
if String.length(str) <= len do |
||||
{:ok, String.pad_trailing(str, len, <<0>>)} |
||||
else |
||||
:error |
||||
end |
||||
end |
||||
end |
||||
Loading…
Reference in new issue