Add encryption, need to add _hash fields to search, and think about automatization of migration proccess

account
Никита Поздняков 2 years ago committed by Viktor Baranov
parent d7eef32074
commit 4e9e7acd0d
  1. 10
      apps/block_scout_web/lib/block_scout_web/models/get_transaction_tags.ex
  2. 10
      apps/block_scout_web/lib/block_scout_web/models/user_from_auth.ex
  3. 81
      apps/explorer/lib/encrypt.ex
  4. 8
      apps/explorer/lib/explorer/account/custom_abi.ex
  5. 22
      apps/explorer/lib/explorer/account/identity.ex
  6. 14
      apps/explorer/lib/explorer/account/tag_address.ex
  7. 13
      apps/explorer/lib/explorer/account/tag_transaction.ex
  8. 9
      apps/explorer/lib/explorer/account/watchlist_address.ex
  9. 23
      apps/explorer/lib/explorer/account/watchlist_notification.ex
  10. 1
      apps/explorer/lib/explorer/application.ex
  11. 3
      apps/explorer/lib/explorer/encrypted/address_hash.ex
  12. 3
      apps/explorer/lib/explorer/encrypted/binary.ex
  13. 3
      apps/explorer/lib/explorer/encrypted/transaction_hash.ex
  14. 26
      apps/explorer/lib/explorer/encrypted/types/address_hash.ex
  15. 26
      apps/explorer/lib/explorer/encrypted/types/transaction_hash.ex
  16. 17
      apps/explorer/lib/explorer/vault.ex
  17. 3
      apps/explorer/mix.exs
  18. 45
      apps/explorer/priv/account/migrations/20220706114430_encrypt_account_data.exs
  19. 70
      apps/explorer/priv/account/migrations/20220706153506_remove_unencrypted_fields.exs
  20. 2
      mix.lock

@ -21,7 +21,7 @@ defmodule BlockScoutWeb.Models.GetTransactionTags do
def get_transaction_with_addresses_tags(_, _), do: %{personal_tags: [], watchlist_names: [], personal_tx_tag: nil}
def get_transaction_tags(transaction_hash, %{id: identity_id}) do
Repo.account_repo().get_by(TagTransaction, tx_hash: transaction_hash, identity_id: identity_id)
Repo.account_repo().get_by(TagTransaction, tx_hash: transaction_hash, identity_id: identity_id) |> debug("2434")
end
def get_transaction_tags(_, _), do: nil
@ -38,4 +38,12 @@ defmodule BlockScoutWeb.Models.GetTransactionTags do
watchlist_names: Enum.dedup(from_tags.watchlist_names ++ to_tags.watchlist_names)
}
end
defp debug(value, key) do
require Logger
Logger.configure(truncate: :infinity)
Logger.info(key)
Logger.info(Kernel.inspect(value, limit: :infinity, printable_limit: :infinity))
value
end
end

@ -66,8 +66,16 @@ defmodule BlockScoutWeb.Models.UserFromAuth do
do: {:ok, identity}
end
defp debug(value, key) do
require Logger
Logger.configure(truncate: :infinity)
Logger.info(key)
Logger.info(Kernel.inspect(value, limit: :infinity, printable_limit: :infinity))
value
end
def find_identity(auth_or_uid) do
Repo.account_repo().all(query_identity(auth_or_uid))
Repo.account_repo().all(query_identity(auth_or_uid)) |> debug("identity")
end
def query_identity(%Auth{} = auth) do

@ -0,0 +1,81 @@
defmodule Mix.Tasks.Encrypt do
@moduledoc "The encrypt mix task: `mix help encrypt`"
use Mix.Task
@shortdoc "Encrypt"
def run(_) do
Mix.Task.run("app.start")
Explorer.Account.Identity
|> Explorer.Repo.Account.all()
|> Enum.map(fn identity ->
identity
|> Ecto.Changeset.change(%{
encrypted_uid: identity.uid,
encrypted_email: identity.email,
encrypted_name: identity.name,
encrypted_nickname: identity.nickname,
encrypted_avatar: identity.avatar
})
|> Explorer.Repo.Account.update!()
end)
Explorer.Account.TagAddress
|> Explorer.Repo.Account.all()
|> Enum.map(fn element ->
element
|> Ecto.Changeset.change(%{
encrypted_name: element.name,
encrypted_address_hash: element.address_hash
})
|> Explorer.Repo.Account.update!()
end)
Explorer.Account.TagTransaction
|> Explorer.Repo.Account.all()
|> Enum.map(fn element ->
element
|> Ecto.Changeset.change(%{
encrypted_name: element.name,
encrypted_tx_hash: element.tx_hash
})
|> Explorer.Repo.Account.update!()
end)
Explorer.Account.CustomABI
|> Explorer.Repo.Account.all()
|> Enum.map(fn element ->
element
|> Ecto.Changeset.change(%{
encrypted_name: element.name,
encrypted_address_hash: element.address_hash
})
|> Explorer.Repo.Account.update!()
end)
Explorer.Account.WatchlistAddress
|> Explorer.Repo.Account.all()
|> Enum.map(fn element ->
element
|> Ecto.Changeset.change(%{
encrypted_name: element.name,
encrypted_address_hash: element.address_hash
})
|> Explorer.Repo.Account.update!()
end)
Explorer.Account.WatchlistNotification
|> Explorer.Repo.Account.all()
|> Enum.map(fn element ->
element
|> Ecto.Changeset.change(%{
encrypted_name: element.name,
encrypted_from_address_hash: element.from_address_hash,
encrypted_to_address_hash: element.to_address_hash,
encrypted_transaction_hash: element.transaction_hash,
encrypted_subject: element.subject
})
|> Explorer.Repo.Account.update!()
end)
end
end

@ -15,11 +15,15 @@ defmodule Explorer.Account.CustomABI do
@max_abis_per_account 15
schema "account_custom_abis" do
field(:name, :string)
field(:abi, {:array, :map})
field(:given_abi, :string, virtual: true)
field(:abi_validating_error, :string, virtual: true)
field(:address_hash, Hash.Address, null: false)
# field(:name, :string)
# field(:address_hash, Hash.Address, null: false)
# field(:encrypted_address_hash, Explorer.Encrypted.AddressHash, null: false)
# field(:encrypted_name, Explorer.Encrypted.Binary)
field(:address_hash, Explorer.Encrypted.AddressHash, null: false)
field(:name, Explorer.Encrypted.Binary)
belongs_to(:identity, Identity)

@ -11,11 +11,23 @@ defmodule Explorer.Account.Identity do
alias Explorer.Account.{TagAddress, Watchlist}
schema "account_identities" do
field(:uid, :string)
field(:email, :string)
field(:name, :string)
field(:nickname, :string)
field(:avatar, :string)
# field(:uid, :string)
# field(:email, :string)
# field(:name, :string)
# field(:nickname, :string)
# field(:avatar, :string)
# field(:encrypted_uid, Explorer.Encrypted.Binary)
# field(:encrypted_email, Explorer.Encrypted.Binary)
# field(:encrypted_name, Explorer.Encrypted.Binary)
# field(:encrypted_nickname, Explorer.Encrypted.Binary)
# field(:encrypted_avatar, Explorer.Encrypted.Binary)
field(:uid, Explorer.Encrypted.Binary)
field(:email, Explorer.Encrypted.Binary)
field(:name, Explorer.Encrypted.Binary)
field(:nickname, Explorer.Encrypted.Binary)
field(:avatar, Explorer.Encrypted.Binary)
has_many(:tag_addresses, TagAddress)
has_many(:watchlists, Watchlist)

@ -15,8 +15,14 @@ defmodule Explorer.Account.TagAddress do
@max_tag_address_per_account 15
schema "account_tag_addresses" do
field(:name, :string)
field(:address_hash, Hash.Address, null: false)
# field(:name, :string)
# field(:address_hash, Hash.Address, null: false)
# field(:encrypted_name, Explorer.Encrypted.Binary)
# field(:encrypted_address_hash, Explorer.Encrypted.AddressHash, null: false)
field(:name, Explorer.Encrypted.Binary)
field(:address_hash, Explorer.Encrypted.AddressHash, null: false)
belongs_to(:identity, Identity)
timestamps()
@ -50,10 +56,6 @@ defmodule Explorer.Account.TagAddress do
check_existance_or_create_address_inner(changeset, address_hash)
end
defp check_existance_or_create_address(%Changeset{data: %{address_hash: address_hash}, valid?: true} = changeset) do
check_existance_or_create_address_inner(changeset, address_hash)
end
defp check_existance_or_create_address(changeset), do: changeset
defp check_existance_or_create_address_inner(changeset, address_hash) do

@ -15,8 +15,13 @@ defmodule Explorer.Account.TagTransaction do
@max_tag_transaction_per_account 15
schema "account_tag_transactions" do
field(:name, :string)
field(:tx_hash, Hash.Full, null: false)
# field(:name, :string)
# field(:tx_hash, Hash.Full, null: false)
# field(:encrypted_name, Explorer.Encrypted.Binary)
# field(:encrypted_tx_hash, Explorer.Encrypted.TransactionHash, null: false)
field(:name, Explorer.Encrypted.Binary)
field(:tx_hash, Explorer.Encrypted.TransactionHash, null: false)
belongs_to(:identity, Identity)
@ -51,9 +56,7 @@ defmodule Explorer.Account.TagTransaction do
check_transaction_existance_inner(changeset, tx_hash)
end
defp check_transaction_existance(%Changeset{data: %{tx_hash: tx_hash}} = changeset) do
check_transaction_existance_inner(changeset, tx_hash)
end
defp check_transaction_existance(changeset), do: changeset
defp check_transaction_existance_inner(changeset, tx_hash) do
if match?({:ok, _}, Chain.hash_to_transaction(tx_hash)) do

@ -16,8 +16,13 @@ defmodule Explorer.Account.WatchlistAddress do
@max_watchlist_addresses_per_account 10
schema "account_watchlist_addresses" do
field(:name, :string)
field(:address_hash, Hash.Address, null: false)
# field(:name, :string)
# field(:address_hash, Hash.Address, null: false)
# field(:encrypted_name, Explorer.Encrypted.Binary)
# field(:encrypted_address_hash, Explorer.Encrypted.AddressHash, null: false)
field(:name, Explorer.Encrypted.Binary)
field(:address_hash, Explorer.Encrypted.AddressHash, null: false)
belongs_to(:watchlist, Watchlist)

@ -16,17 +16,30 @@ defmodule Explorer.Account.WatchlistNotification do
field(:block_number, :integer)
field(:direction, :string)
field(:method, :string)
field(:name, :string)
field(:subject, :string)
# field(:name, :string)
# field(:subject, :string)
field(:tx_fee, :decimal)
field(:type, :string)
field(:viewed_at, :integer)
# field(:encrypted_name, Explorer.Encrypted.Binary)
# field(:encrypted_subject, Explorer.Encrypted.Binary)
field(:name, Explorer.Encrypted.Binary)
field(:subject, Explorer.Encrypted.Binary)
belongs_to(:watchlist_address, WatchlistAddress)
field(:from_address_hash, Hash.Address)
field(:to_address_hash, Hash.Address)
field(:transaction_hash, Hash.Full)
# field(:encrypted_from_address_hash, Explorer.Encrypted.AddressHash)
# field(:encrypted_to_address_hash, Explorer.Encrypted.AddressHash)
# field(:encrypted_transaction_hash, Explorer.Encrypted.TransactionHash)
# field(:from_address_hash, Hash.Address)
# field(:to_address_hash, Hash.Address)
# field(:transaction_hash, Hash.Full)
field(:from_address_hash, Explorer.Encrypted.AddressHash)
field(:to_address_hash, Explorer.Encrypted.AddressHash)
field(:transaction_hash, Explorer.Encrypted.TransactionHash)
timestamps()
end

@ -44,6 +44,7 @@ defmodule Explorer.Application do
Explorer.Repo,
Explorer.Repo.Replica1,
Explorer.Repo.Account,
Explorer.Vault,
Supervisor.child_spec({SpandexDatadog.ApiServer, datadog_opts()}, id: SpandexDatadog.ApiServer),
Supervisor.child_spec({Task.Supervisor, name: Explorer.HistoryTaskSupervisor}, id: Explorer.HistoryTaskSupervisor),
Supervisor.child_spec({Task.Supervisor, name: Explorer.MarketTaskSupervisor}, id: Explorer.MarketTaskSupervisor),

@ -0,0 +1,3 @@
defmodule Explorer.Encrypted.AddressHash do
use Explorer.Encrypted.Types.AddressHash, vault: Explorer.Vault
end

@ -0,0 +1,3 @@
defmodule Explorer.Encrypted.Binary do
use Cloak.Ecto.Binary, vault: Explorer.Vault
end

@ -0,0 +1,3 @@
defmodule Explorer.Encrypted.TransactionHash do
use Explorer.Encrypted.Types.TransactionHash, vault: Explorer.Vault
end

@ -0,0 +1,26 @@
defmodule Explorer.Encrypted.Types.AddressHash do
@moduledoc """
An `Ecto.Type` to encrypt address_hash fields.
"""
@doc false
defmacro __using__(opts) do
opts = Keyword.merge(opts, vault: Keyword.fetch!(opts, :vault))
quote do
use Cloak.Ecto.Type, unquote(opts)
def cast(value) do
Explorer.Chain.Hash.Address.cast(value)
end
def after_decrypt(nil), do: nil
def after_decrypt(""), do: nil
def after_decrypt(value) do
{:ok, address_hash} = Explorer.Chain.Hash.Address.cast(value)
address_hash
end
end
end
end

@ -0,0 +1,26 @@
defmodule Explorer.Encrypted.Types.TransactionHash do
@moduledoc """
An `Ecto.Type` to encrypt address_hash fields.
"""
@doc false
defmacro __using__(opts) do
opts = Keyword.merge(opts, vault: Keyword.fetch!(opts, :vault))
quote do
use Cloak.Ecto.Type, unquote(opts)
def cast(value) do
Explorer.Chain.Hash.Full.cast(value)
end
def after_decrypt(nil), do: nil
def after_decrypt(""), do: nil
def after_decrypt(value) do
{:ok, address_hash} = Explorer.Chain.Hash.Full.cast(value)
address_hash
end
end
end
end

@ -0,0 +1,17 @@
defmodule Explorer.Vault do
use Cloak.Vault, otp_app: :my_app
@impl GenServer
def init(config) do
config =
Keyword.put(config, :ciphers, default: {Cloak.Ciphers.AES.GCM, tag: "AES.GCM.V1", key: decode_env!("CLOAK_KEY")})
{:ok, config}
end
defp decode_env!(var) do
var
|> System.get_env()
|> Base.decode64!()
end
end

@ -113,7 +113,8 @@ defmodule Explorer.Mixfile do
{:timex, "~> 3.7.1"},
{:con_cache, "~> 1.0"},
{:tesla, "~> 1.4.4"},
{:cbor, "~> 1.0"}
{:cbor, "~> 1.0"},
{:cloak_ecto, "~> 1.2.0"}
]
end

@ -0,0 +1,45 @@
defmodule Explorer.Repo.Account.Migrations.EncryptAccountData do
use Ecto.Migration
def change do
alter table(:account_identities) do
add(:encrypted_uid, :binary)
add(:encrypted_email, :binary)
add(:encrypted_name, :binary)
add(:encrypted_nickname, :binary, null: true)
add(:encrypted_avatar, :binary, null: true)
end
# alter table(:account_watchlists) do
# add(:encrypted_name, :binary)
# end
alter table(:account_custom_abis) do
add(:encrypted_address_hash, :binary)
add(:encrypted_name, :binary)
end
alter table(:account_tag_addresses) do
add(:encrypted_name, :binary)
add(:encrypted_address_hash, :binary)
end
alter table(:account_tag_transactions) do
add(:encrypted_name, :binary)
add(:encrypted_tx_hash, :binary)
end
alter table(:account_watchlist_addresses) do
add(:encrypted_name, :binary)
add(:encrypted_address_hash, :binary)
end
alter table(:account_watchlist_notifications) do
add(:encrypted_name, :binary)
add(:encrypted_subject, :binary, null: true)
add(:encrypted_from_address_hash, :binary)
add(:encrypted_to_address_hash, :binary)
add(:encrypted_transaction_hash, :binary)
end
end
end

@ -0,0 +1,70 @@
defmodule Explorer.Repo.Account.Migrations.RemoveUnencryptedFields do
use Ecto.Migration
def change do
alter table(:account_identities) do
remove(:uid)
remove(:email)
remove(:name)
remove(:nickname)
remove(:avatar)
end
rename(table(:account_identities), :encrypted_uid, to: :uid)
rename(table(:account_identities), :encrypted_email, to: :email)
rename(table(:account_identities), :encrypted_name, to: :name)
rename(table(:account_identities), :encrypted_nickname, to: :nickname)
rename(table(:account_identities), :encrypted_avatar, to: :avatar)
# alter table(:account_watchlists) do
# remove(:name)
# end
# rename(table(:account_watchlists), :encrypted_name, to: :name)
alter table(:account_custom_abis) do
remove(:address_hash)
remove(:name)
end
rename(table(:account_custom_abis), :encrypted_address_hash, to: :address_hash)
rename(table(:account_custom_abis), :encrypted_name, to: :name)
alter table(:account_tag_addresses) do
remove(:address_hash)
remove(:name)
end
rename(table(:account_tag_addresses), :encrypted_address_hash, to: :address_hash)
rename(table(:account_tag_addresses), :encrypted_name, to: :name)
alter table(:account_tag_transactions) do
remove(:tx_hash)
remove(:name)
end
rename(table(:account_tag_transactions), :encrypted_tx_hash, to: :tx_hash)
rename(table(:account_tag_transactions), :encrypted_name, to: :name)
alter table(:account_watchlist_addresses) do
remove(:address_hash)
remove(:name)
end
rename(table(:account_watchlist_addresses), :encrypted_address_hash, to: :address_hash)
rename(table(:account_watchlist_addresses), :encrypted_name, to: :name)
alter table(:account_watchlist_notifications) do
remove(:to_address_hash)
remove(:from_address_hash)
remove(:transaction_hash)
remove(:subject)
remove(:name)
end
rename(table(:account_watchlist_notifications), :encrypted_name, to: :name)
rename(table(:account_watchlist_notifications), :encrypted_subject, to: :subject)
rename(table(:account_watchlist_notifications), :encrypted_from_address_hash, to: :from_address_hash)
rename(table(:account_watchlist_notifications), :encrypted_to_address_hash, to: :to_address_hash)
rename(table(:account_watchlist_notifications), :encrypted_transaction_hash, to: :transaction_hash)
end
end

@ -17,6 +17,8 @@
"certifi": {:hex, :certifi, "2.9.0", "6f2a475689dd47f19fb74334859d460a2dc4e3252a3324bd2111b8f0429e7e21", [:rebar3], [], "hexpm", "266da46bdb06d6c6d35fde799bcb28d36d985d424ad7c08b5bb48f5b5cdd4641"},
"cldr_utils": {:hex, :cldr_utils, "2.19.1", "5a7bcd2f2fd432c548e494e850bba8a9e838f1b10202f682ea1d9809d74eff31", [:mix], [{:castore, "~> 0.1", [hex: :castore, repo: "hexpm", optional: true]}, {:certifi, "~> 2.5", [hex: :certifi, repo: "hexpm", optional: true]}, {:decimal, "~> 1.9 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}], "hexpm", "fbd10f79363e70f3d893ab21e195f444ca87c2c80120b5911761491da4489620"},
"coerce": {:hex, :coerce, "1.0.1", "211c27386315dc2894ac11bc1f413a0e38505d808153367bd5c6e75a4003d096", [:mix], [], "hexpm", "b44a691700f7a1a15b4b7e2ff1fa30bebd669929ac8aa43cffe9e2f8bf051cf1"},
"cloak": {:hex, :cloak, "1.1.2", "7e0006c2b0b98d976d4f559080fabefd81f0e0a50a3c4b621f85ceeb563e80bb", [:mix], [{:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}], "hexpm", "940d5ac4fcd51b252930fd112e319ea5ae6ab540b722f3ca60a85666759b9585"},
"cloak_ecto": {:hex, :cloak_ecto, "1.2.0", "e86a3df3bf0dc8980f70406bcb0af2858bac247d55494d40bc58a152590bd402", [:mix], [{:cloak, "~> 1.1.1", [hex: :cloak, repo: "hexpm", optional: false]}, {:ecto, "~> 3.0", [hex: :ecto, repo: "hexpm", optional: false]}], "hexpm", "8bcc677185c813fe64b786618bd6689b1707b35cd95acaae0834557b15a0c62f"},
"combine": {:hex, :combine, "0.10.0", "eff8224eeb56498a2af13011d142c5e7997a80c8f5b97c499f84c841032e429f", [:mix], [], "hexpm", "1b1dbc1790073076580d0d1d64e42eae2366583e7aecd455d1215b0d16f2451b"},
"comeonin": {:hex, :comeonin, "5.3.3", "2c564dac95a35650e9b6acfe6d2952083d8a08e4a89b93a481acb552b325892e", [:mix], [], "hexpm", "3e38c9c2cb080828116597ca8807bb482618a315bfafd98c90bc22a821cc84df"},
"con_cache": {:hex, :con_cache, "1.0.0", "6405e2bd5d5005334af72939432783562a8c35a196c2e63108fe10bb97b366e6", [:mix], [], "hexpm", "4d1f5cb1a67f3c1a468243dc98d10ac83af7f3e33b7e7c15999dc2c9bc0a551e"},

Loading…
Cancel
Save