From 44361545f8e6f880203b8d551fbe647847af2979 Mon Sep 17 00:00:00 2001 From: Mattie Conover Date: Thu, 8 Sep 2022 12:57:44 -0400 Subject: [PATCH] Message explorer entities (#998) * Make domain id the primary key * Generation code to build entities * generated entity code * update docs * Switch to hex strings for hashes and addresses * fix log config * Switch to varchar(64) * Fix stdio -> stderr --- rust/Cargo.lock | 15 +-- rust/agents/scraper/README.md | 17 ++- rust/agents/scraper/migration/Cargo.toml | 14 +- rust/agents/scraper/migration/bin/common.rs | 21 +++ .../migration/bin/generate_entities.rs | 113 ++++++++++++++++ rust/agents/scraper/migration/bin/init_db.rs | 18 +-- .../scraper/migration/bin/recreate_db.rs | 14 ++ .../scraper/migration/src/l20220805_types.rs | 6 +- rust/agents/scraper/src/db/block.rs | 90 +++++++++++++ rust/agents/scraper/src/db/checkpoint.rs | 99 ++++++++++++++ .../scraper/src/db/checkpoint_update.rs | 91 +++++++++++++ rust/agents/scraper/src/db/cursor.rs | 73 +++++++++++ .../scraper/src/db/delivered_message.rs | 90 +++++++++++++ rust/agents/scraper/src/db/domain.rs | 106 +++++++++++++++ rust/agents/scraper/src/db/gas_payment.rs | 96 ++++++++++++++ rust/agents/scraper/src/db/message.rs | 123 ++++++++++++++++++ rust/agents/scraper/src/db/message_state.rs | 88 +++++++++++++ rust/agents/scraper/src/db/mod.rs | 15 +++ rust/agents/scraper/src/db/prelude.rs | 12 ++ .../scraper/src/db/sea_orm_active_enums.rs | 16 +++ rust/agents/scraper/src/db/transaction.rs | 114 ++++++++++++++++ rust/agents/scraper/src/main.rs | 3 + rust/utils/run-locally/Cargo.toml | 2 +- 23 files changed, 1203 insertions(+), 33 deletions(-) create mode 100644 rust/agents/scraper/migration/bin/common.rs create mode 100644 rust/agents/scraper/migration/bin/generate_entities.rs create mode 100644 rust/agents/scraper/migration/bin/recreate_db.rs create mode 100644 rust/agents/scraper/src/db/block.rs create mode 100644 rust/agents/scraper/src/db/checkpoint.rs create mode 100644 rust/agents/scraper/src/db/checkpoint_update.rs create mode 100644 rust/agents/scraper/src/db/cursor.rs create mode 100644 rust/agents/scraper/src/db/delivered_message.rs create mode 100644 rust/agents/scraper/src/db/domain.rs create mode 100644 rust/agents/scraper/src/db/gas_payment.rs create mode 100644 rust/agents/scraper/src/db/message.rs create mode 100644 rust/agents/scraper/src/db/message_state.rs create mode 100644 rust/agents/scraper/src/db/mod.rs create mode 100644 rust/agents/scraper/src/db/prelude.rs create mode 100644 rust/agents/scraper/src/db/sea_orm_active_enums.rs create mode 100644 rust/agents/scraper/src/db/transaction.rs diff --git a/rust/Cargo.lock b/rust/Cargo.lock index 1e30217a7..77d826d15 100644 --- a/rust/Cargo.lock +++ b/rust/Cargo.lock @@ -1094,7 +1094,7 @@ version = "3.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1d91974fbbe88ec1df0c24a4f00f99583667a7e2e6272b2b92d294d81e462173" dependencies = [ - "nix 0.25.0", + "nix", "winapi", ] @@ -2788,17 +2788,6 @@ dependencies = [ "tempfile", ] -[[package]] -name = "nix" -version = "0.24.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "195cdbc1741b8134346d515b3a56a1c94b0912758009cfd53f99ea0f57b065fc" -dependencies = [ - "bitflags", - "cfg-if", - "libc", -] - [[package]] name = "nix" version = "0.25.0" @@ -3807,7 +3796,7 @@ version = "0.1.0" dependencies = [ "ctrlc", "maplit", - "nix 0.24.2", + "nix", "tempfile", "ureq", ] diff --git a/rust/agents/scraper/README.md b/rust/agents/scraper/README.md index 29e7949d3..b36b6403c 100644 --- a/rust/agents/scraper/README.md +++ b/rust/agents/scraper/README.md @@ -14,5 +14,20 @@ docker rm -v scraper To init the database, run from `rust` dir ```bash -cargo run --package migration --bin init-db --features tracing,tracing-subscriber +cargo run --package migration --bin init-db ``` + +To re-create the database, run from `rust` dir + +```bash +cargo run --package migration --bin recreate-db +``` + +To re-generate the sea-orm entity code, when no database is running in docker and from the `rust` dir, run + +```bash +cargo run --package migration --bin generate-entities +``` + +_Note:_ This will install sea-orm-cli, start a docker container for postgresql, and then replace the existing entities. +It will not work if docker is not setup or if anything is already bound on port 5432. diff --git a/rust/agents/scraper/migration/Cargo.toml b/rust/agents/scraper/migration/Cargo.toml index 7fe3b254d..714a28401 100644 --- a/rust/agents/scraper/migration/Cargo.toml +++ b/rust/agents/scraper/migration/Cargo.toml @@ -13,8 +13,10 @@ tokio = { version = "1", features = ["rt", "macros"] } sea-orm-migration = { version = "0.9", features = ["runtime-tokio-native-tls", "sqlx-postgres"] } serde = { version = "1.0", features = ["derive"] } abacus-core = { path = "../../../abacus-core" } -tracing-subscriber = { version = "0.3", optional = true } -tracing = { version = "0.1", optional = true } + +# bin-only deps +tracing-subscriber = { version = "0.3" } +tracing = { version = "0.1" } [dependencies.sea-orm] features = ["macros"] @@ -25,5 +27,13 @@ version = "*" name = "init-db" path = "bin/init_db.rs" +[[bin]] +name = "recreate-db" +path = "bin/recreate_db.rs" + +[[bin]] +name = "generate-entities" +path = "bin/generate_entities.rs" + [features] default = [] diff --git a/rust/agents/scraper/migration/bin/common.rs b/rust/agents/scraper/migration/bin/common.rs new file mode 100644 index 000000000..096f7628c --- /dev/null +++ b/rust/agents/scraper/migration/bin/common.rs @@ -0,0 +1,21 @@ +use std::env; + +use migration::sea_orm::{Database, DatabaseConnection}; +pub use migration::{DbErr, Migrator, MigratorTrait as _}; + +const LOCAL_DATABASE_URL: &str = "postgresql://postgres:47221c18c610@localhost:5432/postgres"; + +pub fn url() -> String { + env::var("DATABASE_URL").unwrap_or_else(|_| LOCAL_DATABASE_URL.into()) +} + +pub async fn init() -> Result { + tracing_subscriber::fmt() + .with_max_level(tracing::Level::DEBUG) + .with_test_writer() + .init(); + + let url = url(); + println!("Connecting to {url}"); + Database::connect(url).await +} diff --git a/rust/agents/scraper/migration/bin/generate_entities.rs b/rust/agents/scraper/migration/bin/generate_entities.rs new file mode 100644 index 000000000..197c679b1 --- /dev/null +++ b/rust/agents/scraper/migration/bin/generate_entities.rs @@ -0,0 +1,113 @@ +use std::path::Path; +use std::process::Stdio; +use std::time::Duration; + +use tokio::fs::remove_dir_all; +use tokio::process::Command; +use tokio::time::sleep; + +use common::*; + +mod common; + +const RAW_DB_PATH: &str = "./agents/scraper/src/db"; +const DOCKER_NAME: &str = "scraper-entity-generator"; + +struct PostgresDockerContainer; + +impl PostgresDockerContainer { + async fn start() -> Result { + let status = Command::new("docker") + .args([ + "run", + "--name", + DOCKER_NAME, + "-e", + "POSTGRES_PASSWORD=47221c18c610", + "-p", + "5432:5432", + "--rm", + "-d", + "postgres:14", + ]) + .stdout(Stdio::inherit()) + .stderr(Stdio::inherit()) + .status() + .await; + if let Ok(status) = status { + if status.success() { + sleep(Duration::from_secs(1)).await; + return Ok(Self); + } + } + Err(()) + } +} + +impl Drop for PostgresDockerContainer { + fn drop(&mut self) { + let status = std::process::Command::new("docker") + .args(["stop", DOCKER_NAME]) + .stdout(Stdio::inherit()) + .stderr(Stdio::inherit()) + .status(); + if let Err(e) = status { + eprintln!("Encountered error when stopping postgres: {e}"); + } + } +} + +#[tokio::main] +async fn main() -> Result<(), DbErr> { + assert_eq!( + std::env::current_dir().unwrap().file_name().unwrap(), + "rust", + "Must run from the rust dir" + ); + let postgres = PostgresDockerContainer::start(); + + let install_cli = tokio::spawn( + Command::new("cargo") + .args(["install", "sea-orm-cli"]) + .stdout(Stdio::inherit()) + .stderr(Stdio::inherit()) + .status(), + ); + + let postgres = postgres.await.unwrap(); + let db = init().await?; + Migrator::up(&db, None).await?; + drop(db); + + let db_path = Path::new(RAW_DB_PATH); + if db_path.exists() { + remove_dir_all(db_path) + .await + .expect("Failed to delete old entity code"); + } + + assert!(install_cli.await.unwrap().unwrap().success()); + let generate_status = Command::new("sea-orm-cli") + .env("DATABASE_URL", url()) + .args([ + "generate", + "entity", + "--output-dir", + db_path.to_str().unwrap(), + // we want expanded format because it plays nicely with the IDEs + "--expanded-format", + "--date-time-crate", + "time", + "--with-copy-enums", + ]) + .stdout(Stdio::inherit()) + .stderr(Stdio::inherit()) + .status() + .await + .expect("Failed to generate entities") + .success(); + assert!(generate_status, "Failed to generate entities"); + drop(postgres); + + Ok(()) +} diff --git a/rust/agents/scraper/migration/bin/init_db.rs b/rust/agents/scraper/migration/bin/init_db.rs index 8d9aa3d87..661080336 100644 --- a/rust/agents/scraper/migration/bin/init_db.rs +++ b/rust/agents/scraper/migration/bin/init_db.rs @@ -1,23 +1,13 @@ -use std::env; +//! Initialize a new, empty database using the migrations. -use migration::sea_orm::Database; -use migration::{DbErr, Migrator, MigratorTrait as _}; +use common::*; -const LOCAL_DATABASE_URL: &str = "postgresql://postgres:47221c18c610@localhost:5432"; +mod common; #[tokio::main] async fn main() -> Result<(), DbErr> { - #[cfg(all(feature = "tracing", feature = "tracing-subscriber"))] - tracing_subscriber::fmt() - .with_max_level(tracing::Level::DEBUG) - .with_test_writer() - .init(); + let db = init().await?; - let url = env::var("DATABASE_URL").unwrap_or_else(|_| LOCAL_DATABASE_URL.into()); - println!("Connecting to {url}"); - let db = Database::connect(url).await?; - - // Migrator::down(&db, None).await?; Migrator::up(&db, None).await?; Ok(()) diff --git a/rust/agents/scraper/migration/bin/recreate_db.rs b/rust/agents/scraper/migration/bin/recreate_db.rs new file mode 100644 index 000000000..ed6b0ac48 --- /dev/null +++ b/rust/agents/scraper/migration/bin/recreate_db.rs @@ -0,0 +1,14 @@ +//! Tare down an existing database and then re-initialize it. +use common::*; + +mod common; + +#[tokio::main] +async fn main() -> Result<(), DbErr> { + let db = init().await?; + + Migrator::down(&db, None).await?; + Migrator::up(&db, None).await?; + + Ok(()) +} diff --git a/rust/agents/scraper/migration/src/l20220805_types.rs b/rust/agents/scraper/migration/src/l20220805_types.rs index 334582391..48334f0c0 100644 --- a/rust/agents/scraper/migration/src/l20220805_types.rs +++ b/rust/agents/scraper/migration/src/l20220805_types.rs @@ -1,9 +1,11 @@ use sea_orm_migration::prelude::*; +/// Hashes are to be stored as lowercase hex chars without a 0x prefix #[allow(non_upper_case_globals)] -pub const Hash: ColumnType = ColumnType::Binary(BlobSize::Blob(Some(256 / 8))); +pub const Hash: ColumnType = ColumnType::String(Some(64)); +/// Addresses are to be stored as lowercase hex chars without a 0x prefix #[allow(non_upper_case_globals)] -pub const Address: ColumnType = ColumnType::Binary(BlobSize::Blob(Some(256 / 8))); +pub const Address: ColumnType = ColumnType::String(Some(64)); /// 256-bit integer as base-10 digits: ceil(log_10(2^256)) const SIGNIFICANT_DIGITS_IN_256_BIT_INTEGER: u32 = 78; diff --git a/rust/agents/scraper/src/db/block.rs b/rust/agents/scraper/src/db/block.rs new file mode 100644 index 000000000..0e1f30cb4 --- /dev/null +++ b/rust/agents/scraper/src/db/block.rs @@ -0,0 +1,90 @@ +//! SeaORM Entity. Generated by sea-orm-codegen 0.9.2 + +use sea_orm::entity::prelude::*; + +#[derive(Copy, Clone, Default, Debug, DeriveEntity)] +pub struct Entity; + +impl EntityName for Entity { + fn table_name(&self) -> &str { + "block" + } +} + +#[derive(Clone, Debug, PartialEq, DeriveModel, DeriveActiveModel)] +pub struct Model { + pub id: i64, + pub time_created: TimeDateTime, + pub domain: i32, + pub hash: String, + pub height: i64, + pub timestamp: TimeDateTime, +} + +#[derive(Copy, Clone, Debug, EnumIter, DeriveColumn)] +pub enum Column { + Id, + TimeCreated, + Domain, + Hash, + Height, + Timestamp, +} + +#[derive(Copy, Clone, Debug, EnumIter, DerivePrimaryKey)] +pub enum PrimaryKey { + Id, +} + +impl PrimaryKeyTrait for PrimaryKey { + type ValueType = i64; + fn auto_increment() -> bool { + true + } +} + +#[derive(Copy, Clone, Debug, EnumIter)] +pub enum Relation { + Domain, + Transaction, +} + +impl ColumnTrait for Column { + type EntityName = Entity; + fn def(&self) -> ColumnDef { + match self { + Self::Id => ColumnType::BigInteger.def(), + Self::TimeCreated => ColumnType::DateTime.def(), + Self::Domain => ColumnType::Integer.def(), + Self::Hash => ColumnType::String(Some(64u32)).def().unique(), + Self::Height => ColumnType::BigInteger.def(), + Self::Timestamp => ColumnType::DateTime.def(), + } + } +} + +impl RelationTrait for Relation { + fn def(&self) -> RelationDef { + match self { + Self::Domain => Entity::belongs_to(super::domain::Entity) + .from(Column::Domain) + .to(super::domain::Column::Id) + .into(), + Self::Transaction => Entity::has_many(super::transaction::Entity).into(), + } + } +} + +impl Related for Entity { + fn to() -> RelationDef { + Relation::Domain.def() + } +} + +impl Related for Entity { + fn to() -> RelationDef { + Relation::Transaction.def() + } +} + +impl ActiveModelBehavior for ActiveModel {} diff --git a/rust/agents/scraper/src/db/checkpoint.rs b/rust/agents/scraper/src/db/checkpoint.rs new file mode 100644 index 000000000..2d72ceb7a --- /dev/null +++ b/rust/agents/scraper/src/db/checkpoint.rs @@ -0,0 +1,99 @@ +//! SeaORM Entity. Generated by sea-orm-codegen 0.9.2 + +use sea_orm::entity::prelude::*; + +#[derive(Copy, Clone, Default, Debug, DeriveEntity)] +pub struct Entity; + +impl EntityName for Entity { + fn table_name(&self) -> &str { + "checkpoint" + } +} + +#[derive(Clone, Debug, PartialEq, DeriveModel, DeriveActiveModel)] +pub struct Model { + pub id: i64, + pub time_created: TimeDateTime, + pub timestamp: TimeDateTime, + pub signature: Vec, + pub validator: String, + pub root: String, + pub index: i32, + pub origin_domain: i32, + pub outbox_address: String, +} + +#[derive(Copy, Clone, Debug, EnumIter, DeriveColumn)] +pub enum Column { + Id, + TimeCreated, + Timestamp, + Signature, + Validator, + Root, + Index, + OriginDomain, + OutboxAddress, +} + +#[derive(Copy, Clone, Debug, EnumIter, DerivePrimaryKey)] +pub enum PrimaryKey { + Id, +} + +impl PrimaryKeyTrait for PrimaryKey { + type ValueType = i64; + fn auto_increment() -> bool { + true + } +} + +#[derive(Copy, Clone, Debug, EnumIter)] +pub enum Relation { + Domain, + CheckpointUpdate, +} + +impl ColumnTrait for Column { + type EntityName = Entity; + fn def(&self) -> ColumnDef { + match self { + Self::Id => ColumnType::BigInteger.def(), + Self::TimeCreated => ColumnType::DateTime.def(), + Self::Timestamp => ColumnType::DateTime.def(), + Self::Signature => ColumnType::Binary.def(), + Self::Validator => ColumnType::String(Some(64u32)).def(), + Self::Root => ColumnType::String(Some(64u32)).def(), + Self::Index => ColumnType::Integer.def(), + Self::OriginDomain => ColumnType::Integer.def(), + Self::OutboxAddress => ColumnType::String(Some(64u32)).def(), + } + } +} + +impl RelationTrait for Relation { + fn def(&self) -> RelationDef { + match self { + Self::Domain => Entity::belongs_to(super::domain::Entity) + .from(Column::OriginDomain) + .to(super::domain::Column::Id) + .into(), + Self::CheckpointUpdate => Entity::has_many(super::checkpoint_update::Entity).into(), + } + } +} + +impl Related for Entity { + fn to() -> RelationDef { + Relation::Domain.def() + } +} + +impl Related for Entity { + fn to() -> RelationDef { + Relation::CheckpointUpdate.def() + } +} + +impl ActiveModelBehavior for ActiveModel {} diff --git a/rust/agents/scraper/src/db/checkpoint_update.rs b/rust/agents/scraper/src/db/checkpoint_update.rs new file mode 100644 index 000000000..5e08277aa --- /dev/null +++ b/rust/agents/scraper/src/db/checkpoint_update.rs @@ -0,0 +1,91 @@ +//! SeaORM Entity. Generated by sea-orm-codegen 0.9.2 + +use super::sea_orm_active_enums::CheckpointUpdateType; +use sea_orm::entity::prelude::*; + +#[derive(Copy, Clone, Default, Debug, DeriveEntity)] +pub struct Entity; + +impl EntityName for Entity { + fn table_name(&self) -> &str { + "checkpoint_update" + } +} + +#[derive(Clone, Debug, PartialEq, DeriveModel, DeriveActiveModel)] +pub struct Model { + pub id: i64, + pub time_created: TimeDateTime, + pub checkpoint_id: i64, + pub update_type: CheckpointUpdateType, + pub tx_id: i64, +} + +#[derive(Copy, Clone, Debug, EnumIter, DeriveColumn)] +pub enum Column { + Id, + TimeCreated, + CheckpointId, + UpdateType, + TxId, +} + +#[derive(Copy, Clone, Debug, EnumIter, DerivePrimaryKey)] +pub enum PrimaryKey { + Id, +} + +impl PrimaryKeyTrait for PrimaryKey { + type ValueType = i64; + fn auto_increment() -> bool { + true + } +} + +#[derive(Copy, Clone, Debug, EnumIter)] +pub enum Relation { + Checkpoint, + Transaction, +} + +impl ColumnTrait for Column { + type EntityName = Entity; + fn def(&self) -> ColumnDef { + match self { + Self::Id => ColumnType::BigInteger.def(), + Self::TimeCreated => ColumnType::DateTime.def(), + Self::CheckpointId => ColumnType::BigInteger.def(), + Self::UpdateType => CheckpointUpdateType::db_type(), + Self::TxId => ColumnType::BigInteger.def(), + } + } +} + +impl RelationTrait for Relation { + fn def(&self) -> RelationDef { + match self { + Self::Checkpoint => Entity::belongs_to(super::checkpoint::Entity) + .from(Column::CheckpointId) + .to(super::checkpoint::Column::Id) + .into(), + Self::Transaction => Entity::belongs_to(super::transaction::Entity) + .from(Column::TxId) + .to(super::transaction::Column::Id) + .into(), + } + } +} + +impl Related for Entity { + fn to() -> RelationDef { + Relation::Checkpoint.def() + } +} + +impl Related for Entity { + fn to() -> RelationDef { + Relation::Transaction.def() + } +} + +impl ActiveModelBehavior for ActiveModel {} diff --git a/rust/agents/scraper/src/db/cursor.rs b/rust/agents/scraper/src/db/cursor.rs new file mode 100644 index 000000000..67a879b59 --- /dev/null +++ b/rust/agents/scraper/src/db/cursor.rs @@ -0,0 +1,73 @@ +//! SeaORM Entity. Generated by sea-orm-codegen 0.9.2 + +use sea_orm::entity::prelude::*; + +#[derive(Copy, Clone, Default, Debug, DeriveEntity)] +pub struct Entity; + +impl EntityName for Entity { + fn table_name(&self) -> &str { + "cursor" + } +} + +#[derive(Clone, Debug, PartialEq, DeriveModel, DeriveActiveModel)] +pub struct Model { + pub domain: i32, + pub time_updated: TimeDateTime, + pub height: i64, +} + +#[derive(Copy, Clone, Debug, EnumIter, DeriveColumn)] +pub enum Column { + Domain, + TimeUpdated, + Height, +} + +#[derive(Copy, Clone, Debug, EnumIter, DerivePrimaryKey)] +pub enum PrimaryKey { + Domain, +} + +impl PrimaryKeyTrait for PrimaryKey { + type ValueType = i32; + fn auto_increment() -> bool { + false + } +} + +#[derive(Copy, Clone, Debug, EnumIter)] +pub enum Relation { + Domain, +} + +impl ColumnTrait for Column { + type EntityName = Entity; + fn def(&self) -> ColumnDef { + match self { + Self::Domain => ColumnType::Integer.def(), + Self::TimeUpdated => ColumnType::DateTime.def(), + Self::Height => ColumnType::BigInteger.def(), + } + } +} + +impl RelationTrait for Relation { + fn def(&self) -> RelationDef { + match self { + Self::Domain => Entity::belongs_to(super::domain::Entity) + .from(Column::Domain) + .to(super::domain::Column::Id) + .into(), + } + } +} + +impl Related for Entity { + fn to() -> RelationDef { + Relation::Domain.def() + } +} + +impl ActiveModelBehavior for ActiveModel {} diff --git a/rust/agents/scraper/src/db/delivered_message.rs b/rust/agents/scraper/src/db/delivered_message.rs new file mode 100644 index 000000000..b7981a5dc --- /dev/null +++ b/rust/agents/scraper/src/db/delivered_message.rs @@ -0,0 +1,90 @@ +//! SeaORM Entity. Generated by sea-orm-codegen 0.9.2 + +use sea_orm::entity::prelude::*; + +#[derive(Copy, Clone, Default, Debug, DeriveEntity)] +pub struct Entity; + +impl EntityName for Entity { + fn table_name(&self) -> &str { + "delivered_message" + } +} + +#[derive(Clone, Debug, PartialEq, DeriveModel, DeriveActiveModel)] +pub struct Model { + pub id: i64, + pub time_created: TimeDateTime, + pub inbox_address: String, + pub msg_id: i64, + pub tx_id: i64, +} + +#[derive(Copy, Clone, Debug, EnumIter, DeriveColumn)] +pub enum Column { + Id, + TimeCreated, + InboxAddress, + MsgId, + TxId, +} + +#[derive(Copy, Clone, Debug, EnumIter, DerivePrimaryKey)] +pub enum PrimaryKey { + Id, +} + +impl PrimaryKeyTrait for PrimaryKey { + type ValueType = i64; + fn auto_increment() -> bool { + true + } +} + +#[derive(Copy, Clone, Debug, EnumIter)] +pub enum Relation { + Message, + Transaction, +} + +impl ColumnTrait for Column { + type EntityName = Entity; + fn def(&self) -> ColumnDef { + match self { + Self::Id => ColumnType::BigInteger.def(), + Self::TimeCreated => ColumnType::DateTime.def(), + Self::InboxAddress => ColumnType::String(Some(64u32)).def(), + Self::MsgId => ColumnType::BigInteger.def().unique(), + Self::TxId => ColumnType::BigInteger.def(), + } + } +} + +impl RelationTrait for Relation { + fn def(&self) -> RelationDef { + match self { + Self::Message => Entity::belongs_to(super::message::Entity) + .from(Column::MsgId) + .to(super::message::Column::Id) + .into(), + Self::Transaction => Entity::belongs_to(super::transaction::Entity) + .from(Column::TxId) + .to(super::transaction::Column::Id) + .into(), + } + } +} + +impl Related for Entity { + fn to() -> RelationDef { + Relation::Message.def() + } +} + +impl Related for Entity { + fn to() -> RelationDef { + Relation::Transaction.def() + } +} + +impl ActiveModelBehavior for ActiveModel {} diff --git a/rust/agents/scraper/src/db/domain.rs b/rust/agents/scraper/src/db/domain.rs new file mode 100644 index 000000000..aac288cd9 --- /dev/null +++ b/rust/agents/scraper/src/db/domain.rs @@ -0,0 +1,106 @@ +//! SeaORM Entity. Generated by sea-orm-codegen 0.9.2 + +use sea_orm::entity::prelude::*; + +#[derive(Copy, Clone, Default, Debug, DeriveEntity)] +pub struct Entity; + +impl EntityName for Entity { + fn table_name(&self) -> &str { + "domain" + } +} + +#[derive(Clone, Debug, PartialEq, DeriveModel, DeriveActiveModel)] +pub struct Model { + pub id: i32, + pub time_created: TimeDateTime, + pub time_updated: TimeDateTime, + pub name: String, + pub native_token: String, + pub chain_id: Option, + pub is_test_net: bool, +} + +#[derive(Copy, Clone, Debug, EnumIter, DeriveColumn)] +pub enum Column { + Id, + TimeCreated, + TimeUpdated, + Name, + NativeToken, + ChainId, + IsTestNet, +} + +#[derive(Copy, Clone, Debug, EnumIter, DerivePrimaryKey)] +pub enum PrimaryKey { + Id, +} + +impl PrimaryKeyTrait for PrimaryKey { + type ValueType = i32; + fn auto_increment() -> bool { + false + } +} + +#[derive(Copy, Clone, Debug, EnumIter)] +pub enum Relation { + Block, + Checkpoint, + GasPayment, + Cursor, +} + +impl ColumnTrait for Column { + type EntityName = Entity; + fn def(&self) -> ColumnDef { + match self { + Self::Id => ColumnType::Integer.def(), + Self::TimeCreated => ColumnType::DateTime.def(), + Self::TimeUpdated => ColumnType::DateTime.def(), + Self::Name => ColumnType::Text.def(), + Self::NativeToken => ColumnType::Text.def(), + Self::ChainId => ColumnType::BigInteger.def().null().unique(), + Self::IsTestNet => ColumnType::Boolean.def(), + } + } +} + +impl RelationTrait for Relation { + fn def(&self) -> RelationDef { + match self { + Self::Block => Entity::has_many(super::block::Entity).into(), + Self::Checkpoint => Entity::has_many(super::checkpoint::Entity).into(), + Self::GasPayment => Entity::has_many(super::gas_payment::Entity).into(), + Self::Cursor => Entity::has_many(super::cursor::Entity).into(), + } + } +} + +impl Related for Entity { + fn to() -> RelationDef { + Relation::Block.def() + } +} + +impl Related for Entity { + fn to() -> RelationDef { + Relation::Checkpoint.def() + } +} + +impl Related for Entity { + fn to() -> RelationDef { + Relation::GasPayment.def() + } +} + +impl Related for Entity { + fn to() -> RelationDef { + Relation::Cursor.def() + } +} + +impl ActiveModelBehavior for ActiveModel {} diff --git a/rust/agents/scraper/src/db/gas_payment.rs b/rust/agents/scraper/src/db/gas_payment.rs new file mode 100644 index 000000000..032c5d2d4 --- /dev/null +++ b/rust/agents/scraper/src/db/gas_payment.rs @@ -0,0 +1,96 @@ +//! SeaORM Entity. Generated by sea-orm-codegen 0.9.2 + +use sea_orm::entity::prelude::*; + +#[derive(Copy, Clone, Default, Debug, DeriveEntity)] +pub struct Entity; + +impl EntityName for Entity { + fn table_name(&self) -> &str { + "gas_payment" + } +} + +#[derive(Clone, Debug, PartialEq, DeriveModel, DeriveActiveModel)] +pub struct Model { + pub id: i64, + pub time_created: TimeDateTime, + pub domain: i32, + pub leaf_index: i32, + pub outbox_address: String, + pub amount: Decimal, + pub tx_id: i64, +} + +#[derive(Copy, Clone, Debug, EnumIter, DeriveColumn)] +pub enum Column { + Id, + TimeCreated, + Domain, + LeafIndex, + OutboxAddress, + Amount, + TxId, +} + +#[derive(Copy, Clone, Debug, EnumIter, DerivePrimaryKey)] +pub enum PrimaryKey { + Id, +} + +impl PrimaryKeyTrait for PrimaryKey { + type ValueType = i64; + fn auto_increment() -> bool { + true + } +} + +#[derive(Copy, Clone, Debug, EnumIter)] +pub enum Relation { + Domain, + Transaction, +} + +impl ColumnTrait for Column { + type EntityName = Entity; + fn def(&self) -> ColumnDef { + match self { + Self::Id => ColumnType::BigInteger.def(), + Self::TimeCreated => ColumnType::DateTime.def(), + Self::Domain => ColumnType::Integer.def(), + Self::LeafIndex => ColumnType::Integer.def(), + Self::OutboxAddress => ColumnType::String(Some(64u32)).def(), + Self::Amount => ColumnType::Decimal(Some((78u32, 18u32))).def(), + Self::TxId => ColumnType::BigInteger.def(), + } + } +} + +impl RelationTrait for Relation { + fn def(&self) -> RelationDef { + match self { + Self::Domain => Entity::belongs_to(super::domain::Entity) + .from(Column::Domain) + .to(super::domain::Column::Id) + .into(), + Self::Transaction => Entity::belongs_to(super::transaction::Entity) + .from(Column::TxId) + .to(super::transaction::Column::Id) + .into(), + } + } +} + +impl Related for Entity { + fn to() -> RelationDef { + Relation::Domain.def() + } +} + +impl Related for Entity { + fn to() -> RelationDef { + Relation::Transaction.def() + } +} + +impl ActiveModelBehavior for ActiveModel {} diff --git a/rust/agents/scraper/src/db/message.rs b/rust/agents/scraper/src/db/message.rs new file mode 100644 index 000000000..770895b93 --- /dev/null +++ b/rust/agents/scraper/src/db/message.rs @@ -0,0 +1,123 @@ +//! SeaORM Entity. Generated by sea-orm-codegen 0.9.2 + +use sea_orm::entity::prelude::*; + +#[derive(Copy, Clone, Default, Debug, DeriveEntity)] +pub struct Entity; + +impl EntityName for Entity { + fn table_name(&self) -> &str { + "message" + } +} + +#[derive(Clone, Debug, PartialEq, DeriveModel, DeriveActiveModel)] +pub struct Model { + pub id: i64, + pub time_created: TimeDateTime, + pub origin: i32, + pub destination: i32, + pub leaf_index: i32, + pub sender: String, + pub recipient: String, + pub msg_body: Option>, + pub outbox_address: String, + pub timestamp: TimeDateTime, + pub origin_tx_id: i64, +} + +#[derive(Copy, Clone, Debug, EnumIter, DeriveColumn)] +pub enum Column { + Id, + TimeCreated, + Origin, + Destination, + LeafIndex, + Sender, + Recipient, + MsgBody, + OutboxAddress, + Timestamp, + OriginTxId, +} + +#[derive(Copy, Clone, Debug, EnumIter, DerivePrimaryKey)] +pub enum PrimaryKey { + Id, +} + +impl PrimaryKeyTrait for PrimaryKey { + type ValueType = i64; + fn auto_increment() -> bool { + true + } +} + +#[derive(Copy, Clone, Debug, EnumIter)] +pub enum Relation { + Domain2, + Domain1, + Transaction, + DeliveredMessage, + MessageState, +} + +impl ColumnTrait for Column { + type EntityName = Entity; + fn def(&self) -> ColumnDef { + match self { + Self::Id => ColumnType::BigInteger.def(), + Self::TimeCreated => ColumnType::DateTime.def(), + Self::Origin => ColumnType::Integer.def(), + Self::Destination => ColumnType::Integer.def(), + Self::LeafIndex => ColumnType::Integer.def(), + Self::Sender => ColumnType::String(Some(64u32)).def(), + Self::Recipient => ColumnType::String(Some(64u32)).def(), + Self::MsgBody => ColumnType::Binary.def().null(), + Self::OutboxAddress => ColumnType::String(Some(64u32)).def(), + Self::Timestamp => ColumnType::DateTime.def(), + Self::OriginTxId => ColumnType::BigInteger.def(), + } + } +} + +impl RelationTrait for Relation { + fn def(&self) -> RelationDef { + match self { + Self::Domain2 => Entity::belongs_to(super::domain::Entity) + .from(Column::Destination) + .to(super::domain::Column::Id) + .into(), + Self::Domain1 => Entity::belongs_to(super::domain::Entity) + .from(Column::Origin) + .to(super::domain::Column::Id) + .into(), + Self::Transaction => Entity::belongs_to(super::transaction::Entity) + .from(Column::OriginTxId) + .to(super::transaction::Column::Id) + .into(), + Self::DeliveredMessage => Entity::has_one(super::delivered_message::Entity).into(), + Self::MessageState => Entity::has_many(super::message_state::Entity).into(), + } + } +} + +impl Related for Entity { + fn to() -> RelationDef { + Relation::Transaction.def() + } +} + +impl Related for Entity { + fn to() -> RelationDef { + Relation::DeliveredMessage.def() + } +} + +impl Related for Entity { + fn to() -> RelationDef { + Relation::MessageState.def() + } +} + +impl ActiveModelBehavior for ActiveModel {} diff --git a/rust/agents/scraper/src/db/message_state.rs b/rust/agents/scraper/src/db/message_state.rs new file mode 100644 index 000000000..3e432a8ee --- /dev/null +++ b/rust/agents/scraper/src/db/message_state.rs @@ -0,0 +1,88 @@ +//! SeaORM Entity. Generated by sea-orm-codegen 0.9.2 + +use sea_orm::entity::prelude::*; + +#[derive(Copy, Clone, Default, Debug, DeriveEntity)] +pub struct Entity; + +impl EntityName for Entity { + fn table_name(&self) -> &str { + "message_state" + } +} + +#[derive(Clone, Debug, PartialEq, DeriveModel, DeriveActiveModel)] +pub struct Model { + pub id: i64, + pub time_created: TimeDateTime, + pub msg_id: i64, + pub block_height: i64, + pub block_timestamp: TimeDateTime, + pub processable: bool, + pub estimated_gas_cost: Option, + pub error_msg: Option, +} + +#[derive(Copy, Clone, Debug, EnumIter, DeriveColumn)] +pub enum Column { + Id, + TimeCreated, + MsgId, + BlockHeight, + BlockTimestamp, + Processable, + EstimatedGasCost, + ErrorMsg, +} + +#[derive(Copy, Clone, Debug, EnumIter, DerivePrimaryKey)] +pub enum PrimaryKey { + Id, +} + +impl PrimaryKeyTrait for PrimaryKey { + type ValueType = i64; + fn auto_increment() -> bool { + true + } +} + +#[derive(Copy, Clone, Debug, EnumIter)] +pub enum Relation { + Message, +} + +impl ColumnTrait for Column { + type EntityName = Entity; + fn def(&self) -> ColumnDef { + match self { + Self::Id => ColumnType::BigInteger.def(), + Self::TimeCreated => ColumnType::DateTime.def(), + Self::MsgId => ColumnType::BigInteger.def(), + Self::BlockHeight => ColumnType::BigInteger.def(), + Self::BlockTimestamp => ColumnType::DateTime.def(), + Self::Processable => ColumnType::Boolean.def(), + Self::EstimatedGasCost => ColumnType::Decimal(Some((78u32, 18u32))).def().null(), + Self::ErrorMsg => ColumnType::Text.def().null(), + } + } +} + +impl RelationTrait for Relation { + fn def(&self) -> RelationDef { + match self { + Self::Message => Entity::belongs_to(super::message::Entity) + .from(Column::MsgId) + .to(super::message::Column::Id) + .into(), + } + } +} + +impl Related for Entity { + fn to() -> RelationDef { + Relation::Message.def() + } +} + +impl ActiveModelBehavior for ActiveModel {} diff --git a/rust/agents/scraper/src/db/mod.rs b/rust/agents/scraper/src/db/mod.rs new file mode 100644 index 000000000..050271b1a --- /dev/null +++ b/rust/agents/scraper/src/db/mod.rs @@ -0,0 +1,15 @@ +//! SeaORM Entity. Generated by sea-orm-codegen 0.9.2 + +pub mod prelude; + +pub mod block; +pub mod checkpoint; +pub mod checkpoint_update; +pub mod cursor; +pub mod delivered_message; +pub mod domain; +pub mod gas_payment; +pub mod message; +pub mod message_state; +pub mod sea_orm_active_enums; +pub mod transaction; diff --git a/rust/agents/scraper/src/db/prelude.rs b/rust/agents/scraper/src/db/prelude.rs new file mode 100644 index 000000000..f2177aea2 --- /dev/null +++ b/rust/agents/scraper/src/db/prelude.rs @@ -0,0 +1,12 @@ +//! SeaORM Entity. Generated by sea-orm-codegen 0.9.2 + +pub use super::block::Entity as Block; +pub use super::checkpoint::Entity as Checkpoint; +pub use super::checkpoint_update::Entity as CheckpointUpdate; +pub use super::cursor::Entity as Cursor; +pub use super::delivered_message::Entity as DeliveredMessage; +pub use super::domain::Entity as Domain; +pub use super::gas_payment::Entity as GasPayment; +pub use super::message::Entity as Message; +pub use super::message_state::Entity as MessageState; +pub use super::transaction::Entity as Transaction; diff --git a/rust/agents/scraper/src/db/sea_orm_active_enums.rs b/rust/agents/scraper/src/db/sea_orm_active_enums.rs new file mode 100644 index 000000000..b2e6d5c82 --- /dev/null +++ b/rust/agents/scraper/src/db/sea_orm_active_enums.rs @@ -0,0 +1,16 @@ +//! SeaORM Entity. Generated by sea-orm-codegen 0.9.2 + +use sea_orm::entity::prelude::*; + +#[derive(Debug, Clone, PartialEq, EnumIter, DeriveActiveEnum, Copy)] +#[sea_orm( + rs_type = "String", + db_type = "Enum", + enum_name = "checkpoint_update_type" +)] +pub enum CheckpointUpdateType { + #[sea_orm(string_value = "fraudulent")] + Fraudulent, + #[sea_orm(string_value = "premature")] + Premature, +} diff --git a/rust/agents/scraper/src/db/transaction.rs b/rust/agents/scraper/src/db/transaction.rs new file mode 100644 index 000000000..8ccc9dda7 --- /dev/null +++ b/rust/agents/scraper/src/db/transaction.rs @@ -0,0 +1,114 @@ +//! SeaORM Entity. Generated by sea-orm-codegen 0.9.2 + +use sea_orm::entity::prelude::*; + +#[derive(Copy, Clone, Default, Debug, DeriveEntity)] +pub struct Entity; + +impl EntityName for Entity { + fn table_name(&self) -> &str { + "transaction" + } +} + +#[derive(Clone, Debug, PartialEq, DeriveModel, DeriveActiveModel)] +pub struct Model { + pub id: i64, + pub time_created: TimeDateTime, + pub hash: String, + pub block_id: i64, + pub gas_used: Decimal, + pub sender: String, +} + +#[derive(Copy, Clone, Debug, EnumIter, DeriveColumn)] +pub enum Column { + Id, + TimeCreated, + Hash, + BlockId, + GasUsed, + Sender, +} + +#[derive(Copy, Clone, Debug, EnumIter, DerivePrimaryKey)] +pub enum PrimaryKey { + Id, +} + +impl PrimaryKeyTrait for PrimaryKey { + type ValueType = i64; + fn auto_increment() -> bool { + true + } +} + +#[derive(Copy, Clone, Debug, EnumIter)] +pub enum Relation { + Block, + GasPayment, + DeliveredMessage, + CheckpointUpdate, + Message, +} + +impl ColumnTrait for Column { + type EntityName = Entity; + fn def(&self) -> ColumnDef { + match self { + Self::Id => ColumnType::BigInteger.def(), + Self::TimeCreated => ColumnType::DateTime.def(), + Self::Hash => ColumnType::String(Some(64u32)).def().unique(), + Self::BlockId => ColumnType::BigInteger.def(), + Self::GasUsed => ColumnType::Decimal(Some((78u32, 18u32))).def(), + Self::Sender => ColumnType::String(Some(64u32)).def(), + } + } +} + +impl RelationTrait for Relation { + fn def(&self) -> RelationDef { + match self { + Self::Block => Entity::belongs_to(super::block::Entity) + .from(Column::BlockId) + .to(super::block::Column::Id) + .into(), + Self::GasPayment => Entity::has_many(super::gas_payment::Entity).into(), + Self::DeliveredMessage => Entity::has_many(super::delivered_message::Entity).into(), + Self::CheckpointUpdate => Entity::has_many(super::checkpoint_update::Entity).into(), + Self::Message => Entity::has_many(super::message::Entity).into(), + } + } +} + +impl Related for Entity { + fn to() -> RelationDef { + Relation::Block.def() + } +} + +impl Related for Entity { + fn to() -> RelationDef { + Relation::GasPayment.def() + } +} + +impl Related for Entity { + fn to() -> RelationDef { + Relation::DeliveredMessage.def() + } +} + +impl Related for Entity { + fn to() -> RelationDef { + Relation::CheckpointUpdate.def() + } +} + +impl Related for Entity { + fn to() -> RelationDef { + Relation::Message.def() + } +} + +impl ActiveModelBehavior for ActiveModel {} diff --git a/rust/agents/scraper/src/main.rs b/rust/agents/scraper/src/main.rs index b5e7e3f8c..91cc16267 100644 --- a/rust/agents/scraper/src/main.rs +++ b/rust/agents/scraper/src/main.rs @@ -16,6 +16,9 @@ use eyre::Result; +#[allow(clippy::all)] +mod db; + mod scraper; mod settings; diff --git a/rust/utils/run-locally/Cargo.toml b/rust/utils/run-locally/Cargo.toml index c8cbd7652..b06fe5489 100644 --- a/rust/utils/run-locally/Cargo.toml +++ b/rust/utils/run-locally/Cargo.toml @@ -7,5 +7,5 @@ edition = "2021" ctrlc = "3.2" tempfile = "3.3" maplit = "1.0" -nix = { version = "0.24", default-features = false, features = ["signal"] } +nix = { version = "0.25", default-features = false, features = ["signal"] } ureq = { version = "2.4", default-features = false }