Message explorer entities (#998)

* Make domain id the primary key

* Generation code to build entities

* generated entity code

* update docs

* Switch to hex strings for hashes and addresses

* fix log config

* Switch to varchar(64)

* Fix stdio -> stderr

rust/Cargo.lock

@@ -1094,7 +1094,7 @@ version = "3.2.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "1d91974fbbe88ec1df0c24a4f00f99583667a7e2e6272b2b92d294d81e462173"
 dependencies = [
- "nix 0.25.0",
+ "nix",
  "winapi",
 ]
 
@@ -2788,17 +2788,6 @@ dependencies = [
  "tempfile",
 ]
 
-[[package]]
-name = "nix"
-version = "0.24.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "195cdbc1741b8134346d515b3a56a1c94b0912758009cfd53f99ea0f57b065fc"
-dependencies = [
- "bitflags",
- "cfg-if",
- "libc",
-]
-
 [[package]]
 name = "nix"
 version = "0.25.0"
@@ -3807,7 +3796,7 @@ version = "0.1.0"
 dependencies = [
  "ctrlc",
  "maplit",
- "nix 0.24.2",
+ "nix",
  "tempfile",
  "ureq",
 ]

rust/agents/scraper/README.md

@@ -14,5 +14,20 @@ docker rm -v scraper
 To init the database, run from `rust` dir
 
 ```bash
-cargo run --package migration --bin init-db --features tracing,tracing-subscriber
+cargo run --package migration --bin init-db
 ```
+
+To re-create the database, run from `rust` dir
+
+```bash
+cargo run --package migration --bin recreate-db
+```
+
+To re-generate the sea-orm entity code, when no database is running in docker and from the `rust` dir, run
+
+```bash
+cargo run --package migration --bin generate-entities
+```
+
+_Note:_ This will install sea-orm-cli, start a docker container for postgresql, and then replace the existing entities.
+It will not work if docker is not setup or if anything is already bound on port 5432.

rust/agents/scraper/migration/Cargo.toml

@@ -13,8 +13,10 @@ tokio = { version = "1", features = ["rt", "macros"] }
 sea-orm-migration = { version = "0.9", features = ["runtime-tokio-native-tls", "sqlx-postgres"] }
 serde = { version = "1.0", features = ["derive"] }
 abacus-core = { path = "../../../abacus-core" }
-tracing-subscriber = { version = "0.3", optional = true }
-tracing = { version = "0.1", optional = true }
+
+# bin-only deps
+tracing-subscriber = { version = "0.3" }
+tracing = { version = "0.1" }
 
 [dependencies.sea-orm]
 features = ["macros"]
@@ -25,5 +27,13 @@ version = "*"
 name = "init-db"
 path = "bin/init_db.rs"
 
+[[bin]]
+name = "recreate-db"
+path = "bin/recreate_db.rs"
+
+[[bin]]
+name = "generate-entities"
+path = "bin/generate_entities.rs"
+
 [features]
 default = []

rust/agents/scraper/migration/bin/common.rs

@@ -0,0 +1,21 @@
+use std::env;
+
+use migration::sea_orm::{Database, DatabaseConnection};
+pub use migration::{DbErr, Migrator, MigratorTrait as _};
+
+const LOCAL_DATABASE_URL: &str = "postgresql://postgres:47221c18c610@localhost:5432/postgres";
+
+pub fn url() -> String {
+    env::var("DATABASE_URL").unwrap_or_else(|_| LOCAL_DATABASE_URL.into())
+}
+
+pub async fn init() -> Result<DatabaseConnection, DbErr> {
+    tracing_subscriber::fmt()
+        .with_max_level(tracing::Level::DEBUG)
+        .with_test_writer()
+        .init();
+
+    let url = url();
+    println!("Connecting to {url}");
+    Database::connect(url).await
+}

rust/agents/scraper/migration/bin/generate_entities.rs

@@ -0,0 +1,113 @@
+use std::path::Path;
+use std::process::Stdio;
+use std::time::Duration;
+
+use tokio::fs::remove_dir_all;
+use tokio::process::Command;
+use tokio::time::sleep;
+
+use common::*;
+
+mod common;
+
+const RAW_DB_PATH: &str = "./agents/scraper/src/db";
+const DOCKER_NAME: &str = "scraper-entity-generator";
+
+struct PostgresDockerContainer;
+
+impl PostgresDockerContainer {
+    async fn start() -> Result<Self, ()> {
+        let status = Command::new("docker")
+            .args([
+                "run",
+                "--name",
+                DOCKER_NAME,
+                "-e",
+                "POSTGRES_PASSWORD=47221c18c610",
+                "-p",
+                "5432:5432",
+                "--rm",
+                "-d",
+                "postgres:14",
+            ])
+            .stdout(Stdio::inherit())
+            .stderr(Stdio::inherit())
+            .status()
+            .await;
+        if let Ok(status) = status {
+            if status.success() {
+                sleep(Duration::from_secs(1)).await;
+                return Ok(Self);
+            }
+        }
+        Err(())
+    }
+}
+
+impl Drop for PostgresDockerContainer {
+    fn drop(&mut self) {
+        let status = std::process::Command::new("docker")
+            .args(["stop", DOCKER_NAME])
+            .stdout(Stdio::inherit())
+            .stderr(Stdio::inherit())
+            .status();
+        if let Err(e) = status {
+            eprintln!("Encountered error when stopping postgres: {e}");
+        }
+    }
+}
+
+#[tokio::main]
+async fn main() -> Result<(), DbErr> {
+    assert_eq!(
+        std::env::current_dir().unwrap().file_name().unwrap(),
+        "rust",
+        "Must run from the rust dir"
+    );
+    let postgres = PostgresDockerContainer::start();
+
+    let install_cli = tokio::spawn(
+        Command::new("cargo")
+            .args(["install", "sea-orm-cli"])
+            .stdout(Stdio::inherit())
+            .stderr(Stdio::inherit())
+            .status(),
+    );
+
+    let postgres = postgres.await.unwrap();
+    let db = init().await?;
+    Migrator::up(&db, None).await?;
+    drop(db);
+
+    let db_path = Path::new(RAW_DB_PATH);
+    if db_path.exists() {
+        remove_dir_all(db_path)
+            .await
+            .expect("Failed to delete old entity code");
+    }
+
+    assert!(install_cli.await.unwrap().unwrap().success());
+    let generate_status = Command::new("sea-orm-cli")
+        .env("DATABASE_URL", url())
+        .args([
+            "generate",
+            "entity",
+            "--output-dir",
+            db_path.to_str().unwrap(),
+            // we want expanded format because it plays nicely with the IDEs
+            "--expanded-format",
+            "--date-time-crate",
+            "time",
+            "--with-copy-enums",
+        ])
+        .stdout(Stdio::inherit())
+        .stderr(Stdio::inherit())
+        .status()
+        .await
+        .expect("Failed to generate entities")
+        .success();
+    assert!(generate_status, "Failed to generate entities");
+    drop(postgres);
+
+    Ok(())
+}

rust/agents/scraper/migration/bin/init_db.rs

@@ -1,23 +1,13 @@
-use std::env;
+//! Initialize a new, empty database using the migrations.
 
-use migration::sea_orm::Database;
-use migration::{DbErr, Migrator, MigratorTrait as _};
+use common::*;
 
-const LOCAL_DATABASE_URL: &str = "postgresql://postgres:47221c18c610@localhost:5432";
+mod common;
 
 #[tokio::main]
 async fn main() -> Result<(), DbErr> {
-    #[cfg(all(feature = "tracing", feature = "tracing-subscriber"))]
-    tracing_subscriber::fmt()
-        .with_max_level(tracing::Level::DEBUG)
-        .with_test_writer()
-        .init();
+    let db = init().await?;
 
-    let url = env::var("DATABASE_URL").unwrap_or_else(|_| LOCAL_DATABASE_URL.into());
-    println!("Connecting to {url}");
-    let db = Database::connect(url).await?;
-
-    // Migrator::down(&db, None).await?;
     Migrator::up(&db, None).await?;
 
     Ok(())

rust/agents/scraper/migration/bin/recreate_db.rs

@@ -0,0 +1,14 @@
+//! Tare down an existing database and then re-initialize it.
+use common::*;
+
+mod common;
+
+#[tokio::main]
+async fn main() -> Result<(), DbErr> {
+    let db = init().await?;
+
+    Migrator::down(&db, None).await?;
+    Migrator::up(&db, None).await?;
+
+    Ok(())
+}

rust/agents/scraper/migration/src/l20220805_types.rs

@@ -1,9 +1,11 @@
 use sea_orm_migration::prelude::*;
 
+/// Hashes are to be stored as lowercase hex chars without a 0x prefix
 #[allow(non_upper_case_globals)]
-pub const Hash: ColumnType = ColumnType::Binary(BlobSize::Blob(Some(256 / 8)));
+pub const Hash: ColumnType = ColumnType::String(Some(64));
+/// Addresses are to be stored as lowercase hex chars without a 0x prefix
 #[allow(non_upper_case_globals)]
-pub const Address: ColumnType = ColumnType::Binary(BlobSize::Blob(Some(256 / 8)));
+pub const Address: ColumnType = ColumnType::String(Some(64));
 
 /// 256-bit integer as base-10 digits: ceil(log_10(2^256))
 const SIGNIFICANT_DIGITS_IN_256_BIT_INTEGER: u32 = 78;

rust/agents/scraper/src/db/block.rs

@@ -0,0 +1,90 @@
+//! SeaORM Entity. Generated by sea-orm-codegen 0.9.2
+
+use sea_orm::entity::prelude::*;
+
+#[derive(Copy, Clone, Default, Debug, DeriveEntity)]
+pub struct Entity;
+
+impl EntityName for Entity {
+    fn table_name(&self) -> &str {
+        "block"
+    }
+}
+
+#[derive(Clone, Debug, PartialEq, DeriveModel, DeriveActiveModel)]
+pub struct Model {
+    pub id: i64,
+    pub time_created: TimeDateTime,
+    pub domain: i32,
+    pub hash: String,
+    pub height: i64,
+    pub timestamp: TimeDateTime,
+}
+
+#[derive(Copy, Clone, Debug, EnumIter, DeriveColumn)]
+pub enum Column {
+    Id,
+    TimeCreated,
+    Domain,
+    Hash,
+    Height,
+    Timestamp,
+}
+
+#[derive(Copy, Clone, Debug, EnumIter, DerivePrimaryKey)]
+pub enum PrimaryKey {
+    Id,
+}
+
+impl PrimaryKeyTrait for PrimaryKey {
+    type ValueType = i64;
+    fn auto_increment() -> bool {
+        true
+    }
+}
+
+#[derive(Copy, Clone, Debug, EnumIter)]
+pub enum Relation {
+    Domain,
+    Transaction,
+}
+
+impl ColumnTrait for Column {
+    type EntityName = Entity;
+    fn def(&self) -> ColumnDef {
+        match self {
+            Self::Id => ColumnType::BigInteger.def(),
+            Self::TimeCreated => ColumnType::DateTime.def(),
+            Self::Domain => ColumnType::Integer.def(),
+            Self::Hash => ColumnType::String(Some(64u32)).def().unique(),
+            Self::Height => ColumnType::BigInteger.def(),
+            Self::Timestamp => ColumnType::DateTime.def(),
+        }
+    }
+}
+
+impl RelationTrait for Relation {
+    fn def(&self) -> RelationDef {
+        match self {
+            Self::Domain => Entity::belongs_to(super::domain::Entity)
+                .from(Column::Domain)
+                .to(super::domain::Column::Id)
+                .into(),
+            Self::Transaction => Entity::has_many(super::transaction::Entity).into(),
+        }
+    }
+}
+
+impl Related<super::domain::Entity> for Entity {
+    fn to() -> RelationDef {
+        Relation::Domain.def()
+    }
+}
+
+impl Related<super::transaction::Entity> for Entity {
+    fn to() -> RelationDef {
+        Relation::Transaction.def()
+    }
+}
+
+impl ActiveModelBehavior for ActiveModel {}

rust/agents/scraper/src/db/checkpoint.rs

@@ -0,0 +1,99 @@
+//! SeaORM Entity. Generated by sea-orm-codegen 0.9.2
+
+use sea_orm::entity::prelude::*;
+
+#[derive(Copy, Clone, Default, Debug, DeriveEntity)]
+pub struct Entity;
+
+impl EntityName for Entity {
+    fn table_name(&self) -> &str {
+        "checkpoint"
+    }
+}
+
+#[derive(Clone, Debug, PartialEq, DeriveModel, DeriveActiveModel)]
+pub struct Model {
+    pub id: i64,
+    pub time_created: TimeDateTime,
+    pub timestamp: TimeDateTime,
+    pub signature: Vec<u8>,
+    pub validator: String,
+    pub root: String,
+    pub index: i32,
+    pub origin_domain: i32,
+    pub outbox_address: String,
+}
+
+#[derive(Copy, Clone, Debug, EnumIter, DeriveColumn)]
+pub enum Column {
+    Id,
+    TimeCreated,
+    Timestamp,
+    Signature,
+    Validator,
+    Root,
+    Index,
+    OriginDomain,
+    OutboxAddress,
+}
+
+#[derive(Copy, Clone, Debug, EnumIter, DerivePrimaryKey)]
+pub enum PrimaryKey {
+    Id,
+}
+
+impl PrimaryKeyTrait for PrimaryKey {
+    type ValueType = i64;
+    fn auto_increment() -> bool {
+        true
+    }
+}
+
+#[derive(Copy, Clone, Debug, EnumIter)]
+pub enum Relation {
+    Domain,
+    CheckpointUpdate,
+}
+
+impl ColumnTrait for Column {
+    type EntityName = Entity;
+    fn def(&self) -> ColumnDef {
+        match self {
+            Self::Id => ColumnType::BigInteger.def(),
+            Self::TimeCreated => ColumnType::DateTime.def(),
+            Self::Timestamp => ColumnType::DateTime.def(),
+            Self::Signature => ColumnType::Binary.def(),
+            Self::Validator => ColumnType::String(Some(64u32)).def(),
+            Self::Root => ColumnType::String(Some(64u32)).def(),
+            Self::Index => ColumnType::Integer.def(),
+            Self::OriginDomain => ColumnType::Integer.def(),
+            Self::OutboxAddress => ColumnType::String(Some(64u32)).def(),
+        }
+    }
+}
+
+impl RelationTrait for Relation {
+    fn def(&self) -> RelationDef {
+        match self {
+            Self::Domain => Entity::belongs_to(super::domain::Entity)
+                .from(Column::OriginDomain)
+                .to(super::domain::Column::Id)
+                .into(),
+            Self::CheckpointUpdate => Entity::has_many(super::checkpoint_update::Entity).into(),
+        }
+    }
+}
+
+impl Related<super::domain::Entity> for Entity {
+    fn to() -> RelationDef {
+        Relation::Domain.def()
+    }
+}
+
+impl Related<super::checkpoint_update::Entity> for Entity {
+    fn to() -> RelationDef {
+        Relation::CheckpointUpdate.def()
+    }
+}
+
+impl ActiveModelBehavior for ActiveModel {}

rust/agents/scraper/src/db/checkpoint_update.rs

@@ -0,0 +1,91 @@
+//! SeaORM Entity. Generated by sea-orm-codegen 0.9.2
+
+use super::sea_orm_active_enums::CheckpointUpdateType;
+use sea_orm::entity::prelude::*;
+
+#[derive(Copy, Clone, Default, Debug, DeriveEntity)]
+pub struct Entity;
+
+impl EntityName for Entity {
+    fn table_name(&self) -> &str {
+        "checkpoint_update"
+    }
+}
+
+#[derive(Clone, Debug, PartialEq, DeriveModel, DeriveActiveModel)]
+pub struct Model {
+    pub id: i64,
+    pub time_created: TimeDateTime,
+    pub checkpoint_id: i64,
+    pub update_type: CheckpointUpdateType,
+    pub tx_id: i64,
+}
+
+#[derive(Copy, Clone, Debug, EnumIter, DeriveColumn)]
+pub enum Column {
+    Id,
+    TimeCreated,
+    CheckpointId,
+    UpdateType,
+    TxId,
+}
+
+#[derive(Copy, Clone, Debug, EnumIter, DerivePrimaryKey)]
+pub enum PrimaryKey {
+    Id,
+}
+
+impl PrimaryKeyTrait for PrimaryKey {
+    type ValueType = i64;
+    fn auto_increment() -> bool {
+        true
+    }
+}
+
+#[derive(Copy, Clone, Debug, EnumIter)]
+pub enum Relation {
+    Checkpoint,
+    Transaction,
+}
+
+impl ColumnTrait for Column {
+    type EntityName = Entity;
+    fn def(&self) -> ColumnDef {
+        match self {
+            Self::Id => ColumnType::BigInteger.def(),
+            Self::TimeCreated => ColumnType::DateTime.def(),
+            Self::CheckpointId => ColumnType::BigInteger.def(),
+            Self::UpdateType => CheckpointUpdateType::db_type(),
+            Self::TxId => ColumnType::BigInteger.def(),
+        }
+    }
+}
+
+impl RelationTrait for Relation {
+    fn def(&self) -> RelationDef {
+        match self {
+            Self::Checkpoint => Entity::belongs_to(super::checkpoint::Entity)
+                .from(Column::CheckpointId)
+                .to(super::checkpoint::Column::Id)
+                .into(),
+            Self::Transaction => Entity::belongs_to(super::transaction::Entity)
+                .from(Column::TxId)
+                .to(super::transaction::Column::Id)
+                .into(),
+        }
+    }
+}
+
+impl Related<super::checkpoint::Entity> for Entity {
+    fn to() -> RelationDef {
+        Relation::Checkpoint.def()
+    }
+}
+
+impl Related<super::transaction::Entity> for Entity {
+    fn to() -> RelationDef {
+        Relation::Transaction.def()
+    }
+}
+
+impl ActiveModelBehavior for ActiveModel {}

rust/agents/scraper/src/db/cursor.rs

@@ -0,0 +1,73 @@
+//! SeaORM Entity. Generated by sea-orm-codegen 0.9.2
+
+use sea_orm::entity::prelude::*;
+
+#[derive(Copy, Clone, Default, Debug, DeriveEntity)]
+pub struct Entity;
+
+impl EntityName for Entity {
+    fn table_name(&self) -> &str {
+        "cursor"
+    }
+}
+
+#[derive(Clone, Debug, PartialEq, DeriveModel, DeriveActiveModel)]
+pub struct Model {
+    pub domain: i32,
+    pub time_updated: TimeDateTime,
+    pub height: i64,
+}
+
+#[derive(Copy, Clone, Debug, EnumIter, DeriveColumn)]
+pub enum Column {
+    Domain,
+    TimeUpdated,
+    Height,
+}
+
+#[derive(Copy, Clone, Debug, EnumIter, DerivePrimaryKey)]
+pub enum PrimaryKey {
+    Domain,
+}
+
+impl PrimaryKeyTrait for PrimaryKey {
+    type ValueType = i32;
+    fn auto_increment() -> bool {
+        false
+    }
+}
+
+#[derive(Copy, Clone, Debug, EnumIter)]
+pub enum Relation {
+    Domain,
+}
+
+impl ColumnTrait for Column {
+    type EntityName = Entity;
+    fn def(&self) -> ColumnDef {
+        match self {
+            Self::Domain => ColumnType::Integer.def(),
+            Self::TimeUpdated => ColumnType::DateTime.def(),
+            Self::Height => ColumnType::BigInteger.def(),
+        }
+    }
+}
+
+impl RelationTrait for Relation {
+    fn def(&self) -> RelationDef {
+        match self {
+            Self::Domain => Entity::belongs_to(super::domain::Entity)
+                .from(Column::Domain)
+                .to(super::domain::Column::Id)
+                .into(),
+        }
+    }
+}
+
+impl Related<super::domain::Entity> for Entity {
+    fn to() -> RelationDef {
+        Relation::Domain.def()
+    }
+}
+
+impl ActiveModelBehavior for ActiveModel {}

rust/agents/scraper/src/db/delivered_message.rs

@@ -0,0 +1,90 @@
+//! SeaORM Entity. Generated by sea-orm-codegen 0.9.2
+
+use sea_orm::entity::prelude::*;
+
+#[derive(Copy, Clone, Default, Debug, DeriveEntity)]
+pub struct Entity;
+
+impl EntityName for Entity {
+    fn table_name(&self) -> &str {
+        "delivered_message"
+    }
+}
+
+#[derive(Clone, Debug, PartialEq, DeriveModel, DeriveActiveModel)]
+pub struct Model {
+    pub id: i64,
+    pub time_created: TimeDateTime,
+    pub inbox_address: String,
+    pub msg_id: i64,
+    pub tx_id: i64,
+}
+
+#[derive(Copy, Clone, Debug, EnumIter, DeriveColumn)]
+pub enum Column {
+    Id,
+    TimeCreated,
+    InboxAddress,
+    MsgId,
+    TxId,
+}
+
+#[derive(Copy, Clone, Debug, EnumIter, DerivePrimaryKey)]
+pub enum PrimaryKey {
+    Id,
+}
+
+impl PrimaryKeyTrait for PrimaryKey {
+    type ValueType = i64;
+    fn auto_increment() -> bool {
+        true
+    }
+}
+
+#[derive(Copy, Clone, Debug, EnumIter)]
+pub enum Relation {
+    Message,
+    Transaction,
+}
+
+impl ColumnTrait for Column {
+    type EntityName = Entity;
+    fn def(&self) -> ColumnDef {
+        match self {
+            Self::Id => ColumnType::BigInteger.def(),
+            Self::TimeCreated => ColumnType::DateTime.def(),
+            Self::InboxAddress => ColumnType::String(Some(64u32)).def(),
+            Self::MsgId => ColumnType::BigInteger.def().unique(),
+            Self::TxId => ColumnType::BigInteger.def(),
+        }
+    }
+}
+
+impl RelationTrait for Relation {
+    fn def(&self) -> RelationDef {
+        match self {
+            Self::Message => Entity::belongs_to(super::message::Entity)
+                .from(Column::MsgId)
+                .to(super::message::Column::Id)
+                .into(),
+            Self::Transaction => Entity::belongs_to(super::transaction::Entity)
+                .from(Column::TxId)
+                .to(super::transaction::Column::Id)
+                .into(),
+        }
+    }
+}
+
+impl Related<super::message::Entity> for Entity {
+    fn to() -> RelationDef {
+        Relation::Message.def()
+    }
+}
+
+impl Related<super::transaction::Entity> for Entity {
+    fn to() -> RelationDef {
+        Relation::Transaction.def()
+    }
+}
+
+impl ActiveModelBehavior for ActiveModel {}

rust/agents/scraper/src/db/domain.rs

@@ -0,0 +1,106 @@
+//! SeaORM Entity. Generated by sea-orm-codegen 0.9.2
+
+use sea_orm::entity::prelude::*;
+
+#[derive(Copy, Clone, Default, Debug, DeriveEntity)]
+pub struct Entity;
+
+impl EntityName for Entity {
+    fn table_name(&self) -> &str {
+        "domain"
+    }
+}
+
+#[derive(Clone, Debug, PartialEq, DeriveModel, DeriveActiveModel)]
+pub struct Model {
+    pub id: i32,
+    pub time_created: TimeDateTime,
+    pub time_updated: TimeDateTime,
+    pub name: String,
+    pub native_token: String,
+    pub chain_id: Option<i64>,
+    pub is_test_net: bool,
+}
+
+#[derive(Copy, Clone, Debug, EnumIter, DeriveColumn)]
+pub enum Column {
+    Id,
+    TimeCreated,
+    TimeUpdated,
+    Name,
+    NativeToken,
+    ChainId,
+    IsTestNet,
+}
+
+#[derive(Copy, Clone, Debug, EnumIter, DerivePrimaryKey)]
+pub enum PrimaryKey {
+    Id,
+}
+
+impl PrimaryKeyTrait for PrimaryKey {
+    type ValueType = i32;
+    fn auto_increment() -> bool {
+        false
+    }
+}
+
+#[derive(Copy, Clone, Debug, EnumIter)]
+pub enum Relation {
+    Block,
+    Checkpoint,
+    GasPayment,
+    Cursor,
+}
+
+impl ColumnTrait for Column {
+    type EntityName = Entity;
+    fn def(&self) -> ColumnDef {
+        match self {
+            Self::Id => ColumnType::Integer.def(),
+            Self::TimeCreated => ColumnType::DateTime.def(),
+            Self::TimeUpdated => ColumnType::DateTime.def(),
+            Self::Name => ColumnType::Text.def(),
+            Self::NativeToken => ColumnType::Text.def(),
+            Self::ChainId => ColumnType::BigInteger.def().null().unique(),
+            Self::IsTestNet => ColumnType::Boolean.def(),
+        }
+    }
+}
+
+impl RelationTrait for Relation {
+    fn def(&self) -> RelationDef {
+        match self {
+            Self::Block => Entity::has_many(super::block::Entity).into(),
+            Self::Checkpoint => Entity::has_many(super::checkpoint::Entity).into(),
+            Self::GasPayment => Entity::has_many(super::gas_payment::Entity).into(),
+            Self::Cursor => Entity::has_many(super::cursor::Entity).into(),
+        }
+    }
+}
+
+impl Related<super::block::Entity> for Entity {
+    fn to() -> RelationDef {
+        Relation::Block.def()
+    }
+}
+
+impl Related<super::checkpoint::Entity> for Entity {
+    fn to() -> RelationDef {
+        Relation::Checkpoint.def()
+    }
+}
+
+impl Related<super::gas_payment::Entity> for Entity {
+    fn to() -> RelationDef {
+        Relation::GasPayment.def()
+    }
+}
+
+impl Related<super::cursor::Entity> for Entity {
+    fn to() -> RelationDef {
+        Relation::Cursor.def()
+    }
+}
+
+impl ActiveModelBehavior for ActiveModel {}

rust/agents/scraper/src/db/gas_payment.rs

@@ -0,0 +1,96 @@
+//! SeaORM Entity. Generated by sea-orm-codegen 0.9.2
+
+use sea_orm::entity::prelude::*;
+
+#[derive(Copy, Clone, Default, Debug, DeriveEntity)]
+pub struct Entity;
+
+impl EntityName for Entity {
+    fn table_name(&self) -> &str {
+        "gas_payment"
+    }
+}
+
+#[derive(Clone, Debug, PartialEq, DeriveModel, DeriveActiveModel)]
+pub struct Model {
+    pub id: i64,
+    pub time_created: TimeDateTime,
+    pub domain: i32,
+    pub leaf_index: i32,
+    pub outbox_address: String,
+    pub amount: Decimal,
+    pub tx_id: i64,
+}
+
+#[derive(Copy, Clone, Debug, EnumIter, DeriveColumn)]
+pub enum Column {
+    Id,
+    TimeCreated,
+    Domain,
+    LeafIndex,
+    OutboxAddress,
+    Amount,
+    TxId,
+}
+
+#[derive(Copy, Clone, Debug, EnumIter, DerivePrimaryKey)]
+pub enum PrimaryKey {
+    Id,
+}
+
+impl PrimaryKeyTrait for PrimaryKey {
+    type ValueType = i64;
+    fn auto_increment() -> bool {
+        true
+    }
+}
+
+#[derive(Copy, Clone, Debug, EnumIter)]
+pub enum Relation {
+    Domain,
+    Transaction,
+}
+
+impl ColumnTrait for Column {
+    type EntityName = Entity;
+    fn def(&self) -> ColumnDef {
+        match self {
+            Self::Id => ColumnType::BigInteger.def(),
+            Self::TimeCreated => ColumnType::DateTime.def(),
+            Self::Domain => ColumnType::Integer.def(),
+            Self::LeafIndex => ColumnType::Integer.def(),
+            Self::OutboxAddress => ColumnType::String(Some(64u32)).def(),
+            Self::Amount => ColumnType::Decimal(Some((78u32, 18u32))).def(),
+            Self::TxId => ColumnType::BigInteger.def(),
+        }
+    }
+}
+
+impl RelationTrait for Relation {
+    fn def(&self) -> RelationDef {
+        match self {
+            Self::Domain => Entity::belongs_to(super::domain::Entity)
+                .from(Column::Domain)
+                .to(super::domain::Column::Id)
+                .into(),
+            Self::Transaction => Entity::belongs_to(super::transaction::Entity)
+                .from(Column::TxId)
+                .to(super::transaction::Column::Id)
+                .into(),
+        }
+    }
+}
+
+impl Related<super::domain::Entity> for Entity {
+    fn to() -> RelationDef {
+        Relation::Domain.def()
+    }
+}
+
+impl Related<super::transaction::Entity> for Entity {
+    fn to() -> RelationDef {
+        Relation::Transaction.def()
+    }
+}
+
+impl ActiveModelBehavior for ActiveModel {}

rust/agents/scraper/src/db/message.rs

@@ -0,0 +1,123 @@
+//! SeaORM Entity. Generated by sea-orm-codegen 0.9.2
+
+use sea_orm::entity::prelude::*;
+
+#[derive(Copy, Clone, Default, Debug, DeriveEntity)]
+pub struct Entity;
+
+impl EntityName for Entity {
+    fn table_name(&self) -> &str {
+        "message"
+    }
+}
+
+#[derive(Clone, Debug, PartialEq, DeriveModel, DeriveActiveModel)]
+pub struct Model {
+    pub id: i64,
+    pub time_created: TimeDateTime,
+    pub origin: i32,
+    pub destination: i32,
+    pub leaf_index: i32,
+    pub sender: String,
+    pub recipient: String,
+    pub msg_body: Option<Vec<u8>>,
+    pub outbox_address: String,
+    pub timestamp: TimeDateTime,
+    pub origin_tx_id: i64,
+}
+
+#[derive(Copy, Clone, Debug, EnumIter, DeriveColumn)]
+pub enum Column {
+    Id,
+    TimeCreated,
+    Origin,
+    Destination,
+    LeafIndex,
+    Sender,
+    Recipient,
+    MsgBody,
+    OutboxAddress,
+    Timestamp,
+    OriginTxId,
+}
+
+#[derive(Copy, Clone, Debug, EnumIter, DerivePrimaryKey)]
+pub enum PrimaryKey {
+    Id,
+}
+
+impl PrimaryKeyTrait for PrimaryKey {
+    type ValueType = i64;
+    fn auto_increment() -> bool {
+        true
+    }
+}
+
+#[derive(Copy, Clone, Debug, EnumIter)]
+pub enum Relation {
+    Domain2,
+    Domain1,
+    Transaction,
+    DeliveredMessage,
+    MessageState,
+}
+
+impl ColumnTrait for Column {
+    type EntityName = Entity;
+    fn def(&self) -> ColumnDef {
+        match self {
+            Self::Id => ColumnType::BigInteger.def(),
+            Self::TimeCreated => ColumnType::DateTime.def(),
+            Self::Origin => ColumnType::Integer.def(),
+            Self::Destination => ColumnType::Integer.def(),
+            Self::LeafIndex => ColumnType::Integer.def(),
+            Self::Sender => ColumnType::String(Some(64u32)).def(),
+            Self::Recipient => ColumnType::String(Some(64u32)).def(),
+            Self::MsgBody => ColumnType::Binary.def().null(),
+            Self::OutboxAddress => ColumnType::String(Some(64u32)).def(),
+            Self::Timestamp => ColumnType::DateTime.def(),
+            Self::OriginTxId => ColumnType::BigInteger.def(),
+        }
+    }
+}
+
+impl RelationTrait for Relation {
+    fn def(&self) -> RelationDef {
+        match self {
+            Self::Domain2 => Entity::belongs_to(super::domain::Entity)
+                .from(Column::Destination)
+                .to(super::domain::Column::Id)
+                .into(),
+            Self::Domain1 => Entity::belongs_to(super::domain::Entity)
+                .from(Column::Origin)
+                .to(super::domain::Column::Id)
+                .into(),
+            Self::Transaction => Entity::belongs_to(super::transaction::Entity)
+                .from(Column::OriginTxId)
+                .to(super::transaction::Column::Id)
+                .into(),
+            Self::DeliveredMessage => Entity::has_one(super::delivered_message::Entity).into(),
+            Self::MessageState => Entity::has_many(super::message_state::Entity).into(),
+        }
+    }
+}
+
+impl Related<super::transaction::Entity> for Entity {
+    fn to() -> RelationDef {
+        Relation::Transaction.def()
+    }
+}
+
+impl Related<super::delivered_message::Entity> for Entity {
+    fn to() -> RelationDef {
+        Relation::DeliveredMessage.def()
+    }
+}
+
+impl Related<super::message_state::Entity> for Entity {
+    fn to() -> RelationDef {
+        Relation::MessageState.def()
+    }
+}
+
+impl ActiveModelBehavior for ActiveModel {}

rust/agents/scraper/src/db/message_state.rs

@@ -0,0 +1,88 @@
+//! SeaORM Entity. Generated by sea-orm-codegen 0.9.2
+
+use sea_orm::entity::prelude::*;
+
+#[derive(Copy, Clone, Default, Debug, DeriveEntity)]
+pub struct Entity;
+
+impl EntityName for Entity {
+    fn table_name(&self) -> &str {
+        "message_state"
+    }
+}
+
+#[derive(Clone, Debug, PartialEq, DeriveModel, DeriveActiveModel)]
+pub struct Model {
+    pub id: i64,
+    pub time_created: TimeDateTime,
+    pub msg_id: i64,
+    pub block_height: i64,
+    pub block_timestamp: TimeDateTime,
+    pub processable: bool,
+    pub estimated_gas_cost: Option<Decimal>,
+    pub error_msg: Option<String>,
+}
+
+#[derive(Copy, Clone, Debug, EnumIter, DeriveColumn)]
+pub enum Column {
+    Id,
+    TimeCreated,
+    MsgId,
+    BlockHeight,
+    BlockTimestamp,
+    Processable,
+    EstimatedGasCost,
+    ErrorMsg,
+}
+
+#[derive(Copy, Clone, Debug, EnumIter, DerivePrimaryKey)]
+pub enum PrimaryKey {
+    Id,
+}
+
+impl PrimaryKeyTrait for PrimaryKey {
+    type ValueType = i64;
+    fn auto_increment() -> bool {
+        true
+    }
+}
+
+#[derive(Copy, Clone, Debug, EnumIter)]
+pub enum Relation {
+    Message,
+}
+
+impl ColumnTrait for Column {
+    type EntityName = Entity;
+    fn def(&self) -> ColumnDef {
+        match self {
+            Self::Id => ColumnType::BigInteger.def(),
+            Self::TimeCreated => ColumnType::DateTime.def(),
+            Self::MsgId => ColumnType::BigInteger.def(),
+            Self::BlockHeight => ColumnType::BigInteger.def(),
+            Self::BlockTimestamp => ColumnType::DateTime.def(),
+            Self::Processable => ColumnType::Boolean.def(),
+            Self::EstimatedGasCost => ColumnType::Decimal(Some((78u32, 18u32))).def().null(),
+            Self::ErrorMsg => ColumnType::Text.def().null(),
+        }
+    }
+}
+
+impl RelationTrait for Relation {
+    fn def(&self) -> RelationDef {
+        match self {
+            Self::Message => Entity::belongs_to(super::message::Entity)
+                .from(Column::MsgId)
+                .to(super::message::Column::Id)
+                .into(),
+        }
+    }
+}
+
+impl Related<super::message::Entity> for Entity {
+    fn to() -> RelationDef {
+        Relation::Message.def()
+    }
+}
+
+impl ActiveModelBehavior for ActiveModel {}

rust/agents/scraper/src/db/mod.rs

@@ -0,0 +1,15 @@
+//! SeaORM Entity. Generated by sea-orm-codegen 0.9.2
+
+pub mod prelude;
+
+pub mod block;
+pub mod checkpoint;
+pub mod checkpoint_update;
+pub mod cursor;
+pub mod delivered_message;
+pub mod domain;
+pub mod gas_payment;
+pub mod message;
+pub mod message_state;
+pub mod sea_orm_active_enums;
+pub mod transaction;

rust/agents/scraper/src/db/prelude.rs

@@ -0,0 +1,12 @@
+//! SeaORM Entity. Generated by sea-orm-codegen 0.9.2
+
+pub use super::block::Entity as Block;
+pub use super::checkpoint::Entity as Checkpoint;
+pub use super::checkpoint_update::Entity as CheckpointUpdate;
+pub use super::cursor::Entity as Cursor;
+pub use super::delivered_message::Entity as DeliveredMessage;
+pub use super::domain::Entity as Domain;
+pub use super::gas_payment::Entity as GasPayment;
+pub use super::message::Entity as Message;
+pub use super::message_state::Entity as MessageState;
+pub use super::transaction::Entity as Transaction;

rust/agents/scraper/src/db/sea_orm_active_enums.rs

@@ -0,0 +1,16 @@
+//! SeaORM Entity. Generated by sea-orm-codegen 0.9.2
+
+use sea_orm::entity::prelude::*;
+
+#[derive(Debug, Clone, PartialEq, EnumIter, DeriveActiveEnum, Copy)]
+#[sea_orm(
+    rs_type = "String",
+    db_type = "Enum",
+    enum_name = "checkpoint_update_type"
+)]
+pub enum CheckpointUpdateType {
+    #[sea_orm(string_value = "fraudulent")]
+    Fraudulent,
+    #[sea_orm(string_value = "premature")]
+    Premature,
+}

rust/agents/scraper/src/db/transaction.rs

@@ -0,0 +1,114 @@
+//! SeaORM Entity. Generated by sea-orm-codegen 0.9.2
+
+use sea_orm::entity::prelude::*;
+
+#[derive(Copy, Clone, Default, Debug, DeriveEntity)]
+pub struct Entity;
+
+impl EntityName for Entity {
+    fn table_name(&self) -> &str {
+        "transaction"
+    }
+}
+
+#[derive(Clone, Debug, PartialEq, DeriveModel, DeriveActiveModel)]
+pub struct Model {
+    pub id: i64,
+    pub time_created: TimeDateTime,
+    pub hash: String,
+    pub block_id: i64,
+    pub gas_used: Decimal,
+    pub sender: String,
+}
+
+#[derive(Copy, Clone, Debug, EnumIter, DeriveColumn)]
+pub enum Column {
+    Id,
+    TimeCreated,
+    Hash,
+    BlockId,
+    GasUsed,
+    Sender,
+}
+
+#[derive(Copy, Clone, Debug, EnumIter, DerivePrimaryKey)]
+pub enum PrimaryKey {
+    Id,
+}
+
+impl PrimaryKeyTrait for PrimaryKey {
+    type ValueType = i64;
+    fn auto_increment() -> bool {
+        true
+    }
+}
+
+#[derive(Copy, Clone, Debug, EnumIter)]
+pub enum Relation {
+    Block,
+    GasPayment,
+    DeliveredMessage,
+    CheckpointUpdate,
+    Message,
+}
+
+impl ColumnTrait for Column {
+    type EntityName = Entity;
+    fn def(&self) -> ColumnDef {
+        match self {
+            Self::Id => ColumnType::BigInteger.def(),
+            Self::TimeCreated => ColumnType::DateTime.def(),
+            Self::Hash => ColumnType::String(Some(64u32)).def().unique(),
+            Self::BlockId => ColumnType::BigInteger.def(),
+            Self::GasUsed => ColumnType::Decimal(Some((78u32, 18u32))).def(),
+            Self::Sender => ColumnType::String(Some(64u32)).def(),
+        }
+    }
+}
+
+impl RelationTrait for Relation {
+    fn def(&self) -> RelationDef {
+        match self {
+            Self::Block => Entity::belongs_to(super::block::Entity)
+                .from(Column::BlockId)
+                .to(super::block::Column::Id)
+                .into(),
+            Self::GasPayment => Entity::has_many(super::gas_payment::Entity).into(),
+            Self::DeliveredMessage => Entity::has_many(super::delivered_message::Entity).into(),
+            Self::CheckpointUpdate => Entity::has_many(super::checkpoint_update::Entity).into(),
+            Self::Message => Entity::has_many(super::message::Entity).into(),
+        }
+    }
+}
+
+impl Related<super::block::Entity> for Entity {
+    fn to() -> RelationDef {
+        Relation::Block.def()
+    }
+}
+
+impl Related<super::gas_payment::Entity> for Entity {
+    fn to() -> RelationDef {
+        Relation::GasPayment.def()
+    }
+}
+
+impl Related<super::delivered_message::Entity> for Entity {
+    fn to() -> RelationDef {
+        Relation::DeliveredMessage.def()
+    }
+}
+
+impl Related<super::checkpoint_update::Entity> for Entity {
+    fn to() -> RelationDef {
+        Relation::CheckpointUpdate.def()
+    }
+}
+
+impl Related<super::message::Entity> for Entity {
+    fn to() -> RelationDef {
+        Relation::Message.def()
+    }
+}
+
+impl ActiveModelBehavior for ActiveModel {}

rust/agents/scraper/src/main.rs

@@ -16,6 +16,9 @@
 
 use eyre::Result;
 
+#[allow(clippy::all)]
+mod db;
+
 mod scraper;
 mod settings;
 

rust/utils/run-locally/Cargo.toml

@@ -7,5 +7,5 @@ edition = "2021"
 ctrlc = "3.2"
 tempfile = "3.3"
 maplit = "1.0"
-nix = { version = "0.24", default-features = false, features = ["signal"] }
+nix = { version = "0.25", default-features = false, features = ["signal"] }
 ureq = { version = "2.4", default-features = false }