Update message processor gauge first (#1512)

rust/agents/relayer/src/msg/processor.rs

@@ -63,92 +63,107 @@ impl MessageProcessor {
         }
     }
 
+    /// Tries to get the next message to process.
+    ///
+    /// If no message with self.message_nonce is found, returns None.
+    /// If the message with self.message_nonce is found and has previously
+    /// been marked as processed, increments self.message_nonce and returns
+    /// None.
+    fn try_get_unprocessed_message(&mut self) -> Result<Option<HyperlaneMessage>> {
+        // First, see if we can find the message so we can update the gauge.
+        if let Some(message) = self.db.message_by_nonce(self.message_nonce)? {
+            // Update the latest nonce gauge if the message is destined for one
+            // of the domains we service.
+            if let Some(metrics) = self.metrics.get(message.destination) {
+                metrics.set(message.nonce as i64);
+            }
+
+            // If this message has already been processed, on to the next one.
+            if self
+                .db
+                .retrieve_message_processed(self.message_nonce)?
+                .is_none()
+            {
+                Ok(Some(message))
+            } else {
+                debug!(
+                    msg_nonce=?self.message_nonce,
+                    "Message already marked as processed in DB");
+                self.message_nonce += 1;
+                Ok(None)
+            }
+        } else {
+            debug!(
+                msg_nonce=?self.message_nonce,
+                "No message found in DB for nonce");
+            Ok(None)
+        }
+    }
+
     /// One round of processing, extracted from infinite work loop for
     /// testing purposes.
     async fn tick(&mut self) -> Result<()> {
         // Scan until we find next nonce without delivery confirmation.
-        if self
+        if let Some(message) = self.try_get_unprocessed_message()? {
-            .db
+            debug!(msg=?message, "Working on message");
-            .retrieve_message_processed(self.message_nonce)?
+
-            .is_some()
+            // Skip if not whitelisted.
-        {
+            if !self.whitelist.msg_matches(&message, true) {
-            debug!(
+                debug!(
-                nonce=?self.message_nonce,
+                    msg_id=?message.id(),
-                "Skipping since message_nonce already in DB");
+                    msg_destination=message.destination,
-            self.message_nonce += 1;
+                    msg_nonce=message.nonce,
-            return Ok(());
+                    whitelist=?self.whitelist,
-        }
+                    "Message not whitelisted, skipping");
-        let message = if let Some(msg) = self
+                self.message_nonce += 1;
-            .db
+                return Ok(());
-            .message_by_nonce(self.message_nonce)?
+            }
-            .map(HyperlaneMessage::from)
+
-        {
+            // Skip if the message is blacklisted
-            debug!(msg=?msg, "Working on msg");
+            if self.blacklist.msg_matches(&message, false) {
-            msg
+                debug!(
-        } else {
+                    msg_id=?message.id(),
-            debug!("Leaf in db without message nonce: {}", self.message_nonce);
+                    msg_destination=message.destination,
-            // Not clear what the best thing to do here is, but there is seemingly an
+                    msg_nonce=message.nonce,
-            // existing race wherein an indexer might non-atomically write leaf
+                    blacklist=?self.blacklist,
-            // info to rocksdb across a few records, so we might see the leaf
+                    "Message blacklisted, skipping");
-            // status above, but not the message contents here.  For now,
+                self.message_nonce += 1;
-            // optimistically yield and then re-enter the loop in hopes that the
+                return Ok(());
-            // DB is now coherent. TODO(webbhorn): Why can't we yield here
+            }
-            // instead of sleep?
+
-            tokio::time::sleep(Duration::from_secs(1)).await;
+            // Skip if the message is intended for a destination we do not service
-            return Ok(());
+            if self.send_channels.get(&message.destination).is_none() {
-        };
+                debug!(
-        if let Some(metrics) = self.metrics.get(message.destination) {
+                    msg_id=?message.id(),
-            metrics.set(self.message_nonce as i64);
+                    msg_destination=message.destination,
-        }
+                    msg_nonce=message.nonce,
+                    "Message destined for unknown domain, skipping");
+                self.message_nonce += 1;
+                return Ok(());
+            }
+
+            // Feed the message to the prover sync
+            self.prover_sync
+                .write()
+                .await
+                .update_to_index(message.nonce)
+                .await?;
 
-        // Skip if not whitelisted.
-        if !self.whitelist.msg_matches(&message, true) {
             debug!(
-                id=?message.id(),
+                msg_id=?message.id(),
-                destination=message.destination,
+                msg_nonce=message.nonce,
-                nonce=message.nonce,
+                "Sending message to submitter"
-                whitelist=?self.whitelist,
+            );
-                "Message not whitelisted, skipping");
-            self.message_nonce += 1;
-            return Ok(());
-        }
-
-        // Skip if the message is blacklisted
-        if self.blacklist.msg_matches(&message, false) {
-            debug!(
-                id=?message.id(),
-                destination=message.destination,
-                nonce=message.nonce,
-                blacklist=?self.blacklist,
-                "Message blacklisted, skipping");
-            self.message_nonce += 1;
-            return Ok(());
-        }
 
-        // Feed the message to the prover sync
+            // Finally, build the submit arg and dispatch it to the submitter.
-        self.prover_sync
+            let submit_args = SubmitMessageArgs::new(message.clone());
-            .write()
+            // Guaranteed to exist as we return early above if it does not.
-            .await
+            let send_channel = self.send_channels.get(&message.destination).unwrap();
-            .update_to_index(message.nonce)
-            .await?;
-
-        debug!(
-            msg_id=?message.id(),
-            msg_nonce=message.nonce,
-            "Sending message to submitter"
-        );
-        // Finally, build the submit arg and dispatch it to the submitter.
-        let submit_args = SubmitMessageArgs::new(message.clone());
-        if let Some(send_channel) = self.send_channels.get(&message.destination) {
             send_channel.send(submit_args)?;
+            self.message_nonce += 1;
         } else {
-            debug!(
+            tokio::time::sleep(Duration::from_secs(1)).await;
-                id=?message.id(),
-                destination=message.destination,
-                nonce=message.nonce,
-                "Message destined for unknown domain, skipping");
         }
-        self.message_nonce += 1;
         Ok(())
     }
 }