Improve indexer state response (#74)

* Improve indexer isSynced flag if RPC behind indexer DB

* Change indexer state output format

* Update comment

* Add isStuck flag

mainnet.env.example

@@ -23,6 +23,7 @@ INDEXER_CONTRACTS_IS_ENABLED=1
 INDEXER_CONTRACTS_TYPES=erc20,erc721
 # onewallet total balances indexer, running once a day
 INDEXER_ONEWALLET_IS_ENABLED=1
+INDEXER_IS_SYNCED_THRESHOLD=10
 
 # shard ids synced by indexer comma separated 0,1,2,3
 INDEXER_SHARDS=0

src/config.ts

@@ -94,6 +94,7 @@ export const config = {
     isSyncingLogsEnabled: toBool(process.env.INDEXER_LOGS_IS_ENABLED || '0'),
     isSyncingContractsEnabled: toBool(process.env.INDEXER_CONTRACTS_IS_ENABLED || '0'),
     isSyncingOneWalletEnabled: toBool(process.env.INDEXER_ONEWALLET_IS_ENABLED || '0'),
+    isSyncedThreshold: +(process.env.INDEXER_IS_SYNCED_THRESHOLD || 10),
     trackContractTypes: getCommaSeparatedList(process.env.INDEXER_CONTRACTS_TYPES),
     initialBlockSyncingHeight: +(process.env.INDEXER_INITIAL_BLOCK_SYNCING_HEIGHT || 0),
     // set to the height where smart contracts were introduced on the chain

src/indexer/indexer/BlockIndexer.ts

@@ -18,6 +18,7 @@ import {contractAddressIndexer} from './сontractAddressIndexer'
 import {config} from 'src/config'
 import {internalTransactionsIgnoreListFilter} from './ignoreList/internalTransactionIgnoreList'
 import * as monitorTransfers from './metrics/transfers'
+import * as monitorBlocks from './metrics/blocks'
 
 const approximateBlockMintingTime = 2000
 const maxBatchCount = 100
@@ -269,6 +270,8 @@ export class BlockIndexer {
 
       const failedCount = RPCUrls.getFailedCount(shardID) - failedCountBefore
 
+      monitorBlocks.addBlock(shardID, lastFetchedBlockNumber)
+
       const syncedToBlock = Math.min(
         lastFetchedBlockNumber,
         startBlock + blockRange * this.batchCount

src/indexer/indexer/metrics/blocks.ts

@@ -0,0 +1,26 @@
+import {ShardID} from 'src/types'
+import {config} from 'src/config'
+
+const {shards} = config.indexer
+
+interface BlockMetrics {
+  lastBlockNumber: number
+  lastBlockUpdatedAt: number
+}
+
+const metrics: Record<ShardID, BlockMetrics> = shards.reduce((acc, shardId) => {
+  acc[shardId] = {
+    lastBlockNumber: 0,
+    lastBlockUpdatedAt: 0,
+  }
+  return acc
+}, {} as any)
+
+export const addBlock = (shardId: ShardID, blockNumber: number) => {
+  if (metrics[shardId].lastBlockNumber !== blockNumber) {
+    metrics[shardId].lastBlockNumber = blockNumber
+    metrics[shardId].lastBlockUpdatedAt = Date.now()
+  }
+}
+
+export const getBlocksMetrics = (shardId: ShardID) => metrics[shardId]

src/indexer/server/index.ts

@@ -7,6 +7,7 @@ import * as RPCClient from 'src/indexer/rpc/client'
 import {cache as LRUCache} from 'src/api/controllers/cache'
 import {getLastLogs} from 'src/logger/lastLogs'
 import * as monitorTransfers from 'src/indexer/indexer/metrics/transfers'
+import {getBlocksMetrics} from 'src/indexer/indexer/metrics/blocks'
 
 const l = logger(module)
 
@@ -24,16 +25,22 @@ export const indexerServer = async () => {
   })
 
   api.get('/', async (req, res) => {
-    const shards = config.indexer.shards
+    const {shards, isSyncedThreshold} = config.indexer
 
     const lastSyncedBlocks = await Promise.all(
       shards.map(async (shardID) => {
-        const latestBlockchainBlock = (await RPCClient.getBlockByNumber(shardID, 'latest', false))
-          .number
+        const {lastBlockUpdatedAt} = getBlocksMetrics(shardID)
+        const rpcHeight = (await RPCClient.getBlockByNumber(shardID, 'latest', false)).number
 
-        const blockNumber = await stores[shardID].indexer.getLastIndexedBlockNumber()
-        const isSynced = blockNumber ? latestBlockchainBlock - blockNumber < 10 : false
-        return {shardID, blockNumber, latestBlockchainBlock, isSynced}
+        const indexerHeight = await stores[shardID].indexer.getLastIndexedBlockNumber()
+        let isSynced = false
+        if (indexerHeight) {
+          // isSynced should be "false" if RPC height lower that indexer height
+          const isRpcAheadOfIndexer = rpcHeight >= indexerHeight
+          isSynced = isRpcAheadOfIndexer && rpcHeight - indexerHeight < isSyncedThreshold
+        }
+        const isStuck = lastBlockUpdatedAt > 0 && Date.now() - lastBlockUpdatedAt >= 60 * 1000
+        return {shardID, indexerHeight, rpcHeight, threshold: isSyncedThreshold, isSynced, isStuck}
       })
     )
     const state = {lastSyncedBlocks}