Improve indexer state response (#74)

* Improve indexer isSynced flag if RPC behind indexer DB

* Change indexer state output format

* Update comment

* Add isStuck flag
pull/75/head
Artem 2 years ago committed by GitHub
parent b0dd52b74c
commit b660e84bea
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
  1. 1
      mainnet.env.example
  2. 1
      src/config.ts
  3. 3
      src/indexer/indexer/BlockIndexer.ts
  4. 26
      src/indexer/indexer/metrics/blocks.ts
  5. 19
      src/indexer/server/index.ts

@ -23,6 +23,7 @@ INDEXER_CONTRACTS_IS_ENABLED=1
INDEXER_CONTRACTS_TYPES=erc20,erc721
# onewallet total balances indexer, running once a day
INDEXER_ONEWALLET_IS_ENABLED=1
INDEXER_IS_SYNCED_THRESHOLD=10
# shard ids synced by indexer comma separated 0,1,2,3
INDEXER_SHARDS=0

@ -94,6 +94,7 @@ export const config = {
isSyncingLogsEnabled: toBool(process.env.INDEXER_LOGS_IS_ENABLED || '0'),
isSyncingContractsEnabled: toBool(process.env.INDEXER_CONTRACTS_IS_ENABLED || '0'),
isSyncingOneWalletEnabled: toBool(process.env.INDEXER_ONEWALLET_IS_ENABLED || '0'),
isSyncedThreshold: +(process.env.INDEXER_IS_SYNCED_THRESHOLD || 10),
trackContractTypes: getCommaSeparatedList(process.env.INDEXER_CONTRACTS_TYPES),
initialBlockSyncingHeight: +(process.env.INDEXER_INITIAL_BLOCK_SYNCING_HEIGHT || 0),
// set to the height where smart contracts were introduced on the chain

@ -18,6 +18,7 @@ import {contractAddressIndexer} from './сontractAddressIndexer'
import {config} from 'src/config'
import {internalTransactionsIgnoreListFilter} from './ignoreList/internalTransactionIgnoreList'
import * as monitorTransfers from './metrics/transfers'
import * as monitorBlocks from './metrics/blocks'
const approximateBlockMintingTime = 2000
const maxBatchCount = 100
@ -269,6 +270,8 @@ export class BlockIndexer {
const failedCount = RPCUrls.getFailedCount(shardID) - failedCountBefore
monitorBlocks.addBlock(shardID, lastFetchedBlockNumber)
const syncedToBlock = Math.min(
lastFetchedBlockNumber,
startBlock + blockRange * this.batchCount

@ -0,0 +1,26 @@
import {ShardID} from 'src/types'
import {config} from 'src/config'
const {shards} = config.indexer
interface BlockMetrics {
lastBlockNumber: number
lastBlockUpdatedAt: number
}
const metrics: Record<ShardID, BlockMetrics> = shards.reduce((acc, shardId) => {
acc[shardId] = {
lastBlockNumber: 0,
lastBlockUpdatedAt: 0,
}
return acc
}, {} as any)
export const addBlock = (shardId: ShardID, blockNumber: number) => {
if (metrics[shardId].lastBlockNumber !== blockNumber) {
metrics[shardId].lastBlockNumber = blockNumber
metrics[shardId].lastBlockUpdatedAt = Date.now()
}
}
export const getBlocksMetrics = (shardId: ShardID) => metrics[shardId]

@ -7,6 +7,7 @@ import * as RPCClient from 'src/indexer/rpc/client'
import {cache as LRUCache} from 'src/api/controllers/cache'
import {getLastLogs} from 'src/logger/lastLogs'
import * as monitorTransfers from 'src/indexer/indexer/metrics/transfers'
import {getBlocksMetrics} from 'src/indexer/indexer/metrics/blocks'
const l = logger(module)
@ -24,16 +25,22 @@ export const indexerServer = async () => {
})
api.get('/', async (req, res) => {
const shards = config.indexer.shards
const {shards, isSyncedThreshold} = config.indexer
const lastSyncedBlocks = await Promise.all(
shards.map(async (shardID) => {
const latestBlockchainBlock = (await RPCClient.getBlockByNumber(shardID, 'latest', false))
.number
const {lastBlockUpdatedAt} = getBlocksMetrics(shardID)
const rpcHeight = (await RPCClient.getBlockByNumber(shardID, 'latest', false)).number
const blockNumber = await stores[shardID].indexer.getLastIndexedBlockNumber()
const isSynced = blockNumber ? latestBlockchainBlock - blockNumber < 10 : false
return {shardID, blockNumber, latestBlockchainBlock, isSynced}
const indexerHeight = await stores[shardID].indexer.getLastIndexedBlockNumber()
let isSynced = false
if (indexerHeight) {
// isSynced should be "false" if RPC height lower that indexer height
const isRpcAheadOfIndexer = rpcHeight >= indexerHeight
isSynced = isRpcAheadOfIndexer && rpcHeight - indexerHeight < isSyncedThreshold
}
const isStuck = lastBlockUpdatedAt > 0 && Date.now() - lastBlockUpdatedAt >= 60 * 1000
return {shardID, indexerHeight, rpcHeight, threshold: isSyncedThreshold, isSynced, isStuck}
})
)
const state = {lastSyncedBlocks}

Loading…
Cancel
Save