[metrics] Add NewGaugeVec metric to record actual duration for RPCs (#3827)

* [metrics] Add rpc_request_delay_gauge

* [metrics] Adjust buckets for rpcRequestDurationVec
pull/3837/head
MathxH Chen 3 years ago committed by GitHub
parent af7e0dee71
commit 5222a4109e
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
  1. 31
      rpc/blockchain.go
  2. 6
      rpc/contract.go
  3. 6
      rpc/filters/api.go
  4. 21
      rpc/metrics.go
  5. 2
      rpc/net.go
  6. 16
      rpc/pool.go
  7. 16
      rpc/staking.go
  8. 10
      rpc/tracer.go
  9. 8
      rpc/transaction.go

@ -176,8 +176,7 @@ func (s *PublicBlockchainService) GetBlockByNumber(
ctx context.Context, blockNumber BlockNumber, opts interface{},
) (response StructuredResponse, err error) {
timer := DoMetricRPCRequest(GetBlockByNumber)
defer timer.ObserveDuration()
defer DoRPCRequestDuration(GetBlockByNumber, timer)
// Process arguments based on version
var blockArgs *rpc_common.BlockArgs
blockArgs, ok := opts.(*rpc_common.BlockArgs)
@ -272,7 +271,7 @@ func (s *PublicBlockchainService) GetBlockByHash(
ctx context.Context, blockHash common.Hash, opts interface{},
) (response StructuredResponse, err error) {
timer := DoMetricRPCRequest(GetBlockByHash)
defer timer.ObserveDuration()
defer DoRPCRequestDuration(GetBlockByHash, timer)
err = s.wait(ctx)
if err != nil {
@ -330,7 +329,7 @@ func (s *PublicBlockchainService) GetBlockByNumberNew(
ctx context.Context, blockNum BlockNumber, blockArgs *rpc_common.BlockArgs,
) (StructuredResponse, error) {
timer := DoMetricRPCRequest(GetBlockByNumberNew)
defer timer.ObserveDuration()
defer DoRPCRequestDuration(GetBlockByNumberNew, timer)
res, err := s.GetBlockByNumber(ctx, blockNum, blockArgs)
if err != nil {
@ -344,7 +343,7 @@ func (s *PublicBlockchainService) GetBlockByHashNew(
ctx context.Context, blockHash common.Hash, blockArgs *rpc_common.BlockArgs,
) (StructuredResponse, error) {
timer := DoMetricRPCRequest(GetBlockByHashNew)
defer timer.ObserveDuration()
defer DoRPCRequestDuration(GetBlockByHashNew, timer)
res, err := s.GetBlockByHash(ctx, blockHash, blockArgs)
if err != nil {
@ -359,7 +358,7 @@ func (s *PublicBlockchainService) GetBlocks(
blockNumberEnd BlockNumber, blockArgs *rpc_common.BlockArgs,
) ([]StructuredResponse, error) {
timer := DoMetricRPCRequest(GetBlocks)
defer timer.ObserveDuration()
defer DoRPCRequestDuration(GetBlocks, timer)
blockStart := blockNumberStart.Int64()
blockEnd := blockNumberEnd.Int64()
@ -584,7 +583,7 @@ func (s *PublicBlockchainService) GetShardingStructure(
ctx context.Context,
) ([]StructuredResponse, error) {
timer := DoMetricRPCRequest(GetShardingStructure)
defer timer.ObserveDuration()
defer DoRPCRequestDuration(GetShardingStructure, timer)
err := s.wait(ctx)
if err != nil {
@ -611,7 +610,7 @@ func (s *PublicBlockchainService) GetBalanceByBlockNumber(
ctx context.Context, address string, blockNumber BlockNumber,
) (interface{}, error) {
timer := DoMetricRPCRequest(GetBalanceByBlockNumber)
defer timer.ObserveDuration()
defer DoRPCRequestDuration(GetBalanceByBlockNumber, timer)
// Process number based on version
blockNum := blockNumber.EthBlockNumber()
@ -641,7 +640,7 @@ func (s *PublicBlockchainService) GetBalanceByBlockNumber(
// LatestHeader returns the latest header information
func (s *PublicBlockchainService) LatestHeader(ctx context.Context) (StructuredResponse, error) {
timer := DoMetricRPCRequest(LatestHeader)
defer timer.ObserveDuration()
defer DoRPCRequestDuration(LatestHeader, timer)
err := s.wait(ctx)
if err != nil {
@ -667,7 +666,7 @@ func (s *PublicBlockchainService) GetLatestChainHeaders(
) (StructuredResponse, error) {
// Response output is the same for all versions
timer := DoMetricRPCRequest(GetLatestChainHeaders)
defer timer.ObserveDuration()
defer DoRPCRequestDuration(GetLatestChainHeaders, timer)
return NewStructuredResponse(s.hmy.GetLatestChainHeaders())
}
@ -676,7 +675,7 @@ func (s *PublicBlockchainService) GetLastCrossLinks(
ctx context.Context,
) ([]StructuredResponse, error) {
timer := DoMetricRPCRequest(GetLastCrossLinks)
defer timer.ObserveDuration()
defer DoRPCRequestDuration(GetLastCrossLinks, timer)
err := s.wait(ctx)
if err != nil {
@ -714,7 +713,7 @@ func (s *PublicBlockchainService) GetHeaderByNumber(
ctx context.Context, blockNumber BlockNumber,
) (StructuredResponse, error) {
timer := DoMetricRPCRequest(GetHeaderByNumber)
defer timer.ObserveDuration()
defer DoRPCRequestDuration(GetHeaderByNumber, timer)
err := s.wait(ctx)
if err != nil {
@ -746,7 +745,7 @@ func (s *PublicBlockchainService) GetCurrentUtilityMetrics(
ctx context.Context,
) (StructuredResponse, error) {
timer := DoMetricRPCRequest(GetCurrentUtilityMetrics)
defer timer.ObserveDuration()
defer DoRPCRequestDuration(GetCurrentUtilityMetrics, timer)
err := s.wait(ctx)
if err != nil {
@ -775,7 +774,7 @@ func (s *PublicBlockchainService) GetSuperCommittees(
ctx context.Context,
) (StructuredResponse, error) {
timer := DoMetricRPCRequest(GetSuperCommittees)
defer timer.ObserveDuration()
defer DoRPCRequestDuration(GetSuperCommittees, timer)
err := s.wait(ctx)
if err != nil {
@ -804,7 +803,7 @@ func (s *PublicBlockchainService) GetCurrentBadBlocks(
ctx context.Context,
) ([]StructuredResponse, error) {
timer := DoMetricRPCRequest(GetCurrentBadBlocks)
defer timer.ObserveDuration()
defer DoRPCRequestDuration(GetCurrentBadBlocks, timer)
err := s.wait(ctx)
if err != nil {
@ -846,7 +845,7 @@ func (s *PublicBlockchainService) GetStakingNetworkInfo(
ctx context.Context,
) (StructuredResponse, error) {
timer := DoMetricRPCRequest(GetStakingNetworkInfo)
defer timer.ObserveDuration()
defer DoRPCRequestDuration(GetStakingNetworkInfo, timer)
err := s.wait(ctx)
if err != nil {

@ -61,7 +61,7 @@ func (s *PublicContractService) GetCode(
ctx context.Context, addr string, blockNumber BlockNumber,
) (hexutil.Bytes, error) {
timer := DoMetricRPCRequest(GetCode)
defer timer.ObserveDuration()
defer DoRPCRequestDuration(GetCode, timer)
// Process number based on version
blockNum := blockNumber.EthBlockNumber()
@ -90,7 +90,7 @@ func (s *PublicContractService) GetStorageAt(
ctx context.Context, addr string, key string, blockNumber BlockNumber,
) (hexutil.Bytes, error) {
timer := DoMetricRPCRequest(GetStorageAt)
defer timer.ObserveDuration()
defer DoRPCRequestDuration(GetStorageAt, timer)
// Process number based on version
blockNum := blockNumber.EthBlockNumber()
@ -117,7 +117,7 @@ func DoEVMCall(
timeout time.Duration,
) (core.ExecutionResult, error) {
timer := DoMetricRPCRequest(DoEvmCall)
defer timer.ObserveDuration()
defer DoRPCRequestDuration(DoEvmCall, timer)
defer func(start time.Time) {
utils.Logger().Debug().

@ -366,7 +366,7 @@ func (api *PublicFilterAPI) NewFilter(crit FilterCriteria) (rpc.ID, error) {
// https://github.com/ethereum/wiki/wiki/JSON-RPC#eth_getlogs
func (api *PublicFilterAPI) GetLogs(ctx context.Context, crit FilterCriteria) ([]*types.Log, error) {
timer := hmy_rpc.DoMetricRPCRequest(hmy_rpc.GetLogs)
defer timer.ObserveDuration()
defer hmy_rpc.DoRPCRequestDuration(hmy_rpc.GetLogs, timer)
var filter *Filter
if crit.BlockHash != nil {
@ -403,7 +403,7 @@ func (api *PublicFilterAPI) GetLogs(ctx context.Context, crit FilterCriteria) ([
// https://github.com/ethereum/wiki/wiki/JSON-RPC#eth_uninstallfilter
func (api *PublicFilterAPI) UninstallFilter(id rpc.ID) bool {
timer := hmy_rpc.DoMetricRPCRequest(hmy_rpc.UninstallFilter)
defer timer.ObserveDuration()
defer hmy_rpc.DoRPCRequestDuration(hmy_rpc.UninstallFilter, timer)
api.filtersMu.Lock()
f, found := api.filters[id]
@ -424,7 +424,7 @@ func (api *PublicFilterAPI) UninstallFilter(id rpc.ID) bool {
// https://github.com/ethereum/wiki/wiki/JSON-RPC#eth_getfilterlogs
func (api *PublicFilterAPI) GetFilterLogs(ctx context.Context, id rpc.ID) ([]*types.Log, error) {
timer := hmy_rpc.DoMetricRPCRequest(hmy_rpc.GetFilterLogs)
defer timer.ObserveDuration()
defer hmy_rpc.DoRPCRequestDuration(hmy_rpc.GetFilterLogs, timer)
api.filtersMu.Lock()
f, found := api.filters[id]

@ -82,6 +82,7 @@ func init() {
rpcRateLimitCounterVec,
rpcQueryInfoCounterVec,
rpcRequestDurationVec,
rpcRequestDurationGaugeVec,
)
}
@ -110,13 +111,22 @@ var (
prometheus.HistogramOpts{
Namespace: "hmy",
Subsystem: "rpc",
Name: "rpc_request_delay",
Name: "request_delay_histogram",
Help: "delay in seconds to do rpc requests",
// buckets: 20ms, 40ms, 80ms, 160ms, 320ms, 640ms, 1280ms, +INF
Buckets: prometheus.ExponentialBuckets(0.02, 2, 8),
// buckets: 50ms, 100ms, 200ms, 400ms, 800ms, 1600ms, 3200ms, +INF
Buckets: prometheus.ExponentialBuckets(0.05, 2, 8),
},
[]string{"rpc_name"},
)
rpcRequestDurationGaugeVec = prometheus.NewGaugeVec(prometheus.GaugeOpts{
Namespace: "hmy",
Subsystem: "rpc",
Name: "request_delay_gauge",
Help: "delay in seconds to do rpc requests",
},
[]string{"rpc_name"},
)
)
func DoMetricRPCRequest(rpcName string) *prometheus.Timer {
@ -126,6 +136,11 @@ func DoMetricRPCRequest(rpcName string) *prometheus.Timer {
return timer
}
func DoRPCRequestDuration(rpcName string, timer *prometheus.Timer) {
pLabel := getRPCDurationPromLabel(rpcName)
rpcRequestDurationGaugeVec.With(pLabel).Set(timer.ObserveDuration().Seconds())
}
func DoMetricRPCQueryInfo(rpcName string, infoType string) {
pLabel := getRPCQueryInfoPromLabel(rpcName, infoType)
rpcQueryInfoCounterVec.With(pLabel).Inc()

@ -46,7 +46,7 @@ func NewPublicNetAPI(net p2p.Host, chainID uint64, version Version) rpc.API {
// Note that the return type is an interface to account for the different versions
func (s *PublicNetService) PeerCount(ctx context.Context) (interface{}, error) {
timer := DoMetricRPCRequest(PeerCount)
defer timer.ObserveDuration()
defer DoRPCRequestDuration(PeerCount, timer)
// Format response according to version
switch s.version {
case V1, Eth:

@ -45,7 +45,7 @@ func (s *PublicPoolService) SendRawTransaction(
ctx context.Context, encodedTx hexutil.Bytes,
) (common.Hash, error) {
timer := DoMetricRPCRequest(SendRawTransaction)
defer timer.ObserveDuration()
defer DoRPCRequestDuration(SendRawTransaction, timer)
// DOS prevention
if len(encodedTx) >= types.MaxEncodedPoolTransactionSize {
@ -132,7 +132,7 @@ func (s *PublicPoolService) SendRawStakingTransaction(
ctx context.Context, encodedTx hexutil.Bytes,
) (common.Hash, error) {
timer := DoMetricRPCRequest(SendRawStakingTransaction)
defer timer.ObserveDuration()
defer DoRPCRequestDuration(SendRawStakingTransaction, timer)
// DOS prevention
if len(encodedTx) >= types.MaxEncodedPoolTransactionSize {
@ -172,7 +172,7 @@ func (s *PublicPoolService) GetPoolStats(
ctx context.Context,
) (StructuredResponse, error) {
timer := DoMetricRPCRequest(GetPoolStats)
defer timer.ObserveDuration()
defer DoRPCRequestDuration(GetPoolStats, timer)
pendingCount, queuedCount := s.hmy.GetPoolStats()
@ -188,7 +188,7 @@ func (s *PublicPoolService) PendingTransactions(
ctx context.Context,
) ([]StructuredResponse, error) {
timer := DoMetricRPCRequest(PendingTransactions)
defer timer.ObserveDuration()
defer DoRPCRequestDuration(PendingTransactions, timer)
// Fetch all pending transactions (stx & plain tx)
pending, err := s.hmy.GetPoolTransactions()
@ -253,7 +253,7 @@ func (s *PublicPoolService) PendingStakingTransactions(
ctx context.Context,
) ([]StructuredResponse, error) {
timer := DoMetricRPCRequest(PendingStakingTransactions)
defer timer.ObserveDuration()
defer DoRPCRequestDuration(PendingStakingTransactions, timer)
// Fetch all pending transactions (stx & plain tx)
pending, err := s.hmy.GetPoolTransactions()
@ -310,7 +310,7 @@ func (s *PublicPoolService) GetCurrentTransactionErrorSink(
ctx context.Context,
) ([]StructuredResponse, error) {
timer := DoMetricRPCRequest(GetCurrentTransactionErrorSink)
defer timer.ObserveDuration()
defer DoRPCRequestDuration(GetCurrentTransactionErrorSink, timer)
// For each transaction error in the error sink, format the response (same format for all versions)
formattedErrors := []StructuredResponse{}
@ -330,7 +330,7 @@ func (s *PublicPoolService) GetCurrentStakingErrorSink(
ctx context.Context,
) ([]StructuredResponse, error) {
timer := DoMetricRPCRequest(GetCurrentStakingErrorSink)
defer timer.ObserveDuration()
defer DoRPCRequestDuration(GetCurrentStakingErrorSink, timer)
// For each staking tx error in the error sink, format the response (same format for all versions)
formattedErrors := []StructuredResponse{}
@ -350,7 +350,7 @@ func (s *PublicPoolService) GetPendingCXReceipts(
ctx context.Context,
) ([]StructuredResponse, error) {
timer := DoMetricRPCRequest(GetPendingCXReceipts)
defer timer.ObserveDuration()
defer DoRPCRequestDuration(GetPendingCXReceipts, timer)
// For each cx receipt, format the response (same format for all versions)
formattedReceipts := []StructuredResponse{}

@ -243,7 +243,7 @@ func (s *PublicStakingService) GetAllValidatorInformation(
ctx context.Context, page int,
) ([]StructuredResponse, error) {
timer := DoMetricRPCRequest(GetAllValidatorInformation)
defer timer.ObserveDuration()
defer DoRPCRequestDuration(GetAllValidatorInformation, timer)
if !isBeaconShard(s.hmy) {
DoMetricRPCQueryInfo(GetAllValidatorInformation, FailedNumber)
@ -280,7 +280,7 @@ func (s *PublicStakingService) GetAllValidatorInformationByBlockNumber(
ctx context.Context, page int, blockNumber BlockNumber,
) ([]StructuredResponse, error) {
timer := DoMetricRPCRequest(GetAllValidatorInformationByBlockNumber)
defer timer.ObserveDuration()
defer DoRPCRequestDuration(GetAllValidatorInformationByBlockNumber, timer)
// Process number based on version
blockNum := blockNumber.EthBlockNumber()
@ -303,7 +303,7 @@ func (s *PublicStakingService) GetValidatorInformation(
ctx context.Context, address string,
) (StructuredResponse, error) {
timer := DoMetricRPCRequest(GetValidatorInformation)
defer timer.ObserveDuration()
defer DoRPCRequestDuration(GetValidatorInformation, timer)
if !isBeaconShard(s.hmy) {
DoMetricRPCQueryInfo(GetValidatorInformation, FailedNumber)
@ -338,7 +338,7 @@ func (s *PublicStakingService) GetValidatorInformationByBlockNumber(
ctx context.Context, address string, blockNumber BlockNumber,
) (StructuredResponse, error) {
timer := DoMetricRPCRequest(GetValidatorInformationByBlockNumber)
defer timer.ObserveDuration()
defer DoRPCRequestDuration(GetValidatorInformationByBlockNumber, timer)
// Process number based on version
blockNum := blockNumber.EthBlockNumber()
@ -440,7 +440,7 @@ func (s *PublicStakingService) GetAllDelegationInformation(
ctx context.Context, page int,
) ([][]StructuredResponse, error) {
timer := DoMetricRPCRequest(GetAllDelegationInformation)
defer timer.ObserveDuration()
defer DoRPCRequestDuration(GetAllDelegationInformation, timer)
if !isBeaconShard(s.hmy) {
DoMetricRPCQueryInfo(GetAllDelegationInformation, FailedNumber)
@ -489,7 +489,7 @@ func (s *PublicStakingService) GetDelegationsByDelegator(
ctx context.Context, address string,
) ([]StructuredResponse, error) {
timer := DoMetricRPCRequest(GetDelegationsByDelegator)
defer timer.ObserveDuration()
defer DoRPCRequestDuration(GetDelegationsByDelegator, timer)
if !isBeaconShard(s.hmy) {
DoMetricRPCQueryInfo(GetDelegationsByDelegator, FailedNumber)
@ -611,7 +611,7 @@ func (s *PublicStakingService) GetDelegationsByValidator(
ctx context.Context, address string,
) ([]StructuredResponse, error) {
timer := DoMetricRPCRequest(GetDelegationsByValidator)
defer timer.ObserveDuration()
defer DoRPCRequestDuration(GetDelegationsByValidator, timer)
if !isBeaconShard(s.hmy) {
DoMetricRPCQueryInfo(GetDelegationsByValidator, FailedNumber)
@ -667,7 +667,7 @@ func (s *PublicStakingService) GetDelegationByDelegatorAndValidator(
ctx context.Context, address string, validator string,
) (StructuredResponse, error) {
timer := DoMetricRPCRequest(GetDelegationByDelegatorAndValidator)
defer timer.ObserveDuration()
defer DoRPCRequestDuration(GetDelegationByDelegatorAndValidator, timer)
if !isBeaconShard(s.hmy) {
DoMetricRPCQueryInfo(GetDelegationByDelegatorAndValidator, FailedNumber)

@ -99,7 +99,7 @@ func (s *PublicTracerService) TraceChain(ctx context.Context, start, end rpc.Blo
// EVM and returns them as a JSON object.
func (s *PublicTracerService) TraceBlockByNumber(ctx context.Context, number rpc.BlockNumber, config *hmy.TraceConfig) ([]*hmy.TxTraceResult, error) {
timer := DoMetricRPCRequest(TraceBlockByNumber)
defer timer.ObserveDuration()
defer DoRPCRequestDuration(TraceBlockByNumber, timer)
// Fetch the block that we want to trace
block := s.hmy.BlockChain.GetBlockByNumber(uint64(number))
@ -111,7 +111,7 @@ func (s *PublicTracerService) TraceBlockByNumber(ctx context.Context, number rpc
// EVM and returns them as a JSON object.
func (s *PublicTracerService) TraceBlockByHash(ctx context.Context, hash common.Hash, config *hmy.TraceConfig) ([]*hmy.TxTraceResult, error) {
timer := DoMetricRPCRequest(TraceBlockByHash)
defer timer.ObserveDuration()
defer DoRPCRequestDuration(TraceBlockByHash, timer)
block := s.hmy.BlockChain.GetBlockByHash(hash)
if block == nil {
@ -125,7 +125,7 @@ func (s *PublicTracerService) TraceBlockByHash(ctx context.Context, hash common.
// and returns them as a JSON object.
func (s *PublicTracerService) TraceBlock(ctx context.Context, blob []byte, config *hmy.TraceConfig) ([]*hmy.TxTraceResult, error) {
timer := DoMetricRPCRequest(TraceBlock)
defer timer.ObserveDuration()
defer DoRPCRequestDuration(TraceBlock, timer)
block := new(types.Block)
if err := rlp.Decode(bytes.NewReader(blob), block); err != nil {
@ -139,7 +139,7 @@ func (s *PublicTracerService) TraceBlock(ctx context.Context, blob []byte, confi
// and returns them as a JSON object.
func (s *PublicTracerService) TraceTransaction(ctx context.Context, hash common.Hash, config *hmy.TraceConfig) (interface{}, error) {
timer := DoMetricRPCRequest(TraceTransaction)
defer timer.ObserveDuration()
defer DoRPCRequestDuration(TraceTransaction, timer)
// Retrieve the transaction and assemble its EVM context
tx, blockHash, _, index := rawdb.ReadTransaction(s.hmy.ChainDb(), hash)
@ -172,7 +172,7 @@ func (s *PublicTracerService) TraceTransaction(ctx context.Context, hash common.
// NOTE: Our version only supports block number as an input
func (s *PublicTracerService) TraceCall(ctx context.Context, args CallArgs, blockNr rpc.BlockNumber, config *hmy.TraceConfig) (interface{}, error) {
timer := DoMetricRPCRequest(TraceCall)
defer timer.ObserveDuration()
defer DoRPCRequestDuration(TraceCall, timer)
// First try to retrieve the state
statedb, header, err := s.hmy.StateAndHeaderByNumber(ctx, blockNr)

@ -169,7 +169,7 @@ func (s *PublicTransactionService) GetTransactionByHash(
ctx context.Context, hash common.Hash,
) (StructuredResponse, error) {
timer := DoMetricRPCRequest(GetTransactionByHash)
defer timer.ObserveDuration()
defer DoRPCRequestDuration(GetTransactionByHash, timer)
// Try to return an already finalized transaction
tx, blockHash, blockNumber, index := rawdb.ReadTransaction(s.hmy.ChainDb(), hash)
if tx == nil {
@ -223,7 +223,7 @@ func (s *PublicTransactionService) GetStakingTransactionByHash(
ctx context.Context, hash common.Hash,
) (StructuredResponse, error) {
timer := DoMetricRPCRequest(GetStakingTransactionByHash)
defer timer.ObserveDuration()
defer DoRPCRequestDuration(GetStakingTransactionByHash, timer)
// Try to return an already finalized transaction
stx, blockHash, blockNumber, index := rawdb.ReadStakingTransaction(s.hmy.ChainDb(), hash)
@ -270,7 +270,7 @@ func (s *PublicTransactionService) GetTransactionsHistory(
ctx context.Context, args TxHistoryArgs,
) (StructuredResponse, error) {
timer := DoMetricRPCRequest(GetTransactionsHistory)
defer timer.ObserveDuration()
defer DoRPCRequestDuration(GetTransactionsHistory, timer)
// Fetch transaction history
var address string
var result []common.Hash
@ -323,7 +323,7 @@ func (s *PublicTransactionService) GetStakingTransactionsHistory(
ctx context.Context, args TxHistoryArgs,
) (StructuredResponse, error) {
timer := DoMetricRPCRequest(GetStakingTransactionsHistory)
defer timer.ObserveDuration()
defer DoRPCRequestDuration(GetStakingTransactionsHistory, timer)
// Fetch transaction history
var address string

Loading…
Cancel
Save