diff --git a/consensus/consensus_service.go b/consensus/consensus_service.go index 8532c6545..29a5de544 100644 --- a/consensus/consensus_service.go +++ b/consensus/consensus_service.go @@ -513,6 +513,7 @@ func (consensus *Consensus) StartFinalityCount() { func (consensus *Consensus) FinishFinalityCount() { d := time.Now().UnixNano() consensus.finality = (d - consensus.finalityCounter) / 1000000 + ConsensusFinalityHistogram.Observe(float64(consensus.finality)) } // GetFinality returns the finality time in milliseconds of previous consensus diff --git a/consensus/consensus_v2.go b/consensus/consensus_v2.go index 18faaf9de..6c4a19f8f 100644 --- a/consensus/consensus_v2.go +++ b/consensus/consensus_v2.go @@ -22,6 +22,7 @@ import ( "github.com/harmony-one/harmony/shard" "github.com/harmony-one/vdf/src/vdf_go" "github.com/pkg/errors" + "github.com/prometheus/client_golang/prometheus" ) var ( @@ -211,6 +212,8 @@ func (consensus *Consensus) finalCommit() { Int("numTxns", len(block.Transactions())). Int("numStakingTxns", len(block.StakingTransactions())). Msg("HOORAY!!!!!!! CONSENSUS REACHED!!!!!!!") + ConsensusCounterVec.With(prometheus.Labels{"consensus": "hooray"}).Inc() + ConsensusGaugeVec.With(prometheus.Labels{"consensus": "num_commits"}).Set(float64(consensus.Decider.SignersCount(quorum.Commit))) // If still the leader, send commit sig/bitmap to finish the new block proposal, // else, the block proposal will timeout by itself. @@ -326,6 +329,7 @@ func (consensus *Consensus) Start( consensus.getLogger().Info().Msg("[syncReadyChan] Start consensus timer") consensus.consensusTimeout[timeoutConsensus].Start() consensus.getLogger().Info().Str("Mode", mode.String()).Msg("Node is IN SYNC") + ConsensusSyncCounterVec.With(prometheus.Labels{"consensus": "in_sync"}).Inc() } consensus.mutex.Unlock() @@ -334,6 +338,7 @@ func (consensus *Consensus) Start( consensus.SetBlockNum(consensus.Blockchain.CurrentHeader().Number().Uint64() + 1) consensus.current.SetMode(Syncing) consensus.getLogger().Info().Msg("[ConsensusMainLoop] Node is OUT OF SYNC") + ConsensusSyncCounterVec.With(prometheus.Labels{"consensus": "out_of_sync"}).Inc() case newBlock := <-blockChannel: consensus.getLogger().Info(). @@ -625,9 +630,9 @@ func (consensus *Consensus) commitBlock(blk *types.Block, committedMsg *FBFTMess return errIncorrectSender } + consensus.FinishFinalityCount() consensus.PostConsensusJob(blk) consensus.SetupForNewConsensus(blk, committedMsg) - consensus.FinishFinalityCount() utils.Logger().Info().Uint64("blockNum", blk.NumberU64()). Str("hash", blk.Header().Hash().Hex()). Msg("Added New Block to Blockchain!!!") diff --git a/consensus/metrics.go b/consensus/metrics.go new file mode 100644 index 000000000..1b108a0c2 --- /dev/null +++ b/consensus/metrics.go @@ -0,0 +1,69 @@ +package consensus + +import ( + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" +) + +var ( + // ConsensusCounterVec is used to keep track of consensus reached + ConsensusCounterVec = promauto.NewCounterVec( + prometheus.CounterOpts{ + Namespace: "hmy", + Subsystem: "consensus", + Name: "bingo", + Help: "counter of consensus", + }, + []string{ + "consensus", + }, + ) + // ConsensusVCCounterVec is used to keep track of number of view change + ConsensusVCCounterVec = promauto.NewCounterVec( + prometheus.CounterOpts{ + Namespace: "hmy", + Subsystem: "consensus", + Name: "viewchange", + Help: "counter of view chagne", + }, + []string{ + "viewchange", + }, + ) + // ConsensusSyncCounterVec is used to keep track of consensus syncing state + ConsensusSyncCounterVec = promauto.NewCounterVec( + prometheus.CounterOpts{ + Namespace: "hmy", + Subsystem: "consensus", + Name: "sync", + Help: "counter of blockchain syncing state", + }, + []string{ + "consensus", + }, + ) + // ConsensusGaugeVec is used to keep track of gauge number of the consensus + ConsensusGaugeVec = promauto.NewGaugeVec( + prometheus.GaugeOpts{ + Namespace: "hmy", + Subsystem: "consensus", + Name: "signatures", + Help: "number of signatures or commits", + }, + []string{ + "consensus", + }, + ) + // ConsensusFinalityHistogram is used to keep track of finality + // 10 ExponentialBuckets are in the unit of millisecond: + // 800, 1000, 1250, 1562, 1953, 2441, 3051, 3814, 4768, 5960, inf + ConsensusFinalityHistogram = promauto.NewHistogram( + prometheus.HistogramOpts{ + Namespace: "hmy", + Subsystem: "consensus", + Name: "finality", + Help: "the latency of the finality", + Buckets: prometheus.ExponentialBuckets(800, 1.25, 10), + }, + ) +) diff --git a/consensus/validator.go b/consensus/validator.go index 3bd0566cb..820676fdf 100644 --- a/consensus/validator.go +++ b/consensus/validator.go @@ -13,6 +13,7 @@ import ( "github.com/harmony-one/harmony/consensus/signature" "github.com/harmony-one/harmony/core/types" "github.com/harmony-one/harmony/p2p" + "github.com/prometheus/client_golang/prometheus" ) func (consensus *Consensus) onAnnounce(msg *msg_pb.Message) { @@ -29,6 +30,7 @@ func (consensus *Consensus) onAnnounce(msg *msg_pb.Message) { if !consensus.onAnnounceSanityChecks(recvMsg) { return } + consensus.StartFinalityCount() consensus.getLogger().Debug(). Uint64("MsgViewID", recvMsg.ViewID). @@ -55,7 +57,6 @@ func (consensus *Consensus) onAnnounce(msg *msg_pb.Message) { } return } - consensus.StartFinalityCount() consensus.prepare() } @@ -293,6 +294,12 @@ func (consensus *Consensus) onCommitted(msg *msg_pb.Message) { } } + ConsensusCounterVec.With(prometheus.Labels{"consensus": "bingo"}).Inc() + numSig := float64(consensus.NumSignaturesIncludedInBlock(blockObj)) + + ConsensusGaugeVec.With(prometheus.Labels{"consensus": "signatures"}).Set(numSig) + ConsensusCounterVec.With(prometheus.Labels{"consensus": "signatures"}).Add(numSig) + initBn := consensus.blockNum consensus.tryCatchup() if recvMsg.BlockNum > consensus.blockNum { diff --git a/consensus/view_change.go b/consensus/view_change.go index 80011a842..e15529131 100644 --- a/consensus/view_change.go +++ b/consensus/view_change.go @@ -15,6 +15,7 @@ import ( "github.com/harmony-one/harmony/p2p" "github.com/harmony-one/harmony/shard" "github.com/pkg/errors" + "github.com/prometheus/client_golang/prometheus" ) // MaxViewIDDiff limits the received view ID to only 249 further from the current view ID @@ -240,6 +241,7 @@ func (consensus *Consensus) startViewChange() { Dur("timeoutDuration", duration). Str("NextLeader", consensus.LeaderPubKey.Bytes.Hex()). Msg("[startViewChange]") + ConsensusVCCounterVec.With(prometheus.Labels{"viewchange": "started"}).Inc() consensus.consensusTimeout[timeoutViewChange].SetDuration(duration) defer consensus.consensusTimeout[timeoutViewChange].Start() @@ -544,6 +546,7 @@ func (consensus *Consensus) onNewView(msg *msg_pb.Message) { Str("newLeaderKey", consensus.LeaderPubKey.Bytes.Hex()). Msg("new leader changed") consensus.consensusTimeout[timeoutConsensus].Start() + ConsensusVCCounterVec.With(prometheus.Labels{"viewchange": "finished"}).Inc() } // ResetViewChangeState resets the view change structure