[prometheus] add consensus metrics

* counter of consensus
* finality histogram
* number of signatures
* number of started view change
* number of finished view change
* counter of state syncing

Signed-off-by: Leo Chen <leo@harmony.one>

consensus/consensus_service.go

@@ -513,6 +513,7 @@ func (consensus *Consensus) StartFinalityCount() {
 func (consensus *Consensus) FinishFinalityCount() {
 	d := time.Now().UnixNano()
 	consensus.finality = (d - consensus.finalityCounter) / 1000000
+	ConsensusFinalityHistogram.Observe(float64(consensus.finality))
 }
 
 // GetFinality returns the finality time in milliseconds of previous consensus

consensus/consensus_v2.go

@@ -22,6 +22,7 @@ import (
 	"github.com/harmony-one/harmony/shard"
 	"github.com/harmony-one/vdf/src/vdf_go"
 	"github.com/pkg/errors"
+	"github.com/prometheus/client_golang/prometheus"
 )
 
 var (
@@ -211,6 +212,8 @@ func (consensus *Consensus) finalCommit() {
 		Int("numTxns", len(block.Transactions())).
 		Int("numStakingTxns", len(block.StakingTransactions())).
 		Msg("HOORAY!!!!!!! CONSENSUS REACHED!!!!!!!")
+	ConsensusCounterVec.With(prometheus.Labels{"consensus": "hooray"}).Inc()
+	ConsensusGaugeVec.With(prometheus.Labels{"consensus": "num_commits"}).Set(float64(consensus.Decider.SignersCount(quorum.Commit)))
 
 	// If still the leader, send commit sig/bitmap to finish the new block proposal,
 	// else, the block proposal will timeout by itself.
@@ -326,6 +329,7 @@ func (consensus *Consensus) Start(
 					consensus.getLogger().Info().Msg("[syncReadyChan] Start consensus timer")
 					consensus.consensusTimeout[timeoutConsensus].Start()
 					consensus.getLogger().Info().Str("Mode", mode.String()).Msg("Node is IN SYNC")
+					ConsensusSyncCounterVec.With(prometheus.Labels{"consensus": "in_sync"}).Inc()
 				}
 				consensus.mutex.Unlock()
 
@@ -334,6 +338,7 @@ func (consensus *Consensus) Start(
 				consensus.SetBlockNum(consensus.Blockchain.CurrentHeader().Number().Uint64() + 1)
 				consensus.current.SetMode(Syncing)
 				consensus.getLogger().Info().Msg("[ConsensusMainLoop] Node is OUT OF SYNC")
+				ConsensusSyncCounterVec.With(prometheus.Labels{"consensus": "out_of_sync"}).Inc()
 
 			case newBlock := <-blockChannel:
 				consensus.getLogger().Info().
@@ -625,9 +630,9 @@ func (consensus *Consensus) commitBlock(blk *types.Block, committedMsg *FBFTMess
 		return errIncorrectSender
 	}
 
+	consensus.FinishFinalityCount()
 	consensus.PostConsensusJob(blk)
 	consensus.SetupForNewConsensus(blk, committedMsg)
-	consensus.FinishFinalityCount()
 	utils.Logger().Info().Uint64("blockNum", blk.NumberU64()).
 		Str("hash", blk.Header().Hash().Hex()).
 		Msg("Added New Block to Blockchain!!!")

consensus/metrics.go

@@ -0,0 +1,69 @@
+package consensus
+
+import (
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promauto"
+)
+
+var (
+	// ConsensusCounterVec is used to keep track of consensus reached
+	ConsensusCounterVec = promauto.NewCounterVec(
+		prometheus.CounterOpts{
+			Namespace: "hmy",
+			Subsystem: "consensus",
+			Name:      "bingo",
+			Help:      "counter of consensus",
+		},
+		[]string{
+			"consensus",
+		},
+	)
+	// ConsensusVCCounterVec is used to keep track of number of view change
+	ConsensusVCCounterVec = promauto.NewCounterVec(
+		prometheus.CounterOpts{
+			Namespace: "hmy",
+			Subsystem: "consensus",
+			Name:      "viewchange",
+			Help:      "counter of view chagne",
+		},
+		[]string{
+			"viewchange",
+		},
+	)
+	// ConsensusSyncCounterVec is used to keep track of consensus syncing state
+	ConsensusSyncCounterVec = promauto.NewCounterVec(
+		prometheus.CounterOpts{
+			Namespace: "hmy",
+			Subsystem: "consensus",
+			Name:      "sync",
+			Help:      "counter of blockchain syncing state",
+		},
+		[]string{
+			"consensus",
+		},
+	)
+	// ConsensusGaugeVec is used to keep track of gauge number of the consensus
+	ConsensusGaugeVec = promauto.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Namespace: "hmy",
+			Subsystem: "consensus",
+			Name:      "signatures",
+			Help:      "number of signatures or commits",
+		},
+		[]string{
+			"consensus",
+		},
+	)
+	// ConsensusFinalityHistogram is used to keep track of finality
+	// 10 ExponentialBuckets are in the unit of millisecond:
+	// 800, 1000, 1250, 1562, 1953, 2441, 3051, 3814, 4768, 5960, inf
+	ConsensusFinalityHistogram = promauto.NewHistogram(
+		prometheus.HistogramOpts{
+			Namespace: "hmy",
+			Subsystem: "consensus",
+			Name:      "finality",
+			Help:      "the latency of the finality",
+			Buckets:   prometheus.ExponentialBuckets(800, 1.25, 10),
+		},
+	)
+)

consensus/validator.go

@@ -13,6 +13,7 @@ import (
 	"github.com/harmony-one/harmony/consensus/signature"
 	"github.com/harmony-one/harmony/core/types"
 	"github.com/harmony-one/harmony/p2p"
+	"github.com/prometheus/client_golang/prometheus"
 )
 
 func (consensus *Consensus) onAnnounce(msg *msg_pb.Message) {
@@ -29,6 +30,7 @@ func (consensus *Consensus) onAnnounce(msg *msg_pb.Message) {
 	if !consensus.onAnnounceSanityChecks(recvMsg) {
 		return
 	}
+	consensus.StartFinalityCount()
 
 	consensus.getLogger().Debug().
 		Uint64("MsgViewID", recvMsg.ViewID).
@@ -55,7 +57,6 @@ func (consensus *Consensus) onAnnounce(msg *msg_pb.Message) {
 		}
 		return
 	}
-	consensus.StartFinalityCount()
 	consensus.prepare()
 }
 
@@ -293,6 +294,12 @@ func (consensus *Consensus) onCommitted(msg *msg_pb.Message) {
 		}
 	}
 
+	ConsensusCounterVec.With(prometheus.Labels{"consensus": "bingo"}).Inc()
+	numSig := float64(consensus.NumSignaturesIncludedInBlock(blockObj))
+
+	ConsensusGaugeVec.With(prometheus.Labels{"consensus": "signatures"}).Set(numSig)
+	ConsensusCounterVec.With(prometheus.Labels{"consensus": "signatures"}).Add(numSig)
+
 	initBn := consensus.blockNum
 	consensus.tryCatchup()
 	if recvMsg.BlockNum > consensus.blockNum {

consensus/view_change.go

@@ -15,6 +15,7 @@ import (
 	"github.com/harmony-one/harmony/p2p"
 	"github.com/harmony-one/harmony/shard"
 	"github.com/pkg/errors"
+	"github.com/prometheus/client_golang/prometheus"
 )
 
 // MaxViewIDDiff limits the received view ID to only 249 further from the current view ID
@@ -240,6 +241,7 @@ func (consensus *Consensus) startViewChange() {
 		Dur("timeoutDuration", duration).
 		Str("NextLeader", consensus.LeaderPubKey.Bytes.Hex()).
 		Msg("[startViewChange]")
+	ConsensusVCCounterVec.With(prometheus.Labels{"viewchange": "started"}).Inc()
 
 	consensus.consensusTimeout[timeoutViewChange].SetDuration(duration)
 	defer consensus.consensusTimeout[timeoutViewChange].Start()
@@ -544,6 +546,7 @@ func (consensus *Consensus) onNewView(msg *msg_pb.Message) {
 		Str("newLeaderKey", consensus.LeaderPubKey.Bytes.Hex()).
 		Msg("new leader changed")
 	consensus.consensusTimeout[timeoutConsensus].Start()
+	ConsensusVCCounterVec.With(prometheus.Labels{"viewchange": "finished"}).Inc()
 }
 
 // ResetViewChangeState resets the view change structure