[rpc] Add rate limit data to metrics (#3727)

* Add rate limit to metrics

* Compliance with harmony's metrics specifications

* Improve rate limit metrics info

* minor: remove the redundancy of rpc metrics name

Signed-off-by: Leo Chen <leo@harmony.one>

* [rpc] set default rate limit to 1000

Signed-off-by: Leo Chen <leo@harmony.one>

* [rpc] fix test errors

Signed-off-by: Leo Chen <leo@harmony.one>

* [metrics] init rpc rate limiter

Signed-off-by: Leo Chen <leo@harmony.one>

Co-authored-by: Leo Chen <leo@harmony.one>

cmd/harmony/flags_test.go

@@ -69,7 +69,7 @@ func TestHarmonyFlags(t *testing.T) {
 				RPCOpt: rpcOptConfig{
 					DebugEnabled:      false,
 					RateLimterEnabled: true,
-					RequestsPerSecond: 300,
+					RequestsPerSecond: 1000,
 				},
 				WS: wsConfig{
 					Enabled: true,
@@ -542,7 +542,7 @@ func TestRPCOptFlags(t *testing.T) {
 			expConfig: rpcOptConfig{
 				DebugEnabled:      true,
 				RateLimterEnabled: true,
-				RequestsPerSecond: 300,
+				RequestsPerSecond: 1000,
 			},
 		},
 
@@ -551,25 +551,25 @@ func TestRPCOptFlags(t *testing.T) {
 			expConfig: rpcOptConfig{
 				DebugEnabled:      false,
 				RateLimterEnabled: true,
-				RequestsPerSecond: 300,
+				RequestsPerSecond: 1000,
 			},
 		},
 
 		{
-			args: []string{"--rpc.ratelimiter", "--rpc.ratelimit", "1000"},
+			args: []string{"--rpc.ratelimiter", "--rpc.ratelimit", "2000"},
 			expConfig: rpcOptConfig{
 				DebugEnabled:      false,
 				RateLimterEnabled: true,
-				RequestsPerSecond: 1000,
+				RequestsPerSecond: 2000,
 			},
 		},
 
 		{
-			args: []string{"--rpc.ratelimiter=false", "--rpc.ratelimit", "1000"},
+			args: []string{"--rpc.ratelimiter=false", "--rpc.ratelimit", "2000"},
 			expConfig: rpcOptConfig{
 				DebugEnabled:      false,
 				RateLimterEnabled: false,
-				RequestsPerSecond: 1000,
+				RequestsPerSecond: 2000,
 			},
 		},
 	}

internal/configs/node/network.go

@@ -58,7 +58,7 @@ const (
 
 const (
 	// DefaultRateLimit for RPC, the number of requests per second
-	DefaultRPCRateLimit = 300
+	DefaultRPCRateLimit = 1000
 )
 
 const (

rpc/blockchain.go

@@ -7,6 +7,8 @@ import (
 	"strconv"
 	"time"
 
+	"github.com/prometheus/client_golang/prometheus"
+
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/common/hexutil"
 	"github.com/ethereum/go-ethereum/rpc"
@@ -46,10 +48,15 @@ const (
 func NewPublicBlockchainAPI(hmy *hmy.Harmony, version Version, limiterEnable bool, limit int) rpc.API {
 	blockCache, _ := lru.New(blockCacheLimit)
 	if limiterEnable {
+		limiter := rate.NewLimiter(rate.Limit(limit), 1)
+		strLimit := fmt.Sprintf("%d", int64(limiter.Limit()))
+		rpcRateLimitCounterVec.With(prometheus.Labels{
+			"rate_limit": strLimit,
+		}).Add(float64(0))
 		return rpc.API{
 			Namespace: version.Namespace(),
 			Version:   APIVersion,
-			Service:   &PublicBlockchainService{hmy, version, rate.NewLimiter(rate.Limit(limit), 1), blockCache},
+			Service:   &PublicBlockchainService{hmy, version, limiter, blockCache},
 			Public:    true,
 		}
 	} else {
@@ -149,6 +156,13 @@ func (s *PublicBlockchainService) wait(ctx context.Context) error {
 	if s.limiter != nil {
 		deadlineCtx, cancel := context.WithTimeout(ctx, DefaultRateLimiterWaitTimeout)
 		defer cancel()
+		if !s.limiter.Allow() {
+			strLimit := fmt.Sprintf("%d", int64(s.limiter.Limit()))
+			rpcRateLimitCounterVec.With(prometheus.Labels{
+				"rate_limit": strLimit,
+			}).Inc()
+		}
+
 		return s.limiter.Wait(deadlineCtx)
 	}
 	return nil

rpc/metrics.go

@@ -0,0 +1,23 @@
+package rpc
+
+import (
+	prom "github.com/harmony-one/harmony/api/service/prometheus"
+	"github.com/prometheus/client_golang/prometheus"
+)
+
+func init() {
+	prom.PromRegistry().MustRegister(
+		rpcRateLimitCounterVec,
+	)
+}
+
+var (
+	rpcRateLimitCounterVec = prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Namespace: "hmy",
+			Subsystem: "rpc",
+			Name:      "over_ratelimit",
+			Help:      "number of times triggered rpc rate limit",
+		},
+		[]string{"rate_limit"})
+)