[RPC] General solution of prometheus metrics for all RPC methods (#3919)

* [rpc] add method wise prometheus metrics

* [misc] remove method APIs for all services (not used anyway)

* [misc] remove use of service.APIs

* [rpc] fix goimport error, renamed new prometheus rpc

api/service/blockproposal/service.go

@@ -3,7 +3,6 @@ package blockproposal
 import (
 	msg_pb "github.com/harmony-one/harmony/api/proto/message"
 	"github.com/harmony-one/harmony/consensus"
-	"github.com/harmony-one/harmony/eth/rpc"
 	"github.com/harmony-one/harmony/internal/utils"
 )
 
@@ -43,8 +42,3 @@ func (s *Service) Stop() error {
 	utils.Logger().Info().Msg("Role conversion stopped.")
 	return nil
 }
-
-// APIs for the services.
-func (s *Service) APIs() []rpc.API {
-	return nil
-}

api/service/consensus/service.go

@@ -4,7 +4,6 @@ import (
 	msg_pb "github.com/harmony-one/harmony/api/proto/message"
 	"github.com/harmony-one/harmony/consensus"
 	"github.com/harmony-one/harmony/core/types"
-	"github.com/harmony-one/harmony/eth/rpc"
 	"github.com/harmony-one/harmony/internal/utils"
 )
 
@@ -41,8 +40,3 @@ func (s *Service) Stop() error {
 	utils.Logger().Info().Msg("Consensus service stopped.")
 	return s.consensus.Close()
 }
-
-// APIs for the services.
-func (s *Service) APIs() []rpc.API {
-	return nil
-}

api/service/explorer/service.go

@@ -17,7 +17,6 @@ import (
 	"github.com/gorilla/mux"
 	msg_pb "github.com/harmony-one/harmony/api/proto/message"
 	"github.com/harmony-one/harmony/core"
-	"github.com/harmony-one/harmony/eth/rpc"
 	"github.com/harmony-one/harmony/hmy"
 	"github.com/harmony-one/harmony/internal/chain"
 	nodeconfig "github.com/harmony-one/harmony/internal/configs/node"
@@ -294,11 +293,6 @@ func (s *Service) SetMessageChan(messageChan chan *msg_pb.Message) {
 	s.messageChan = messageChan
 }
 
-// APIs for the services.
-func (s *Service) APIs() []rpc.API {
-	return nil
-}
-
 func defaultDBPath(ip, port string) string {
 	return path.Join(nodeconfig.GetDefaultConfig().DBDir, "explorer_storage_"+ip+"_"+port)
 }

api/service/manager.go

@@ -3,7 +3,6 @@ package service
 import (
 	"fmt"
 
-	"github.com/harmony-one/harmony/eth/rpc"
 	"github.com/harmony-one/harmony/internal/utils"
 	"github.com/pkg/errors"
 	"github.com/rs/zerolog"
@@ -52,7 +51,6 @@ func (t Type) String() string {
 type Service interface {
 	Start() error
 	Stop() error
-	APIs() []rpc.API // the list of RPC descriptors the service provides
 }
 
 // Manager stores all services for service manager.

api/service/pprof/service.go

@@ -10,7 +10,6 @@ import (
 	"sync"
 	"time"
 
-	"github.com/harmony-one/harmony/eth/rpc"
 	"github.com/harmony-one/harmony/internal/utils"
 )
 
@@ -127,11 +126,6 @@ func (s *Service) Stop() error {
 	return nil
 }
 
-// APIs return all APIs of the service
-func (s *Service) APIs() []rpc.API {
-	return nil
-}
-
 // scheduleProfile schedules the provided profile based on the specified interval (e.g. saves the profile every x seconds)
 func scheduleProfile(profile Profile, dir string) {
 	go func() {

api/service/prometheus/service.go

@@ -11,7 +11,6 @@ import (
 	"sync"
 	"time"
 
-	"github.com/harmony-one/harmony/eth/rpc"
 	"github.com/harmony-one/harmony/internal/utils"
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/client_golang/prometheus/promhttp"
@@ -181,11 +180,6 @@ func (s *Service) Status() error {
 	return nil
 }
 
-// APIs returns the RPC apis of the prometheus service
-func (s *Service) APIs() []rpc.API {
-	return nil
-}
-
 func (s *Service) getRegistry() *prometheus.Registry {
 	s.registryOnce.Do(func() {
 		if svc.registry == nil {

api/service/synchronize/service.go

@@ -2,7 +2,6 @@ package synchronize
 
 import (
 	"github.com/harmony-one/harmony/core"
-	"github.com/harmony-one/harmony/eth/rpc"
 	"github.com/harmony-one/harmony/hmy/downloader"
 	"github.com/harmony-one/harmony/p2p"
 )
@@ -30,8 +29,3 @@ func (s *Service) Stop() error {
 	s.Downloaders.Close()
 	return nil
 }
-
-// APIs return all APIs of the service
-func (s *Service) APIs() []rpc.API {
-	return nil
-}

eth/rpc/handler.go

@@ -367,8 +367,12 @@ func (h *handler) handleSubscribe(cp *callProc, msg *jsonrpcMessage) *jsonrpcMes
 
 // runMethod runs the Go callback for an RPC method.
 func (h *handler) runMethod(ctx context.Context, msg *jsonrpcMessage, callb *callback, args []reflect.Value) *jsonrpcMessage {
+	timer := doMetricRequest(msg.Method)
+	defer doMetricDelayHist(timer)
+
 	result, err := callb.call(ctx, msg.Method, args)
 	if err != nil {
+		doMetricErroredRequest(msg.Method)
 		return msg.errorResponse(err)
 	}
 	return msg.response(result)

eth/rpc/metrics.go

@@ -0,0 +1,68 @@
+package rpc
+
+import (
+	prom "github.com/harmony-one/harmony/api/service/prometheus"
+	"github.com/prometheus/client_golang/prometheus"
+)
+
+func init() {
+	prom.PromRegistry().MustRegister(
+		requestCounterVec,
+		requestErroredCounterVec,
+		requestDurationHistVec,
+	)
+}
+
+var (
+	requestCounterVec = prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Namespace: "hmy",
+			Subsystem: "rpc2",
+			Name:      "request_count",
+			Help:      "counters for each RPC method",
+		},
+		[]string{"method"},
+	)
+
+	requestErroredCounterVec = prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Namespace: "hmy",
+			Subsystem: "rpc2",
+			Name:      "err_count",
+			Help:      "counters of errored RPC method",
+		},
+		[]string{"method"},
+	)
+
+	requestDurationHistVec = prometheus.NewHistogramVec(
+		prometheus.HistogramOpts{
+			Namespace: "hmy",
+			Subsystem: "rpc2",
+			Name:      "delay_histogram",
+			Help:      "delays histogram in seconds",
+			// buckets: 50ms, 100ms, 200ms, 400ms, 800ms, 1600ms, 3200ms, +INF
+			Buckets: prometheus.ExponentialBuckets(0.05, 2, 8),
+		},
+		[]string{"method"},
+	)
+)
+
+func doMetricRequest(method string) *prometheus.Timer {
+	pLabel := prometheus.Labels{
+		"method": method,
+	}
+	requestCounterVec.With(pLabel).Inc()
+	timer := prometheus.NewTimer(requestDurationHistVec.With(pLabel))
+	return timer
+}
+
+func doMetricErroredRequest(method string) {
+	pLabel := prometheus.Labels{
+		"method": method,
+	}
+	requestErroredCounterVec.With(pLabel).Inc()
+}
+
+func doMetricDelayHist(timer *prometheus.Timer) {
+	timer.ObserveDuration()
+}

node/api.go

@@ -70,10 +70,6 @@ func (node *Node) StartRPC() error {
 	// Gather all the possible APIs to surface
 	apis := node.APIs(harmony)
 
-	for _, service := range node.serviceManager.GetServices() {
-		apis = append(apis, service.APIs()...)
-	}
-
 	return hmy_rpc.StartServers(harmony, apis, node.NodeConfig.RPCServer)
 }