From 4bf9c3e52799795cb88c375b58481ff4a3aaeded Mon Sep 17 00:00:00 2001 From: Jacky Wang Date: Mon, 1 Nov 2021 15:15:20 -0400 Subject: [PATCH] [RPC] General solution of prometheus metrics for all RPC methods (#3919) * [rpc] add method wise prometheus metrics * [misc] remove method APIs for all services (not used anyway) * [misc] remove use of service.APIs * [rpc] fix goimport error, renamed new prometheus rpc --- api/service/blockproposal/service.go | 6 --- api/service/consensus/service.go | 6 --- api/service/explorer/service.go | 6 --- api/service/manager.go | 2 - api/service/pprof/service.go | 6 --- api/service/prometheus/service.go | 6 --- api/service/synchronize/service.go | 6 --- consensus/votepower/roster.go | 2 +- eth/rpc/handler.go | 4 ++ eth/rpc/metrics.go | 68 ++++++++++++++++++++++++++++ node/api.go | 4 -- 11 files changed, 73 insertions(+), 43 deletions(-) create mode 100644 eth/rpc/metrics.go diff --git a/api/service/blockproposal/service.go b/api/service/blockproposal/service.go index 916999625..4d42c558a 100644 --- a/api/service/blockproposal/service.go +++ b/api/service/blockproposal/service.go @@ -3,7 +3,6 @@ package blockproposal import ( msg_pb "github.com/harmony-one/harmony/api/proto/message" "github.com/harmony-one/harmony/consensus" - "github.com/harmony-one/harmony/eth/rpc" "github.com/harmony-one/harmony/internal/utils" ) @@ -43,8 +42,3 @@ func (s *Service) Stop() error { utils.Logger().Info().Msg("Role conversion stopped.") return nil } - -// APIs for the services. -func (s *Service) APIs() []rpc.API { - return nil -} diff --git a/api/service/consensus/service.go b/api/service/consensus/service.go index 6362a5cf7..d48c660a7 100644 --- a/api/service/consensus/service.go +++ b/api/service/consensus/service.go @@ -4,7 +4,6 @@ import ( msg_pb "github.com/harmony-one/harmony/api/proto/message" "github.com/harmony-one/harmony/consensus" "github.com/harmony-one/harmony/core/types" - "github.com/harmony-one/harmony/eth/rpc" "github.com/harmony-one/harmony/internal/utils" ) @@ -41,8 +40,3 @@ func (s *Service) Stop() error { utils.Logger().Info().Msg("Consensus service stopped.") return s.consensus.Close() } - -// APIs for the services. -func (s *Service) APIs() []rpc.API { - return nil -} diff --git a/api/service/explorer/service.go b/api/service/explorer/service.go index dd95e6fcb..849b26fbc 100644 --- a/api/service/explorer/service.go +++ b/api/service/explorer/service.go @@ -17,7 +17,6 @@ import ( "github.com/gorilla/mux" msg_pb "github.com/harmony-one/harmony/api/proto/message" "github.com/harmony-one/harmony/core" - "github.com/harmony-one/harmony/eth/rpc" "github.com/harmony-one/harmony/hmy" "github.com/harmony-one/harmony/internal/chain" nodeconfig "github.com/harmony-one/harmony/internal/configs/node" @@ -294,11 +293,6 @@ func (s *Service) SetMessageChan(messageChan chan *msg_pb.Message) { s.messageChan = messageChan } -// APIs for the services. -func (s *Service) APIs() []rpc.API { - return nil -} - func defaultDBPath(ip, port string) string { return path.Join(nodeconfig.GetDefaultConfig().DBDir, "explorer_storage_"+ip+"_"+port) } diff --git a/api/service/manager.go b/api/service/manager.go index c16c03e93..e12d3f78f 100644 --- a/api/service/manager.go +++ b/api/service/manager.go @@ -3,7 +3,6 @@ package service import ( "fmt" - "github.com/harmony-one/harmony/eth/rpc" "github.com/harmony-one/harmony/internal/utils" "github.com/pkg/errors" "github.com/rs/zerolog" @@ -52,7 +51,6 @@ func (t Type) String() string { type Service interface { Start() error Stop() error - APIs() []rpc.API // the list of RPC descriptors the service provides } // Manager stores all services for service manager. diff --git a/api/service/pprof/service.go b/api/service/pprof/service.go index 9cde42b69..ef56c2987 100644 --- a/api/service/pprof/service.go +++ b/api/service/pprof/service.go @@ -10,7 +10,6 @@ import ( "sync" "time" - "github.com/harmony-one/harmony/eth/rpc" "github.com/harmony-one/harmony/internal/utils" ) @@ -127,11 +126,6 @@ func (s *Service) Stop() error { return nil } -// APIs return all APIs of the service -func (s *Service) APIs() []rpc.API { - return nil -} - // scheduleProfile schedules the provided profile based on the specified interval (e.g. saves the profile every x seconds) func scheduleProfile(profile Profile, dir string) { go func() { diff --git a/api/service/prometheus/service.go b/api/service/prometheus/service.go index 601730be8..3e43bdc0a 100644 --- a/api/service/prometheus/service.go +++ b/api/service/prometheus/service.go @@ -11,7 +11,6 @@ import ( "sync" "time" - "github.com/harmony-one/harmony/eth/rpc" "github.com/harmony-one/harmony/internal/utils" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promhttp" @@ -181,11 +180,6 @@ func (s *Service) Status() error { return nil } -// APIs returns the RPC apis of the prometheus service -func (s *Service) APIs() []rpc.API { - return nil -} - func (s *Service) getRegistry() *prometheus.Registry { s.registryOnce.Do(func() { if svc.registry == nil { diff --git a/api/service/synchronize/service.go b/api/service/synchronize/service.go index 434f0f512..5de6303a0 100644 --- a/api/service/synchronize/service.go +++ b/api/service/synchronize/service.go @@ -2,7 +2,6 @@ package synchronize import ( "github.com/harmony-one/harmony/core" - "github.com/harmony-one/harmony/eth/rpc" "github.com/harmony-one/harmony/hmy/downloader" "github.com/harmony-one/harmony/p2p" ) @@ -30,8 +29,3 @@ func (s *Service) Stop() error { s.Downloaders.Close() return nil } - -// APIs return all APIs of the service -func (s *Service) APIs() []rpc.API { - return nil -} diff --git a/consensus/votepower/roster.go b/consensus/votepower/roster.go index 4f1249c92..4369664cd 100644 --- a/consensus/votepower/roster.go +++ b/consensus/votepower/roster.go @@ -103,7 +103,7 @@ type topLevelRegistry struct { type Roster struct { Voters map[bls.SerializedPublicKey]*AccommodateHarmonyVote topLevelRegistry - ShardID uint32 + ShardID uint32 OrderedSlots []bls.SerializedPublicKey } diff --git a/eth/rpc/handler.go b/eth/rpc/handler.go index a0b89d3cd..a5f92cc6b 100644 --- a/eth/rpc/handler.go +++ b/eth/rpc/handler.go @@ -367,8 +367,12 @@ func (h *handler) handleSubscribe(cp *callProc, msg *jsonrpcMessage) *jsonrpcMes // runMethod runs the Go callback for an RPC method. func (h *handler) runMethod(ctx context.Context, msg *jsonrpcMessage, callb *callback, args []reflect.Value) *jsonrpcMessage { + timer := doMetricRequest(msg.Method) + defer doMetricDelayHist(timer) + result, err := callb.call(ctx, msg.Method, args) if err != nil { + doMetricErroredRequest(msg.Method) return msg.errorResponse(err) } return msg.response(result) diff --git a/eth/rpc/metrics.go b/eth/rpc/metrics.go new file mode 100644 index 000000000..3cd4ddd78 --- /dev/null +++ b/eth/rpc/metrics.go @@ -0,0 +1,68 @@ +package rpc + +import ( + prom "github.com/harmony-one/harmony/api/service/prometheus" + "github.com/prometheus/client_golang/prometheus" +) + +func init() { + prom.PromRegistry().MustRegister( + requestCounterVec, + requestErroredCounterVec, + requestDurationHistVec, + ) +} + +var ( + requestCounterVec = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: "hmy", + Subsystem: "rpc2", + Name: "request_count", + Help: "counters for each RPC method", + }, + []string{"method"}, + ) + + requestErroredCounterVec = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: "hmy", + Subsystem: "rpc2", + Name: "err_count", + Help: "counters of errored RPC method", + }, + []string{"method"}, + ) + + requestDurationHistVec = prometheus.NewHistogramVec( + prometheus.HistogramOpts{ + Namespace: "hmy", + Subsystem: "rpc2", + Name: "delay_histogram", + Help: "delays histogram in seconds", + // buckets: 50ms, 100ms, 200ms, 400ms, 800ms, 1600ms, 3200ms, +INF + Buckets: prometheus.ExponentialBuckets(0.05, 2, 8), + }, + []string{"method"}, + ) +) + +func doMetricRequest(method string) *prometheus.Timer { + pLabel := prometheus.Labels{ + "method": method, + } + requestCounterVec.With(pLabel).Inc() + timer := prometheus.NewTimer(requestDurationHistVec.With(pLabel)) + return timer +} + +func doMetricErroredRequest(method string) { + pLabel := prometheus.Labels{ + "method": method, + } + requestErroredCounterVec.With(pLabel).Inc() +} + +func doMetricDelayHist(timer *prometheus.Timer) { + timer.ObserveDuration() +} diff --git a/node/api.go b/node/api.go index a24931861..903858693 100644 --- a/node/api.go +++ b/node/api.go @@ -70,10 +70,6 @@ func (node *Node) StartRPC() error { // Gather all the possible APIs to surface apis := node.APIs(harmony) - for _, service := range node.serviceManager.GetServices() { - apis = append(apis, service.APIs()...) - } - return hmy_rpc.StartServers(harmony, apis, node.NodeConfig.RPCServer) }