Add RPC errors metric (#6919)

* Add RPC errors metric.
This metric has two labels : RPC method and RPC error type.

Signed-off-by: Ameziane H <ameziane.hamlat@consensys.net>

* Spotless.

Signed-off-by: Ameziane H <ameziane.hamlat@consensys.net>

* Add changelog.

Signed-off-by: Ameziane H <ameziane.hamlat@consensys.net>

---------

Signed-off-by: Ameziane H <ameziane.hamlat@consensys.net>
Signed-off-by: Sally MacFarlane <macfarla.github@gmail.com>
Co-authored-by: Sally MacFarlane <macfarla.github@gmail.com>

CHANGELOG.md

@@ -39,6 +39,7 @@
 - Log detailed timing of block creation steps [#6880](https://github.com/hyperledger/besu/pull/6880)
 - Expose transaction count by type metrics for the layered txpool [#6903](https://github.com/hyperledger/besu/pull/6903)
 - Expose bad block events via the BesuEvents plugin API  [#6848](https://github.com/hyperledger/besu/pull/6848)
+- Add RPC errors metric [#6919](https://github.com/hyperledger/besu/pull/6919/)
 
 ### Bug fixes
 - Fix txpool dump/restore race condition [#6665](https://github.com/hyperledger/besu/pull/6665)

ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/EngineJsonRpcService.java

@@ -149,6 +149,8 @@ public class EngineJsonRpcService {
   private final HealthService livenessService;
   private final HealthService readinessService;
 
+  private final MetricsSystem metricsSystem;
+
   /**
    * Construct a EngineJsonRpcService to handle either http or websocket clients
    *
@@ -214,6 +216,7 @@ public class EngineJsonRpcService {
     this.livenessService = livenessService;
     this.readinessService = readinessService;
     this.maxActiveConnections = config.getMaxActiveConnections();
+    this.metricsSystem = metricsSystem;
   }
 
   public CompletableFuture<Void> start() {
@@ -451,7 +454,8 @@ public class EngineJsonRpcService {
               new JsonRpcExecutor(
                   new AuthenticatedJsonRpcProcessor(
                       new TimedJsonRpcProcessor(
-                          new TracedJsonRpcProcessor(new BaseJsonRpcProcessor()), requestTimer),
+                          new TracedJsonRpcProcessor(new BaseJsonRpcProcessor(), metricsSystem),
+                          requestTimer),
                       authenticationService.get(),
                       config.getNoAuthRpcApis()),
                   rpcMethods),
@@ -463,7 +467,8 @@ public class EngineJsonRpcService {
           HandlerFactory.jsonRpcExecutor(
               new JsonRpcExecutor(
                   new TimedJsonRpcProcessor(
-                      new TracedJsonRpcProcessor(new BaseJsonRpcProcessor()), requestTimer),
+                      new TracedJsonRpcProcessor(new BaseJsonRpcProcessor(), metricsSystem),
+                      requestTimer),
                   rpcMethods),
               tracer,
               config),

ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/JsonRpcHttpService.java

@@ -134,6 +134,7 @@ public class JsonRpcHttpService {
   private HttpServer httpServer;
   private final HealthService livenessService;
   private final HealthService readinessService;
+  private final MetricsSystem metricsSystem;
 
   /**
    * Construct a JsonRpcHttpService handler
@@ -204,6 +205,7 @@ public class JsonRpcHttpService {
     if (metricsSystem instanceof OpenTelemetrySystem) {
       this.tracerProvider = ((OpenTelemetrySystem) metricsSystem).getTracerProvider();
     }
+    this.metricsSystem = metricsSystem;
   }
 
   private void validateConfig(final JsonRpcConfiguration config) {
@@ -344,7 +346,8 @@ public class JsonRpcHttpService {
               new JsonRpcExecutor(
                   new AuthenticatedJsonRpcProcessor(
                       new TimedJsonRpcProcessor(
-                          new TracedJsonRpcProcessor(new BaseJsonRpcProcessor()), requestTimer),
+                          new TracedJsonRpcProcessor(new BaseJsonRpcProcessor(), metricsSystem),
+                          requestTimer),
                       authenticationService.get(),
                       config.getNoAuthRpcApis()),
                   rpcMethods),
@@ -356,7 +359,8 @@ public class JsonRpcHttpService {
           HandlerFactory.jsonRpcExecutor(
               new JsonRpcExecutor(
                   new TimedJsonRpcProcessor(
-                      new TracedJsonRpcProcessor(new BaseJsonRpcProcessor()), requestTimer),
+                      new TracedJsonRpcProcessor(new BaseJsonRpcProcessor(), metricsSystem),
+                      requestTimer),
                   rpcMethods),
               tracer,
               config),

ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/execution/TracedJsonRpcProcessor.java

@@ -20,6 +20,10 @@ import org.hyperledger.besu.ethereum.api.jsonrpc.internal.methods.JsonRpcMethod;
 import org.hyperledger.besu.ethereum.api.jsonrpc.internal.response.JsonRpcErrorResponse;
 import org.hyperledger.besu.ethereum.api.jsonrpc.internal.response.JsonRpcResponse;
 import org.hyperledger.besu.ethereum.api.jsonrpc.internal.response.JsonRpcResponseType;
+import org.hyperledger.besu.metrics.BesuMetricCategory;
+import org.hyperledger.besu.plugin.services.MetricsSystem;
+import org.hyperledger.besu.plugin.services.metrics.Counter;
+import org.hyperledger.besu.plugin.services.metrics.LabelledMetric;
 
 import io.opentelemetry.api.trace.Span;
 import io.opentelemetry.api.trace.StatusCode;
@@ -27,9 +31,18 @@ import io.opentelemetry.api.trace.StatusCode;
 public class TracedJsonRpcProcessor implements JsonRpcProcessor {
 
   private final JsonRpcProcessor rpcProcessor;
+  protected final LabelledMetric<Counter> rpcErrorsCounter;
 
-  public TracedJsonRpcProcessor(final JsonRpcProcessor rpcProcessor) {
+  public TracedJsonRpcProcessor(
+      final JsonRpcProcessor rpcProcessor, final MetricsSystem metricsSystem) {
     this.rpcProcessor = rpcProcessor;
+    this.rpcErrorsCounter =
+        metricsSystem.createLabelledCounter(
+            BesuMetricCategory.RPC,
+            "errors_count",
+            "Number of errors per RPC method and RPC error type",
+            "rpcMethod",
+            "errorType");
   }
 
   @Override
@@ -41,6 +54,7 @@ public class TracedJsonRpcProcessor implements JsonRpcProcessor {
     JsonRpcResponse jsonRpcResponse = rpcProcessor.process(id, method, metricSpan, request);
     if (JsonRpcResponseType.ERROR == jsonRpcResponse.getType()) {
       JsonRpcErrorResponse errorResponse = (JsonRpcErrorResponse) jsonRpcResponse;
+      this.rpcErrorsCounter.labels(method.getName(), errorResponse.getErrorType().name()).inc();
       switch (errorResponse.getErrorType()) {
         case INVALID_PARAMS:
           metricSpan.setStatus(StatusCode.ERROR, "Invalid Params");

ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/EthCall.java

@@ -119,6 +119,7 @@ public class EthCall extends AbstractBlockParameterOrBlockHashMethod {
     } else {
       final TransactionProcessingResult resultTrx = result.result();
       if (resultTrx != null && resultTrx.getRevertReason().isPresent()) {
+
         return errorResponse(
             request,
             new JsonRpcError(