Capture metrics on Vertx event loop and worker thread queues. (#1155)

Signed-off-by: Adrian Sutton <adrian.sutton@consensys.net>

ethereum/p2p/src/main/java/tech/pegasys/pantheon/ethereum/p2p/netty/NettyP2PNetwork.java

@@ -256,6 +256,12 @@ public class NettyP2PNetwork implements P2PNetwork {
         "The number of pending tasks in the Netty boss event loop",
         pendingTaskCounter(boss));
 
+    metricsSystem.createIntegerGauge(
+        MetricCategory.NETWORK,
+        "vertx_eventloop_pending_tasks",
+        "The number of pending tasks in the Vertx event loop",
+        pendingTaskCounter(vertx.nettyEventLoopGroup()));
+
     subscribeDisconnect(peerDiscoveryAgent);
     subscribeDisconnect(peerBlacklist);
     subscribeDisconnect(connections);

metrics/src/main/java/tech/pegasys/pantheon/metrics/vertx/PoolMetricsAdapter.java

@@ -0,0 +1,75 @@
+/*
+ * Copyright 2019 ConsenSys AG.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+ * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations under the License.
+ */
+package tech.pegasys.pantheon.metrics.vertx;
+
+import tech.pegasys.pantheon.metrics.Counter;
+import tech.pegasys.pantheon.metrics.MetricCategory;
+import tech.pegasys.pantheon.metrics.MetricsSystem;
+
+import io.vertx.core.spi.metrics.PoolMetrics;
+
+final class PoolMetricsAdapter implements PoolMetrics<Object> {
+
+  private final Counter submittedCounter;
+  private final Counter completedCounter;
+  private final Counter rejectedCounter;
+
+  public PoolMetricsAdapter(
+      final MetricsSystem metricsSystem, final String poolType, final String poolName) {
+    submittedCounter =
+        metricsSystem
+            .createLabelledCounter(
+                MetricCategory.NETWORK,
+                "vertx_worker_pool_submitted_total",
+                "Total number of tasks submitted to the Vertx worker pool",
+                "poolType",
+                "poolName")
+            .labels(poolType, poolName);
+
+    completedCounter =
+        metricsSystem
+            .createLabelledCounter(
+                MetricCategory.NETWORK,
+                "vertx_worker_pool_completed_total",
+                "Total number of tasks completed by the Vertx worker pool",
+                "poolType",
+                "poolName")
+            .labels(poolType, poolName);
+
+    rejectedCounter =
+        metricsSystem
+            .createLabelledCounter(
+                MetricCategory.NETWORK,
+                "vertx_worker_pool_rejected_total",
+                "Total number of tasks rejected by the Vertx worker pool",
+                "poolType",
+                "poolName")
+            .labels(poolType, poolName);
+  }
+
+  @Override
+  public Object submitted() {
+    submittedCounter.inc();
+    return null;
+  }
+
+  @Override
+  public void rejected(final Object o) {
+    rejectedCounter.inc();
+  }
+
+  @Override
+  public void end(final Object o, final boolean succeeded) {
+    completedCounter.inc();
+  }
+}

metrics/src/main/java/tech/pegasys/pantheon/metrics/vertx/VertxMetricsAdapter.java

@@ -0,0 +1,33 @@
+/*
+ * Copyright 2019 ConsenSys AG.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+ * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations under the License.
+ */
+package tech.pegasys.pantheon.metrics.vertx;
+
+import tech.pegasys.pantheon.metrics.MetricsSystem;
+
+import io.vertx.core.spi.metrics.PoolMetrics;
+import io.vertx.core.spi.metrics.VertxMetrics;
+
+public class VertxMetricsAdapter implements VertxMetrics {
+
+  private final MetricsSystem metricsSystem;
+
+  public VertxMetricsAdapter(final MetricsSystem metricsSystem) {
+    this.metricsSystem = metricsSystem;
+  }
+
+  @Override
+  public PoolMetrics<?> createPoolMetrics(
+      final String poolType, final String poolName, final int maxPoolSize) {
+    return new PoolMetricsAdapter(metricsSystem, poolType, poolName);
+  }
+}

metrics/src/main/java/tech/pegasys/pantheon/metrics/vertx/VertxMetricsAdapterFactory.java

@@ -0,0 +1,33 @@
+/*
+ * Copyright 2019 ConsenSys AG.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+ * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations under the License.
+ */
+package tech.pegasys.pantheon.metrics.vertx;
+
+import tech.pegasys.pantheon.metrics.MetricsSystem;
+
+import io.vertx.core.VertxOptions;
+import io.vertx.core.spi.VertxMetricsFactory;
+import io.vertx.core.spi.metrics.VertxMetrics;
+
+public class VertxMetricsAdapterFactory implements VertxMetricsFactory {
+
+  private final MetricsSystem metricsSystem;
+
+  public VertxMetricsAdapterFactory(final MetricsSystem metricsSystem) {
+    this.metricsSystem = metricsSystem;
+  }
+
+  @Override
+  public VertxMetrics metrics(final VertxOptions options) {
+    return new VertxMetricsAdapter(metricsSystem);
+  }
+}

pantheon/src/main/java/tech/pegasys/pantheon/cli/PantheonCommand.java

@@ -57,6 +57,7 @@ import tech.pegasys.pantheon.metrics.MetricCategory;
 import tech.pegasys.pantheon.metrics.MetricsSystem;
 import tech.pegasys.pantheon.metrics.prometheus.MetricsConfiguration;
 import tech.pegasys.pantheon.metrics.prometheus.PrometheusMetricsSystem;
+import tech.pegasys.pantheon.metrics.vertx.VertxMetricsAdapterFactory;
 import tech.pegasys.pantheon.util.BlockImporter;
 import tech.pegasys.pantheon.util.InvalidConfigurationException;
 import tech.pegasys.pantheon.util.PermissioningConfigurationValidator;
@@ -84,7 +85,9 @@ import java.util.stream.Stream;
 import com.google.common.base.Suppliers;
 import com.google.common.io.Resources;
 import io.vertx.core.Vertx;
+import io.vertx.core.VertxOptions;
 import io.vertx.core.json.DecodeException;
+import io.vertx.core.metrics.MetricsOptions;
 import org.apache.logging.log4j.Level;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
@@ -926,9 +929,10 @@ public class PantheonCommand implements DefaultCommandValues, Runnable {
 
     permissioningConfiguration.ifPresent(runnerBuilder::permissioningConfiguration);
 
+    final MetricsSystem metricsSystem = this.metricsSystem.get();
     final Runner runner =
         runnerBuilder
-            .vertx(Vertx.vertx())
+            .vertx(Vertx.vertx(createVertxOptions(metricsSystem)))
             .pantheonController(controller)
             .p2pEnabled(p2pEnabled)
             .discovery(peerDiscoveryEnabled)
@@ -940,7 +944,7 @@ public class PantheonCommand implements DefaultCommandValues, Runnable {
             .webSocketConfiguration(webSocketConfiguration)
             .dataDir(dataDir())
             .bannedNodeIds(bannedNodeIds)
-            .metricsSystem(metricsSystem.get())
+            .metricsSystem(metricsSystem)
             .metricsConfiguration(metricsConfiguration)
             .staticNodes(staticNodes)
             .build();
@@ -950,6 +954,14 @@ public class PantheonCommand implements DefaultCommandValues, Runnable {
     runner.awaitStop();
   }
 
+  private VertxOptions createVertxOptions(final MetricsSystem metricsSystem) {
+    return new VertxOptions()
+        .setMetricsOptions(
+            new MetricsOptions()
+                .setEnabled(true)
+                .setFactory(new VertxMetricsAdapterFactory(metricsSystem)));
+  }
+
   private void addShutdownHook(final Runner runner) {
     Runtime.getRuntime()
         .addShutdownHook(