From fa672529a758a34490eea2126080f7731e6e9de3 Mon Sep 17 00:00:00 2001 From: Adrian Sutton Date: Tue, 26 Mar 2019 06:20:30 +1000 Subject: [PATCH] Capture metrics on Vertx event loop and worker thread queues. (#1155) Signed-off-by: Adrian Sutton --- .../ethereum/p2p/netty/NettyP2PNetwork.java | 6 ++ .../metrics/vertx/PoolMetricsAdapter.java | 75 +++++++++++++++++++ .../metrics/vertx/VertxMetricsAdapter.java | 33 ++++++++ .../vertx/VertxMetricsAdapterFactory.java | 33 ++++++++ .../pegasys/pantheon/cli/PantheonCommand.java | 16 +++- 5 files changed, 161 insertions(+), 2 deletions(-) create mode 100644 metrics/src/main/java/tech/pegasys/pantheon/metrics/vertx/PoolMetricsAdapter.java create mode 100644 metrics/src/main/java/tech/pegasys/pantheon/metrics/vertx/VertxMetricsAdapter.java create mode 100644 metrics/src/main/java/tech/pegasys/pantheon/metrics/vertx/VertxMetricsAdapterFactory.java diff --git a/ethereum/p2p/src/main/java/tech/pegasys/pantheon/ethereum/p2p/netty/NettyP2PNetwork.java b/ethereum/p2p/src/main/java/tech/pegasys/pantheon/ethereum/p2p/netty/NettyP2PNetwork.java index 98a0ea123b..d6c6f42c4a 100644 --- a/ethereum/p2p/src/main/java/tech/pegasys/pantheon/ethereum/p2p/netty/NettyP2PNetwork.java +++ b/ethereum/p2p/src/main/java/tech/pegasys/pantheon/ethereum/p2p/netty/NettyP2PNetwork.java @@ -256,6 +256,12 @@ public class NettyP2PNetwork implements P2PNetwork { "The number of pending tasks in the Netty boss event loop", pendingTaskCounter(boss)); + metricsSystem.createIntegerGauge( + MetricCategory.NETWORK, + "vertx_eventloop_pending_tasks", + "The number of pending tasks in the Vertx event loop", + pendingTaskCounter(vertx.nettyEventLoopGroup())); + subscribeDisconnect(peerDiscoveryAgent); subscribeDisconnect(peerBlacklist); subscribeDisconnect(connections); diff --git a/metrics/src/main/java/tech/pegasys/pantheon/metrics/vertx/PoolMetricsAdapter.java b/metrics/src/main/java/tech/pegasys/pantheon/metrics/vertx/PoolMetricsAdapter.java new file mode 100644 index 0000000000..9fb778da96 --- /dev/null +++ b/metrics/src/main/java/tech/pegasys/pantheon/metrics/vertx/PoolMetricsAdapter.java @@ -0,0 +1,75 @@ +/* + * Copyright 2019 ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ +package tech.pegasys.pantheon.metrics.vertx; + +import tech.pegasys.pantheon.metrics.Counter; +import tech.pegasys.pantheon.metrics.MetricCategory; +import tech.pegasys.pantheon.metrics.MetricsSystem; + +import io.vertx.core.spi.metrics.PoolMetrics; + +final class PoolMetricsAdapter implements PoolMetrics { + + private final Counter submittedCounter; + private final Counter completedCounter; + private final Counter rejectedCounter; + + public PoolMetricsAdapter( + final MetricsSystem metricsSystem, final String poolType, final String poolName) { + submittedCounter = + metricsSystem + .createLabelledCounter( + MetricCategory.NETWORK, + "vertx_worker_pool_submitted_total", + "Total number of tasks submitted to the Vertx worker pool", + "poolType", + "poolName") + .labels(poolType, poolName); + + completedCounter = + metricsSystem + .createLabelledCounter( + MetricCategory.NETWORK, + "vertx_worker_pool_completed_total", + "Total number of tasks completed by the Vertx worker pool", + "poolType", + "poolName") + .labels(poolType, poolName); + + rejectedCounter = + metricsSystem + .createLabelledCounter( + MetricCategory.NETWORK, + "vertx_worker_pool_rejected_total", + "Total number of tasks rejected by the Vertx worker pool", + "poolType", + "poolName") + .labels(poolType, poolName); + } + + @Override + public Object submitted() { + submittedCounter.inc(); + return null; + } + + @Override + public void rejected(final Object o) { + rejectedCounter.inc(); + } + + @Override + public void end(final Object o, final boolean succeeded) { + completedCounter.inc(); + } +} diff --git a/metrics/src/main/java/tech/pegasys/pantheon/metrics/vertx/VertxMetricsAdapter.java b/metrics/src/main/java/tech/pegasys/pantheon/metrics/vertx/VertxMetricsAdapter.java new file mode 100644 index 0000000000..3d8ef65665 --- /dev/null +++ b/metrics/src/main/java/tech/pegasys/pantheon/metrics/vertx/VertxMetricsAdapter.java @@ -0,0 +1,33 @@ +/* + * Copyright 2019 ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ +package tech.pegasys.pantheon.metrics.vertx; + +import tech.pegasys.pantheon.metrics.MetricsSystem; + +import io.vertx.core.spi.metrics.PoolMetrics; +import io.vertx.core.spi.metrics.VertxMetrics; + +public class VertxMetricsAdapter implements VertxMetrics { + + private final MetricsSystem metricsSystem; + + public VertxMetricsAdapter(final MetricsSystem metricsSystem) { + this.metricsSystem = metricsSystem; + } + + @Override + public PoolMetrics createPoolMetrics( + final String poolType, final String poolName, final int maxPoolSize) { + return new PoolMetricsAdapter(metricsSystem, poolType, poolName); + } +} diff --git a/metrics/src/main/java/tech/pegasys/pantheon/metrics/vertx/VertxMetricsAdapterFactory.java b/metrics/src/main/java/tech/pegasys/pantheon/metrics/vertx/VertxMetricsAdapterFactory.java new file mode 100644 index 0000000000..f815f7d967 --- /dev/null +++ b/metrics/src/main/java/tech/pegasys/pantheon/metrics/vertx/VertxMetricsAdapterFactory.java @@ -0,0 +1,33 @@ +/* + * Copyright 2019 ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ +package tech.pegasys.pantheon.metrics.vertx; + +import tech.pegasys.pantheon.metrics.MetricsSystem; + +import io.vertx.core.VertxOptions; +import io.vertx.core.spi.VertxMetricsFactory; +import io.vertx.core.spi.metrics.VertxMetrics; + +public class VertxMetricsAdapterFactory implements VertxMetricsFactory { + + private final MetricsSystem metricsSystem; + + public VertxMetricsAdapterFactory(final MetricsSystem metricsSystem) { + this.metricsSystem = metricsSystem; + } + + @Override + public VertxMetrics metrics(final VertxOptions options) { + return new VertxMetricsAdapter(metricsSystem); + } +} diff --git a/pantheon/src/main/java/tech/pegasys/pantheon/cli/PantheonCommand.java b/pantheon/src/main/java/tech/pegasys/pantheon/cli/PantheonCommand.java index c683bd0a7d..edcb7f2a6c 100644 --- a/pantheon/src/main/java/tech/pegasys/pantheon/cli/PantheonCommand.java +++ b/pantheon/src/main/java/tech/pegasys/pantheon/cli/PantheonCommand.java @@ -57,6 +57,7 @@ import tech.pegasys.pantheon.metrics.MetricCategory; import tech.pegasys.pantheon.metrics.MetricsSystem; import tech.pegasys.pantheon.metrics.prometheus.MetricsConfiguration; import tech.pegasys.pantheon.metrics.prometheus.PrometheusMetricsSystem; +import tech.pegasys.pantheon.metrics.vertx.VertxMetricsAdapterFactory; import tech.pegasys.pantheon.util.BlockImporter; import tech.pegasys.pantheon.util.InvalidConfigurationException; import tech.pegasys.pantheon.util.PermissioningConfigurationValidator; @@ -84,7 +85,9 @@ import java.util.stream.Stream; import com.google.common.base.Suppliers; import com.google.common.io.Resources; import io.vertx.core.Vertx; +import io.vertx.core.VertxOptions; import io.vertx.core.json.DecodeException; +import io.vertx.core.metrics.MetricsOptions; import org.apache.logging.log4j.Level; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -926,9 +929,10 @@ public class PantheonCommand implements DefaultCommandValues, Runnable { permissioningConfiguration.ifPresent(runnerBuilder::permissioningConfiguration); + final MetricsSystem metricsSystem = this.metricsSystem.get(); final Runner runner = runnerBuilder - .vertx(Vertx.vertx()) + .vertx(Vertx.vertx(createVertxOptions(metricsSystem))) .pantheonController(controller) .p2pEnabled(p2pEnabled) .discovery(peerDiscoveryEnabled) @@ -940,7 +944,7 @@ public class PantheonCommand implements DefaultCommandValues, Runnable { .webSocketConfiguration(webSocketConfiguration) .dataDir(dataDir()) .bannedNodeIds(bannedNodeIds) - .metricsSystem(metricsSystem.get()) + .metricsSystem(metricsSystem) .metricsConfiguration(metricsConfiguration) .staticNodes(staticNodes) .build(); @@ -950,6 +954,14 @@ public class PantheonCommand implements DefaultCommandValues, Runnable { runner.awaitStop(); } + private VertxOptions createVertxOptions(final MetricsSystem metricsSystem) { + return new VertxOptions() + .setMetricsOptions( + new MetricsOptions() + .setEnabled(true) + .setFactory(new VertxMetricsAdapterFactory(metricsSystem))); + } + private void addShutdownHook(final Runner runner) { Runtime.getRuntime() .addShutdownHook(