From 3d867532deb893fd267fcd5d4e6bf0d42a76e59b Mon Sep 17 00:00:00 2001 From: Trent Mohay <37158202+rain-on@users.noreply.github.com> Date: Fri, 15 Nov 2019 08:47:22 +1100 Subject: [PATCH] Repaired Metrics name collision between Privacy and RocksDB (#187) It was identified that when both KV_ROCKSDB metrics and Privacy were enabled, that Besu failed to start due to a naming collision in Metrics Collators - this was ultimately due to Besu having 2 keyvalue stores - one for public state, and another for private state - and _both_ using the same metrics. To overcome this issue, the metrics used in the private kv store are prefixed with the word "private". Signed-off-by: Trent Mohay --- .../acceptance/dsl/privacy/PrivacyNode.java | 4 +- .../org/hyperledger/besu/PrivacyTest.java | 4 +- .../java/org/hyperledger/besu/RunnerTest.java | 4 +- .../operations/OperationBenchmarkHelper.java | 4 +- .../WorldStateDownloaderBenchmark.java | 4 +- .../besu/metrics/BesuMetricCategory.java | 12 +- .../besu/metrics/rocksdb/RocksDBStats.java | 10 +- .../RocksDBKeyValuePrivacyStorageFactory.java | 5 +- .../RocksDBKeyValueStorageFactory.java | 18 ++- .../storage/rocksdb/RocksDBMetrics.java | 96 +------------ .../rocksdb/RocksDBMetricsFactory.java | 135 ++++++++++++++++++ .../storage/rocksdb/RocksDBPlugin.java | 8 +- .../RocksDBColumnarKeyValueStorage.java | 6 +- .../unsegmented/RocksDBKeyValueStorage.java | 7 +- .../RocksDBKeyValueStorageFactoryTest.java | 33 +++-- .../storage/rocksdb/RocksDBMetricsTest.java | 2 +- .../segmented/RocksDBKeyValueStorageTest.java | 8 +- .../RocksDBColumnarKeyValueStorageTest.java | 4 +- 18 files changed, 232 insertions(+), 132 deletions(-) create mode 100644 plugins/rocksdb/src/main/java/org/hyperledger/besu/plugin/services/storage/rocksdb/RocksDBMetricsFactory.java diff --git a/acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/privacy/PrivacyNode.java b/acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/privacy/PrivacyNode.java index f01865ffe7..b487fcb83f 100644 --- a/acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/privacy/PrivacyNode.java +++ b/acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/privacy/PrivacyNode.java @@ -28,6 +28,7 @@ import org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier; import org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueStorageProviderBuilder; import org.hyperledger.besu.metrics.noop.NoOpMetricsSystem; import org.hyperledger.besu.plugin.services.storage.rocksdb.RocksDBKeyValuePrivacyStorageFactory; +import org.hyperledger.besu.plugin.services.storage.rocksdb.RocksDBMetricsFactory; import org.hyperledger.besu.plugin.services.storage.rocksdb.configuration.RocksDBFactoryConfiguration; import org.hyperledger.besu.services.BesuConfigurationImpl; import org.hyperledger.besu.tests.acceptance.dsl.condition.Condition; @@ -220,7 +221,8 @@ public class PrivacyNode implements AutoCloseable { MAX_BACKGROUND_COMPACTIONS, BACKGROUND_THREAD_COUNT, CACHE_CAPACITY), - Arrays.asList(KeyValueSegmentIdentifier.values()))) + Arrays.asList(KeyValueSegmentIdentifier.values()), + RocksDBMetricsFactory.PRIVATE_ROCKS_DB_METRICS)) .withCommonConfiguration(new BesuConfigurationImpl(dataLocation, dbLocation)) .withMetricsSystem(new NoOpMetricsSystem()) .build(); diff --git a/besu/src/test/java/org/hyperledger/besu/PrivacyTest.java b/besu/src/test/java/org/hyperledger/besu/PrivacyTest.java index 3f47797876..7175d6b7f7 100644 --- a/besu/src/test/java/org/hyperledger/besu/PrivacyTest.java +++ b/besu/src/test/java/org/hyperledger/besu/PrivacyTest.java @@ -34,6 +34,7 @@ import org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier; import org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueStorageProviderBuilder; import org.hyperledger.besu.metrics.noop.NoOpMetricsSystem; import org.hyperledger.besu.plugin.services.storage.rocksdb.RocksDBKeyValuePrivacyStorageFactory; +import org.hyperledger.besu.plugin.services.storage.rocksdb.RocksDBMetricsFactory; import org.hyperledger.besu.plugin.services.storage.rocksdb.configuration.RocksDBFactoryConfiguration; import org.hyperledger.besu.services.BesuConfigurationImpl; import org.hyperledger.besu.testutil.TestClock; @@ -105,7 +106,8 @@ public class PrivacyTest { MAX_BACKGROUND_COMPACTIONS, BACKGROUND_THREAD_COUNT, CACHE_CAPACITY), - Arrays.asList(KeyValueSegmentIdentifier.values()))) + Arrays.asList(KeyValueSegmentIdentifier.values()), + RocksDBMetricsFactory.PRIVATE_ROCKS_DB_METRICS)) .withCommonConfiguration(new BesuConfigurationImpl(dataDir, dbDir)) .withMetricsSystem(new NoOpMetricsSystem()) .build(); diff --git a/besu/src/test/java/org/hyperledger/besu/RunnerTest.java b/besu/src/test/java/org/hyperledger/besu/RunnerTest.java index 4703b1dc5b..e2f123ff72 100644 --- a/besu/src/test/java/org/hyperledger/besu/RunnerTest.java +++ b/besu/src/test/java/org/hyperledger/besu/RunnerTest.java @@ -52,6 +52,7 @@ import org.hyperledger.besu.metrics.ObservableMetricsSystem; import org.hyperledger.besu.metrics.noop.NoOpMetricsSystem; import org.hyperledger.besu.metrics.prometheus.MetricsConfiguration; import org.hyperledger.besu.plugin.services.storage.rocksdb.RocksDBKeyValueStorageFactory; +import org.hyperledger.besu.plugin.services.storage.rocksdb.RocksDBMetricsFactory; import org.hyperledger.besu.plugin.services.storage.rocksdb.configuration.RocksDBFactoryConfiguration; import org.hyperledger.besu.services.BesuConfigurationImpl; import org.hyperledger.besu.testutil.TestClock; @@ -369,7 +370,8 @@ public final class RunnerTest { MAX_BACKGROUND_COMPACTIONS, BACKGROUND_THREAD_COUNT, CACHE_CAPACITY), - Arrays.asList(KeyValueSegmentIdentifier.values()))) + Arrays.asList(KeyValueSegmentIdentifier.values()), + RocksDBMetricsFactory.PUBLIC_ROCKS_DB_METRICS)) .withCommonConfiguration(new BesuConfigurationImpl(dataDir, dbDir)) .withMetricsSystem(new NoOpMetricsSystem()) .build(); diff --git a/ethereum/core/src/jmh/java/org/hyperledger/besu/ethereum/vm/operations/OperationBenchmarkHelper.java b/ethereum/core/src/jmh/java/org/hyperledger/besu/ethereum/vm/operations/OperationBenchmarkHelper.java index d26d549073..f56d71d318 100644 --- a/ethereum/core/src/jmh/java/org/hyperledger/besu/ethereum/vm/operations/OperationBenchmarkHelper.java +++ b/ethereum/core/src/jmh/java/org/hyperledger/besu/ethereum/vm/operations/OperationBenchmarkHelper.java @@ -25,6 +25,7 @@ import org.hyperledger.besu.ethereum.core.MessageFrameTestFixture; import org.hyperledger.besu.ethereum.vm.MessageFrame; import org.hyperledger.besu.metrics.noop.NoOpMetricsSystem; import org.hyperledger.besu.plugin.services.storage.KeyValueStorage; +import org.hyperledger.besu.plugin.services.storage.rocksdb.RocksDBMetricsFactory; import org.hyperledger.besu.plugin.services.storage.rocksdb.configuration.RocksDBConfigurationBuilder; import org.hyperledger.besu.plugin.services.storage.rocksdb.unsegmented.RocksDBKeyValueStorage; import org.hyperledger.besu.util.uint.UInt256; @@ -56,7 +57,8 @@ public class OperationBenchmarkHelper { final KeyValueStorage keyValueStorage = new RocksDBKeyValueStorage( new RocksDBConfigurationBuilder().databaseDir(storageDirectory).build(), - new NoOpMetricsSystem()); + new NoOpMetricsSystem(), + RocksDBMetricsFactory.PUBLIC_ROCKS_DB_METRICS); final ExecutionContextTestFixture executionContext = ExecutionContextTestFixture.builder().keyValueStorage(keyValueStorage).build(); diff --git a/ethereum/eth/src/jmh/java/org/hyperledger/besu/ethereum/eth/sync/worldstate/WorldStateDownloaderBenchmark.java b/ethereum/eth/src/jmh/java/org/hyperledger/besu/ethereum/eth/sync/worldstate/WorldStateDownloaderBenchmark.java index c720ed5311..8137336ab9 100644 --- a/ethereum/eth/src/jmh/java/org/hyperledger/besu/ethereum/eth/sync/worldstate/WorldStateDownloaderBenchmark.java +++ b/ethereum/eth/src/jmh/java/org/hyperledger/besu/ethereum/eth/sync/worldstate/WorldStateDownloaderBenchmark.java @@ -40,6 +40,7 @@ import org.hyperledger.besu.ethereum.worldstate.WorldStateStorage; import org.hyperledger.besu.metrics.ObservableMetricsSystem; import org.hyperledger.besu.metrics.noop.NoOpMetricsSystem; import org.hyperledger.besu.plugin.services.storage.rocksdb.RocksDBKeyValueStorageFactory; +import org.hyperledger.besu.plugin.services.storage.rocksdb.RocksDBMetricsFactory; import org.hyperledger.besu.plugin.services.storage.rocksdb.configuration.RocksDBFactoryConfiguration; import org.hyperledger.besu.services.BesuConfigurationImpl; import org.hyperledger.besu.services.tasks.CachingTaskCollection; @@ -168,7 +169,8 @@ public class WorldStateDownloaderBenchmark { DEFAULT_MAX_BACKGROUND_COMPACTIONS, DEFAULT_BACKGROUND_THREAD_COUNT, DEFAULT_CACHE_CAPACITY), - Arrays.asList(KeyValueSegmentIdentifier.values()))) + Arrays.asList(KeyValueSegmentIdentifier.values()), + RocksDBMetricsFactory.PUBLIC_ROCKS_DB_METRICS)) .withCommonConfiguration(new BesuConfigurationImpl(dataDir, dbDir)) .withMetricsSystem(new NoOpMetricsSystem()) .build(); diff --git a/metrics/core/src/main/java/org/hyperledger/besu/metrics/BesuMetricCategory.java b/metrics/core/src/main/java/org/hyperledger/besu/metrics/BesuMetricCategory.java index 2ba04587df..85f8629ecc 100644 --- a/metrics/core/src/main/java/org/hyperledger/besu/metrics/BesuMetricCategory.java +++ b/metrics/core/src/main/java/org/hyperledger/besu/metrics/BesuMetricCategory.java @@ -30,7 +30,9 @@ public enum BesuMetricCategory implements MetricCategory { PEERS("peers"), PERMISSIONING("permissioning"), KVSTORE_ROCKSDB("rocksdb"), + KVSTORE_PRIVATE_ROCKSDB("private_rocksdb"), KVSTORE_ROCKSDB_STATS("rocksdb", false), + KVSTORE_PRIVATE_ROCKSDB_STATS("private_rocksdb", false), PRUNER("pruner"), RPC("rpc"), SYNCHRONIZER("synchronizer"), @@ -40,9 +42,15 @@ public enum BesuMetricCategory implements MetricCategory { public static final Set DEFAULT_METRIC_CATEGORIES; static { - // Why not ROCKSDB and KVSTORE_ROCKSDB_STATS? They hurt performance under load. + // Why not KVSTORE_ROCKSDB and KVSTORE_ROCKSDB_STATS, KVSTORE_PRIVATE_ROCKSDB_STATS, + // KVSTORE_PRIVATE_ROCKSDB_STATS? They hurt performance under load. final EnumSet besuCategories = - EnumSet.complementOf(EnumSet.of(KVSTORE_ROCKSDB, KVSTORE_ROCKSDB_STATS)); + EnumSet.complementOf( + EnumSet.of( + KVSTORE_ROCKSDB, + KVSTORE_ROCKSDB_STATS, + KVSTORE_PRIVATE_ROCKSDB, + KVSTORE_PRIVATE_ROCKSDB_STATS)); DEFAULT_METRIC_CATEGORIES = ImmutableSet.builder() diff --git a/metrics/rocksdb/src/main/java/org/hyperledger/besu/metrics/rocksdb/RocksDBStats.java b/metrics/rocksdb/src/main/java/org/hyperledger/besu/metrics/rocksdb/RocksDBStats.java index ea3ca67a5a..5785c4b57c 100644 --- a/metrics/rocksdb/src/main/java/org/hyperledger/besu/metrics/rocksdb/RocksDBStats.java +++ b/metrics/rocksdb/src/main/java/org/hyperledger/besu/metrics/rocksdb/RocksDBStats.java @@ -17,6 +17,7 @@ package org.hyperledger.besu.metrics.rocksdb; import static org.hyperledger.besu.metrics.BesuMetricCategory.KVSTORE_ROCKSDB_STATS; import org.hyperledger.besu.metrics.prometheus.PrometheusMetricsSystem; +import org.hyperledger.besu.plugin.services.metrics.MetricCategory; import java.util.Arrays; import java.util.Collections; @@ -169,19 +170,20 @@ public class RocksDBStats { }; public static void registerRocksDBMetrics( - final Statistics stats, final PrometheusMetricsSystem metricsSystem) { - + final Statistics stats, + final PrometheusMetricsSystem metricsSystem, + final MetricCategory category) { for (final TickerType ticker : TICKERS) { final String promCounterName = ticker.name().toLowerCase(); metricsSystem.createLongGauge( - KVSTORE_ROCKSDB_STATS, + category, promCounterName, "RocksDB reported statistics for " + ticker.name(), () -> stats.getTickerCount(ticker)); } for (final HistogramType histogram : HISTOGRAMS) { - metricsSystem.addCollector(KVSTORE_ROCKSDB_STATS, histogramToCollector(stats, histogram)); + metricsSystem.addCollector(category, histogramToCollector(stats, histogram)); } } diff --git a/plugins/rocksdb/src/main/java/org/hyperledger/besu/plugin/services/storage/rocksdb/RocksDBKeyValuePrivacyStorageFactory.java b/plugins/rocksdb/src/main/java/org/hyperledger/besu/plugin/services/storage/rocksdb/RocksDBKeyValuePrivacyStorageFactory.java index 95c75e45c4..08f5d533e1 100644 --- a/plugins/rocksdb/src/main/java/org/hyperledger/besu/plugin/services/storage/rocksdb/RocksDBKeyValuePrivacyStorageFactory.java +++ b/plugins/rocksdb/src/main/java/org/hyperledger/besu/plugin/services/storage/rocksdb/RocksDBKeyValuePrivacyStorageFactory.java @@ -30,8 +30,9 @@ public class RocksDBKeyValuePrivacyStorageFactory extends RocksDBKeyValueStorage public RocksDBKeyValuePrivacyStorageFactory( final Supplier configuration, - final List segments) { - super(configuration, segments); + final List segments, + final RocksDBMetricsFactory rocksDBMetricsFactory) { + super(configuration, segments, rocksDBMetricsFactory); } @Override diff --git a/plugins/rocksdb/src/main/java/org/hyperledger/besu/plugin/services/storage/rocksdb/RocksDBKeyValueStorageFactory.java b/plugins/rocksdb/src/main/java/org/hyperledger/besu/plugin/services/storage/rocksdb/RocksDBKeyValueStorageFactory.java index 4453a0d125..b3d8909564 100644 --- a/plugins/rocksdb/src/main/java/org/hyperledger/besu/plugin/services/storage/rocksdb/RocksDBKeyValueStorageFactory.java +++ b/plugins/rocksdb/src/main/java/org/hyperledger/besu/plugin/services/storage/rocksdb/RocksDBKeyValueStorageFactory.java @@ -47,6 +47,7 @@ public class RocksDBKeyValueStorageFactory implements KeyValueStorageFactory { private final int DEFAULT_VERSION; private static final Set SUPPORTED_VERSIONS = Set.of(0, 1); private static final String NAME = "rocksdb"; + private final RocksDBMetricsFactory rocksDBMetricsFactory; private Integer databaseVersion; private Boolean isSegmentIsolationSupported; @@ -60,20 +61,24 @@ public class RocksDBKeyValueStorageFactory implements KeyValueStorageFactory { RocksDBKeyValueStorageFactory( final Supplier configuration, final List segments, - final int DEFAULT_VERSION) { + final int DEFAULT_VERSION, + final RocksDBMetricsFactory rocksDBMetricsFactory) { this.configuration = configuration; this.segments = segments; this.DEFAULT_VERSION = DEFAULT_VERSION; + this.rocksDBMetricsFactory = rocksDBMetricsFactory; } public RocksDBKeyValueStorageFactory( final Supplier configuration, - final List segments) { + final List segments, + final RocksDBMetricsFactory rocksDBMetricsFactory) { this( configuration, segments, /** Source of truth for the default database version. */ - 1); + 1, + rocksDBMetricsFactory); } @Override @@ -100,7 +105,9 @@ public class RocksDBKeyValueStorageFactory implements KeyValueStorageFactory { { segmentedStorage = null; if (unsegmentedStorage == null) { - unsegmentedStorage = new RocksDBKeyValueStorage(rocksDBConfiguration, metricsSystem); + unsegmentedStorage = + new RocksDBKeyValueStorage( + rocksDBConfiguration, metricsSystem, rocksDBMetricsFactory); } return unsegmentedStorage; } @@ -109,7 +116,8 @@ public class RocksDBKeyValueStorageFactory implements KeyValueStorageFactory { unsegmentedStorage = null; if (segmentedStorage == null) { segmentedStorage = - new RocksDBColumnarKeyValueStorage(rocksDBConfiguration, segments, metricsSystem); + new RocksDBColumnarKeyValueStorage( + rocksDBConfiguration, segments, metricsSystem, rocksDBMetricsFactory); } return new SegmentedKeyValueStorageAdapter<>(segment, segmentedStorage); } diff --git a/plugins/rocksdb/src/main/java/org/hyperledger/besu/plugin/services/storage/rocksdb/RocksDBMetrics.java b/plugins/rocksdb/src/main/java/org/hyperledger/besu/plugin/services/storage/rocksdb/RocksDBMetrics.java index e323f01efe..9d18e507a2 100644 --- a/plugins/rocksdb/src/main/java/org/hyperledger/besu/plugin/services/storage/rocksdb/RocksDBMetrics.java +++ b/plugins/rocksdb/src/main/java/org/hyperledger/besu/plugin/services/storage/rocksdb/RocksDBMetrics.java @@ -14,31 +14,18 @@ */ package org.hyperledger.besu.plugin.services.storage.rocksdb; -import org.hyperledger.besu.metrics.BesuMetricCategory; -import org.hyperledger.besu.metrics.prometheus.PrometheusMetricsSystem; -import org.hyperledger.besu.metrics.rocksdb.RocksDBStats; -import org.hyperledger.besu.plugin.services.MetricsSystem; import org.hyperledger.besu.plugin.services.metrics.Counter; import org.hyperledger.besu.plugin.services.metrics.OperationTimer; -import org.hyperledger.besu.plugin.services.storage.rocksdb.configuration.RocksDBConfiguration; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.rocksdb.RocksDBException; -import org.rocksdb.Statistics; -import org.rocksdb.TransactionDB; public class RocksDBMetrics { - private static final Logger LOG = LogManager.getLogger(); - private final OperationTimer readLatency; private final OperationTimer removeLatency; private final OperationTimer writeLatency; private final OperationTimer commitLatency; private final Counter rollbackCount; - private RocksDBMetrics( + public RocksDBMetrics( final OperationTimer readLatency, final OperationTimer removeLatency, final OperationTimer writeLatency, @@ -51,87 +38,6 @@ public class RocksDBMetrics { this.rollbackCount = rollbackCount; } - public static RocksDBMetrics of( - final MetricsSystem metricsSystem, - final RocksDBConfiguration rocksDbConfiguration, - final TransactionDB db, - final Statistics stats) { - final OperationTimer readLatency = - metricsSystem - .createLabelledTimer( - BesuMetricCategory.KVSTORE_ROCKSDB, - "read_latency_seconds", - "Latency for read from RocksDB.", - "database") - .labels(rocksDbConfiguration.getLabel()); - final OperationTimer removeLatency = - metricsSystem - .createLabelledTimer( - BesuMetricCategory.KVSTORE_ROCKSDB, - "remove_latency_seconds", - "Latency of remove requests from RocksDB.", - "database") - .labels(rocksDbConfiguration.getLabel()); - final OperationTimer writeLatency = - metricsSystem - .createLabelledTimer( - BesuMetricCategory.KVSTORE_ROCKSDB, - "write_latency_seconds", - "Latency for write to RocksDB.", - "database") - .labels(rocksDbConfiguration.getLabel()); - final OperationTimer commitLatency = - metricsSystem - .createLabelledTimer( - BesuMetricCategory.KVSTORE_ROCKSDB, - "commit_latency_seconds", - "Latency for commits to RocksDB.", - "database") - .labels(rocksDbConfiguration.getLabel()); - - if (metricsSystem instanceof PrometheusMetricsSystem) { - RocksDBStats.registerRocksDBMetrics(stats, (PrometheusMetricsSystem) metricsSystem); - } - - metricsSystem.createLongGauge( - BesuMetricCategory.KVSTORE_ROCKSDB, - "rocks_db_table_readers_memory_bytes", - "Estimated memory used for RocksDB index and filter blocks in bytes", - () -> { - try { - return db.getLongProperty("rocksdb.estimate-table-readers-mem"); - } catch (final RocksDBException e) { - LOG.debug("Failed to get RocksDB metric", e); - return 0L; - } - }); - - metricsSystem.createLongGauge( - BesuMetricCategory.KVSTORE_ROCKSDB, - "rocks_db_files_size_bytes", - "Estimated database size in bytes", - () -> { - try { - return db.getLongProperty("rocksdb.live-sst-files-size"); - } catch (final RocksDBException e) { - LOG.debug("Failed to get RocksDB metric", e); - return 0L; - } - }); - - final Counter rollbackCount = - metricsSystem - .createLabelledCounter( - BesuMetricCategory.KVSTORE_ROCKSDB, - "rollback_count", - "Number of RocksDB transactions rolled back.", - "database") - .labels(rocksDbConfiguration.getLabel()); - - return new RocksDBMetrics( - readLatency, removeLatency, writeLatency, commitLatency, rollbackCount); - } - public OperationTimer getReadLatency() { return readLatency; } diff --git a/plugins/rocksdb/src/main/java/org/hyperledger/besu/plugin/services/storage/rocksdb/RocksDBMetricsFactory.java b/plugins/rocksdb/src/main/java/org/hyperledger/besu/plugin/services/storage/rocksdb/RocksDBMetricsFactory.java new file mode 100644 index 0000000000..00762318e8 --- /dev/null +++ b/plugins/rocksdb/src/main/java/org/hyperledger/besu/plugin/services/storage/rocksdb/RocksDBMetricsFactory.java @@ -0,0 +1,135 @@ +/* + * Copyright ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ +package org.hyperledger.besu.plugin.services.storage.rocksdb; + +import org.hyperledger.besu.metrics.BesuMetricCategory; +import org.hyperledger.besu.metrics.prometheus.PrometheusMetricsSystem; +import org.hyperledger.besu.metrics.rocksdb.RocksDBStats; +import org.hyperledger.besu.plugin.services.MetricsSystem; +import org.hyperledger.besu.plugin.services.metrics.Counter; +import org.hyperledger.besu.plugin.services.metrics.MetricCategory; +import org.hyperledger.besu.plugin.services.metrics.OperationTimer; +import org.hyperledger.besu.plugin.services.storage.rocksdb.configuration.RocksDBConfiguration; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.rocksdb.RocksDBException; +import org.rocksdb.Statistics; +import org.rocksdb.TransactionDB; + +public class RocksDBMetricsFactory { + + public static final RocksDBMetricsFactory PUBLIC_ROCKS_DB_METRICS = + new RocksDBMetricsFactory( + BesuMetricCategory.KVSTORE_ROCKSDB, BesuMetricCategory.KVSTORE_ROCKSDB_STATS); + + public static final RocksDBMetricsFactory PRIVATE_ROCKS_DB_METRICS = + new RocksDBMetricsFactory( + BesuMetricCategory.KVSTORE_PRIVATE_ROCKSDB, + BesuMetricCategory.KVSTORE_PRIVATE_ROCKSDB_STATS); + + private static final Logger LOG = LogManager.getLogger(); + + private final MetricCategory rocksDbMetricCategory; + private final MetricCategory statsDbMetricCategory; + + public RocksDBMetricsFactory( + final MetricCategory rocksDbMetricCategory, final MetricCategory statsDbMetricCategory) { + this.rocksDbMetricCategory = rocksDbMetricCategory; + this.statsDbMetricCategory = statsDbMetricCategory; + } + + public RocksDBMetrics create( + final MetricsSystem metricsSystem, + final RocksDBConfiguration rocksDbConfiguration, + final TransactionDB db, + final Statistics stats) { + final OperationTimer readLatency = + metricsSystem + .createLabelledTimer( + rocksDbMetricCategory, + "read_latency_seconds", + "Latency for read from RocksDB.", + "database") + .labels(rocksDbConfiguration.getLabel()); + final OperationTimer removeLatency = + metricsSystem + .createLabelledTimer( + rocksDbMetricCategory, + "remove_latency_seconds", + "Latency of remove requests from RocksDB.", + "database") + .labels(rocksDbConfiguration.getLabel()); + final OperationTimer writeLatency = + metricsSystem + .createLabelledTimer( + rocksDbMetricCategory, + "write_latency_seconds", + "Latency for write to RocksDB.", + "database") + .labels(rocksDbConfiguration.getLabel()); + final OperationTimer commitLatency = + metricsSystem + .createLabelledTimer( + rocksDbMetricCategory, + "commit_latency_seconds", + "Latency for commits to RocksDB.", + "database") + .labels(rocksDbConfiguration.getLabel()); + + if (metricsSystem instanceof PrometheusMetricsSystem) { + RocksDBStats.registerRocksDBMetrics( + stats, (PrometheusMetricsSystem) metricsSystem, statsDbMetricCategory); + } + + metricsSystem.createLongGauge( + rocksDbMetricCategory, + "rocks_db_table_readers_memory_bytes", + "Estimated memory used for RocksDB index and filter blocks in bytes", + () -> { + try { + return db.getLongProperty("rocksdb.estimate-table-readers-mem"); + } catch (final RocksDBException e) { + LOG.debug("Failed to get RocksDB metric", e); + return 0L; + } + }); + + metricsSystem.createLongGauge( + rocksDbMetricCategory, + "rocks_db_files_size_bytes", + "Estimated database size in bytes", + () -> { + try { + return db.getLongProperty("rocksdb.live-sst-files-size"); + } catch (final RocksDBException e) { + LOG.debug("Failed to get RocksDB metric", e); + return 0L; + } + }); + + final Counter rollbackCount = + metricsSystem + .createLabelledCounter( + rocksDbMetricCategory, + "rollback_count", + "Number of RocksDB transactions rolled back.", + "database") + .labels(rocksDbConfiguration.getLabel()); + + return new RocksDBMetrics( + readLatency, removeLatency, writeLatency, commitLatency, rollbackCount); + } +} diff --git a/plugins/rocksdb/src/main/java/org/hyperledger/besu/plugin/services/storage/rocksdb/RocksDBPlugin.java b/plugins/rocksdb/src/main/java/org/hyperledger/besu/plugin/services/storage/rocksdb/RocksDBPlugin.java index 3cd3241b38..bca673f444 100644 --- a/plugins/rocksdb/src/main/java/org/hyperledger/besu/plugin/services/storage/rocksdb/RocksDBPlugin.java +++ b/plugins/rocksdb/src/main/java/org/hyperledger/besu/plugin/services/storage/rocksdb/RocksDBPlugin.java @@ -100,8 +100,12 @@ public class RocksDBPlugin implements BesuPlugin { final Supplier configuration = Suppliers.memoize(options::toDomainObject); - factory = new RocksDBKeyValueStorageFactory(configuration, segments); - privacyFactory = new RocksDBKeyValuePrivacyStorageFactory(configuration, segments); + factory = + new RocksDBKeyValueStorageFactory( + configuration, segments, RocksDBMetricsFactory.PUBLIC_ROCKS_DB_METRICS); + privacyFactory = + new RocksDBKeyValuePrivacyStorageFactory( + configuration, segments, RocksDBMetricsFactory.PRIVATE_ROCKS_DB_METRICS); service.registerKeyValueStorage(factory); service.registerKeyValueStorage(privacyFactory); diff --git a/plugins/rocksdb/src/main/java/org/hyperledger/besu/plugin/services/storage/rocksdb/segmented/RocksDBColumnarKeyValueStorage.java b/plugins/rocksdb/src/main/java/org/hyperledger/besu/plugin/services/storage/rocksdb/segmented/RocksDBColumnarKeyValueStorage.java index a169d492a2..938d30833b 100644 --- a/plugins/rocksdb/src/main/java/org/hyperledger/besu/plugin/services/storage/rocksdb/segmented/RocksDBColumnarKeyValueStorage.java +++ b/plugins/rocksdb/src/main/java/org/hyperledger/besu/plugin/services/storage/rocksdb/segmented/RocksDBColumnarKeyValueStorage.java @@ -21,6 +21,7 @@ import org.hyperledger.besu.plugin.services.exception.StorageException; import org.hyperledger.besu.plugin.services.metrics.OperationTimer; import org.hyperledger.besu.plugin.services.storage.SegmentIdentifier; import org.hyperledger.besu.plugin.services.storage.rocksdb.RocksDBMetrics; +import org.hyperledger.besu.plugin.services.storage.rocksdb.RocksDBMetricsFactory; import org.hyperledger.besu.plugin.services.storage.rocksdb.RocksDbUtil; import org.hyperledger.besu.plugin.services.storage.rocksdb.configuration.RocksDBConfiguration; import org.hyperledger.besu.services.kvstore.SegmentedKeyValueStorage; @@ -76,7 +77,8 @@ public class RocksDBColumnarKeyValueStorage public RocksDBColumnarKeyValueStorage( final RocksDBConfiguration configuration, final List segments, - final MetricsSystem metricsSystem) + final MetricsSystem metricsSystem, + final RocksDBMetricsFactory rocksDBMetricsFactory) throws StorageException { try { @@ -110,7 +112,7 @@ public class RocksDBColumnarKeyValueStorage configuration.getDatabaseDir().toString(), columnDescriptors, columnHandles); - metrics = RocksDBMetrics.of(metricsSystem, configuration, db, stats); + metrics = rocksDBMetricsFactory.create(metricsSystem, configuration, db, stats); final Map segmentsById = segments.stream() .collect( diff --git a/plugins/rocksdb/src/main/java/org/hyperledger/besu/plugin/services/storage/rocksdb/unsegmented/RocksDBKeyValueStorage.java b/plugins/rocksdb/src/main/java/org/hyperledger/besu/plugin/services/storage/rocksdb/unsegmented/RocksDBKeyValueStorage.java index b858f156af..0ecaee39f3 100644 --- a/plugins/rocksdb/src/main/java/org/hyperledger/besu/plugin/services/storage/rocksdb/unsegmented/RocksDBKeyValueStorage.java +++ b/plugins/rocksdb/src/main/java/org/hyperledger/besu/plugin/services/storage/rocksdb/unsegmented/RocksDBKeyValueStorage.java @@ -20,6 +20,7 @@ import org.hyperledger.besu.plugin.services.metrics.OperationTimer; import org.hyperledger.besu.plugin.services.storage.KeyValueStorage; import org.hyperledger.besu.plugin.services.storage.KeyValueStorageTransaction; import org.hyperledger.besu.plugin.services.storage.rocksdb.RocksDBMetrics; +import org.hyperledger.besu.plugin.services.storage.rocksdb.RocksDBMetricsFactory; import org.hyperledger.besu.plugin.services.storage.rocksdb.RocksDbUtil; import org.hyperledger.besu.plugin.services.storage.rocksdb.configuration.RocksDBConfiguration; import org.hyperledger.besu.services.kvstore.KeyValueStorageTransactionTransitionValidatorDecorator; @@ -57,7 +58,9 @@ public class RocksDBKeyValueStorage implements KeyValueStorage { private final RocksDBMetrics rocksDBMetrics; public RocksDBKeyValueStorage( - final RocksDBConfiguration configuration, final MetricsSystem metricsSystem) { + final RocksDBConfiguration configuration, + final MetricsSystem metricsSystem, + final RocksDBMetricsFactory rocksDBMetricsFactory) { try { final Statistics stats = new Statistics(); @@ -72,7 +75,7 @@ public class RocksDBKeyValueStorage implements KeyValueStorage { txOptions = new TransactionDBOptions(); db = TransactionDB.open(options, txOptions, configuration.getDatabaseDir().toString()); - rocksDBMetrics = RocksDBMetrics.of(metricsSystem, configuration, db, stats); + rocksDBMetrics = rocksDBMetricsFactory.create(metricsSystem, configuration, db, stats); } catch (final RocksDBException e) { throw new StorageException(e); } diff --git a/plugins/rocksdb/src/test/java/org/hyperledger/besu/plugin/services/storage/rocksdb/RocksDBKeyValueStorageFactoryTest.java b/plugins/rocksdb/src/test/java/org/hyperledger/besu/plugin/services/storage/rocksdb/RocksDBKeyValueStorageFactoryTest.java index 20780ce79a..6ea9cc1a0a 100644 --- a/plugins/rocksdb/src/test/java/org/hyperledger/besu/plugin/services/storage/rocksdb/RocksDBKeyValueStorageFactoryTest.java +++ b/plugins/rocksdb/src/test/java/org/hyperledger/besu/plugin/services/storage/rocksdb/RocksDBKeyValueStorageFactoryTest.java @@ -60,7 +60,8 @@ public class RocksDBKeyValueStorageFactoryTest { when(commonConfiguration.getDataPath()).thenReturn(tempDataDir); final RocksDBKeyValueStorageFactory storageFactory = - new RocksDBKeyValueStorageFactory(() -> rocksDbConfiguration, segments); + new RocksDBKeyValueStorageFactory( + () -> rocksDbConfiguration, segments, RocksDBMetricsFactory.PUBLIC_ROCKS_DB_METRICS); // Side effect is creation of the Metadata version file storageFactory.create(segment, commonConfiguration, metricsSystem); @@ -83,7 +84,8 @@ public class RocksDBKeyValueStorageFactoryTest { when(commonConfiguration.getDataPath()).thenReturn(tempDataDir); final RocksDBKeyValueStorageFactory storageFactory = - new RocksDBKeyValueStorageFactory(() -> rocksDbConfiguration, segments); + new RocksDBKeyValueStorageFactory( + () -> rocksDbConfiguration, segments, RocksDBMetricsFactory.PUBLIC_ROCKS_DB_METRICS); storageFactory.create(segment, commonConfiguration, metricsSystem); @@ -100,7 +102,8 @@ public class RocksDBKeyValueStorageFactoryTest { when(commonConfiguration.getDataPath()).thenReturn(tempDataDir); final RocksDBKeyValueStorageFactory storageFactory = - new RocksDBKeyValueStorageFactory(() -> rocksDbConfiguration, segments); + new RocksDBKeyValueStorageFactory( + () -> rocksDbConfiguration, segments, RocksDBMetricsFactory.PUBLIC_ROCKS_DB_METRICS); storageFactory.create(segment, commonConfiguration, metricsSystem); @@ -119,13 +122,15 @@ public class RocksDBKeyValueStorageFactoryTest { when(commonConfiguration.getDataPath()).thenReturn(tempDataDir); final RocksDBKeyValueStorageFactory storageFactory = - new RocksDBKeyValueStorageFactory(() -> rocksDbConfiguration, segments, 1); + new RocksDBKeyValueStorageFactory( + () -> rocksDbConfiguration, segments, 1, RocksDBMetricsFactory.PUBLIC_ROCKS_DB_METRICS); storageFactory.create(segment, commonConfiguration, metricsSystem); storageFactory.close(); final RocksDBKeyValueStorageFactory rolledbackStorageFactory = - new RocksDBKeyValueStorageFactory(() -> rocksDbConfiguration, segments, 0); + new RocksDBKeyValueStorageFactory( + () -> rocksDbConfiguration, segments, 0, RocksDBMetricsFactory.PUBLIC_ROCKS_DB_METRICS); rolledbackStorageFactory.create(segment, commonConfiguration, metricsSystem); } @@ -141,7 +146,10 @@ public class RocksDBKeyValueStorageFactoryTest { new DatabaseMetadata(-1).writeToDirectory(tempDatabaseDir); assertThatThrownBy( () -> - new RocksDBKeyValueStorageFactory(() -> rocksDbConfiguration, segments) + new RocksDBKeyValueStorageFactory( + () -> rocksDbConfiguration, + segments, + RocksDBMetricsFactory.PUBLIC_ROCKS_DB_METRICS) .create(segment, commonConfiguration, metricsSystem)) .isInstanceOf(StorageException.class); } @@ -156,7 +164,8 @@ public class RocksDBKeyValueStorageFactoryTest { when(commonConfiguration.getDataPath()).thenReturn(tempDataDir); final RocksDBKeyValueStorageFactory storageFactory = - new RocksDBKeyValueStorageFactory(() -> rocksDbConfiguration, segments); + new RocksDBKeyValueStorageFactory( + () -> rocksDbConfiguration, segments, RocksDBMetricsFactory.PUBLIC_ROCKS_DB_METRICS); storageFactory.create(segment, commonConfiguration, metricsSystem); assertThatCode(storageFactory::isSegmentIsolationSupported).doesNotThrowAnyException(); } @@ -177,7 +186,10 @@ public class RocksDBKeyValueStorageFactoryTest { assertThatThrownBy( () -> - new RocksDBKeyValueStorageFactory(() -> rocksDbConfiguration, segments) + new RocksDBKeyValueStorageFactory( + () -> rocksDbConfiguration, + segments, + RocksDBMetricsFactory.PUBLIC_ROCKS_DB_METRICS) .create(segment, commonConfiguration, metricsSystem)) .isInstanceOf(IllegalStateException.class); @@ -187,7 +199,10 @@ public class RocksDBKeyValueStorageFactoryTest { assertThatThrownBy( () -> - new RocksDBKeyValueStorageFactory(() -> rocksDbConfiguration, segments) + new RocksDBKeyValueStorageFactory( + () -> rocksDbConfiguration, + segments, + RocksDBMetricsFactory.PUBLIC_ROCKS_DB_METRICS) .create(segment, commonConfiguration, metricsSystem)) .isInstanceOf(IllegalStateException.class); } diff --git a/plugins/rocksdb/src/test/java/org/hyperledger/besu/plugin/services/storage/rocksdb/RocksDBMetricsTest.java b/plugins/rocksdb/src/test/java/org/hyperledger/besu/plugin/services/storage/rocksdb/RocksDBMetricsTest.java index f3dfe10ec0..ddcf58e4c2 100644 --- a/plugins/rocksdb/src/test/java/org/hyperledger/besu/plugin/services/storage/rocksdb/RocksDBMetricsTest.java +++ b/plugins/rocksdb/src/test/java/org/hyperledger/besu/plugin/services/storage/rocksdb/RocksDBMetricsTest.java @@ -74,7 +74,7 @@ public class RocksDBMetricsTest { final ArgumentCaptor longGaugesMetricsNameArgs = ArgumentCaptor.forClass(String.class); final ArgumentCaptor longGaugesHelpArgs = ArgumentCaptor.forClass(String.class); - RocksDBMetrics.of(metricsSystemMock, config(), db, stats); + RocksDBMetricsFactory.PUBLIC_ROCKS_DB_METRICS.create(metricsSystemMock, config(), db, stats); verify(metricsSystemMock, times(4)) .createLabelledTimer( diff --git a/plugins/rocksdb/src/test/java/org/hyperledger/besu/plugin/services/storage/rocksdb/segmented/RocksDBKeyValueStorageTest.java b/plugins/rocksdb/src/test/java/org/hyperledger/besu/plugin/services/storage/rocksdb/segmented/RocksDBKeyValueStorageTest.java index 33bd780e70..85d175f33f 100644 --- a/plugins/rocksdb/src/test/java/org/hyperledger/besu/plugin/services/storage/rocksdb/segmented/RocksDBKeyValueStorageTest.java +++ b/plugins/rocksdb/src/test/java/org/hyperledger/besu/plugin/services/storage/rocksdb/segmented/RocksDBKeyValueStorageTest.java @@ -30,6 +30,7 @@ import org.hyperledger.besu.plugin.services.metrics.Counter; import org.hyperledger.besu.plugin.services.metrics.LabelledMetric; import org.hyperledger.besu.plugin.services.metrics.OperationTimer; import org.hyperledger.besu.plugin.services.storage.KeyValueStorage; +import org.hyperledger.besu.plugin.services.storage.rocksdb.RocksDBMetricsFactory; import org.hyperledger.besu.plugin.services.storage.rocksdb.configuration.RocksDBConfiguration; import org.hyperledger.besu.plugin.services.storage.rocksdb.configuration.RocksDBConfigurationBuilder; import org.hyperledger.besu.plugin.services.storage.rocksdb.unsegmented.RocksDBKeyValueStorage; @@ -55,7 +56,8 @@ public class RocksDBKeyValueStorageTest extends AbstractKeyValueStorageTest { @Override protected KeyValueStorage createStore() throws Exception { - return new RocksDBKeyValueStorage(config(), new NoOpMetricsSystem()); + return new RocksDBKeyValueStorage( + config(), new NoOpMetricsSystem(), RocksDBMetricsFactory.PUBLIC_ROCKS_DB_METRICS); } @Test @@ -79,7 +81,9 @@ public class RocksDBKeyValueStorageTest extends AbstractKeyValueStorageTest { final ArgumentCaptor longGaugesHelpArgs = ArgumentCaptor.forClass(String.class); // Actual call - final KeyValueStorage keyValueStorage = new RocksDBKeyValueStorage(config(), metricsSystemMock); + final KeyValueStorage keyValueStorage = + new RocksDBKeyValueStorage( + config(), metricsSystemMock, RocksDBMetricsFactory.PUBLIC_ROCKS_DB_METRICS); // Assertions assertThat(keyValueStorage).isNotNull(); diff --git a/plugins/rocksdb/src/test/java/org/hyperledger/besu/plugin/services/storage/rocksdb/unsegmented/RocksDBColumnarKeyValueStorageTest.java b/plugins/rocksdb/src/test/java/org/hyperledger/besu/plugin/services/storage/rocksdb/unsegmented/RocksDBColumnarKeyValueStorageTest.java index 3776b9646f..b45029a6b7 100644 --- a/plugins/rocksdb/src/test/java/org/hyperledger/besu/plugin/services/storage/rocksdb/unsegmented/RocksDBColumnarKeyValueStorageTest.java +++ b/plugins/rocksdb/src/test/java/org/hyperledger/besu/plugin/services/storage/rocksdb/unsegmented/RocksDBColumnarKeyValueStorageTest.java @@ -20,6 +20,7 @@ import org.hyperledger.besu.kvstore.AbstractKeyValueStorageTest; import org.hyperledger.besu.metrics.noop.NoOpMetricsSystem; import org.hyperledger.besu.plugin.services.storage.KeyValueStorage; import org.hyperledger.besu.plugin.services.storage.SegmentIdentifier; +import org.hyperledger.besu.plugin.services.storage.rocksdb.RocksDBMetricsFactory; import org.hyperledger.besu.plugin.services.storage.rocksdb.configuration.RocksDBConfigurationBuilder; import org.hyperledger.besu.plugin.services.storage.rocksdb.segmented.RocksDBColumnarKeyValueStorage; import org.hyperledger.besu.services.kvstore.SegmentedKeyValueStorage; @@ -147,7 +148,8 @@ public class RocksDBColumnarKeyValueStorageTest extends AbstractKeyValueStorageT return new RocksDBColumnarKeyValueStorage( new RocksDBConfigurationBuilder().databaseDir(folder.newFolder().toPath()).build(), Arrays.asList(TestSegment.FOO, TestSegment.BAR), - new NoOpMetricsSystem()); + new NoOpMetricsSystem(), + RocksDBMetricsFactory.PUBLIC_ROCKS_DB_METRICS); } @Override