From eab908441dc9df7e280414e4d5ea1bae9ef8b671 Mon Sep 17 00:00:00 2001 From: Danno Ferrin Date: Wed, 9 Jan 2019 09:26:46 -0700 Subject: [PATCH] RocksDB Metrics (#531) * Measure read/get, write/put, remove, and commit latencies * Measure rollback count (latency seemed a bit much) --- ethereum/core/build.gradle | 1 + .../operations/OperationBenchmarkHelper.java | 4 +- .../keyvalue/RocksDbStorageProvider.java | 7 ++- .../pantheon/metrics/MetricCategory.java | 3 +- .../cli/PantheonControllerBuilder.java | 2 +- .../tech/pegasys/pantheon/RunnerTest.java | 2 +- services/kvstore/build.gradle | 1 + .../kvstore/RocksDbKeyValueStorage.java | 52 +++++++++++++++---- .../kvstore/RocksDbKeyValueStorageTest.java | 4 +- 9 files changed, 60 insertions(+), 16 deletions(-) diff --git a/ethereum/core/build.gradle b/ethereum/core/build.gradle index d23e2acb05..6dbf266495 100644 --- a/ethereum/core/build.gradle +++ b/ethereum/core/build.gradle @@ -67,6 +67,7 @@ dependencies { jmhImplementation project(path: ':ethereum:core', configuration: 'testSupportArtifacts') jmhImplementation project(':ethereum:rlp') jmhImplementation project(':ethereum:trie') + jmhImplementation project(':metrics') jmhImplementation project(':services:kvstore') jmhImplementation project(':util') diff --git a/ethereum/core/src/jmh/java/tech/pegasys/pantheon/ethereum/vm/operations/OperationBenchmarkHelper.java b/ethereum/core/src/jmh/java/tech/pegasys/pantheon/ethereum/vm/operations/OperationBenchmarkHelper.java index 6537e1ff2e..b31f53640a 100644 --- a/ethereum/core/src/jmh/java/tech/pegasys/pantheon/ethereum/vm/operations/OperationBenchmarkHelper.java +++ b/ethereum/core/src/jmh/java/tech/pegasys/pantheon/ethereum/vm/operations/OperationBenchmarkHelper.java @@ -21,6 +21,7 @@ import tech.pegasys.pantheon.ethereum.core.BlockHeaderTestFixture; import tech.pegasys.pantheon.ethereum.core.ExecutionContextTestFixture; import tech.pegasys.pantheon.ethereum.core.MessageFrameTestFixture; import tech.pegasys.pantheon.ethereum.vm.MessageFrame; +import tech.pegasys.pantheon.metrics.noop.NoOpMetricsSystem; import tech.pegasys.pantheon.services.kvstore.KeyValueStorage; import tech.pegasys.pantheon.services.kvstore.RocksDbKeyValueStorage; import tech.pegasys.pantheon.util.uint.UInt256; @@ -49,7 +50,8 @@ public class OperationBenchmarkHelper { public static OperationBenchmarkHelper create() throws IOException { final Path storageDirectory = Files.createTempDirectory("benchmark"); - final KeyValueStorage keyValueStorage = RocksDbKeyValueStorage.create(storageDirectory); + final KeyValueStorage keyValueStorage = + RocksDbKeyValueStorage.create(storageDirectory, new NoOpMetricsSystem()); final ExecutionContextTestFixture executionContext = ExecutionContextTestFixture.builder().keyValueStorage(keyValueStorage).build(); diff --git a/ethereum/core/src/main/java/tech/pegasys/pantheon/ethereum/storage/keyvalue/RocksDbStorageProvider.java b/ethereum/core/src/main/java/tech/pegasys/pantheon/ethereum/storage/keyvalue/RocksDbStorageProvider.java index 073e9e42d7..7492f203c4 100644 --- a/ethereum/core/src/main/java/tech/pegasys/pantheon/ethereum/storage/keyvalue/RocksDbStorageProvider.java +++ b/ethereum/core/src/main/java/tech/pegasys/pantheon/ethereum/storage/keyvalue/RocksDbStorageProvider.java @@ -13,6 +13,7 @@ package tech.pegasys.pantheon.ethereum.storage.keyvalue; import tech.pegasys.pantheon.ethereum.storage.StorageProvider; +import tech.pegasys.pantheon.metrics.MetricsSystem; import tech.pegasys.pantheon.services.kvstore.KeyValueStorage; import tech.pegasys.pantheon.services.kvstore.RocksDbKeyValueStorage; @@ -22,8 +23,10 @@ import java.nio.file.Path; public class RocksDbStorageProvider { - public static StorageProvider create(final Path databaseDir) throws IOException { - final KeyValueStorage kv = RocksDbKeyValueStorage.create(Files.createDirectories(databaseDir)); + public static StorageProvider create(final Path databaseDir, final MetricsSystem metricsSystem) + throws IOException { + final KeyValueStorage kv = + RocksDbKeyValueStorage.create(Files.createDirectories(databaseDir), metricsSystem); return new KeyValueStorageProvider(kv); } } diff --git a/metrics/src/main/java/tech/pegasys/pantheon/metrics/MetricCategory.java b/metrics/src/main/java/tech/pegasys/pantheon/metrics/MetricCategory.java index 9640aec0c7..b7e0db0eb8 100644 --- a/metrics/src/main/java/tech/pegasys/pantheon/metrics/MetricCategory.java +++ b/metrics/src/main/java/tech/pegasys/pantheon/metrics/MetricCategory.java @@ -19,7 +19,8 @@ public enum MetricCategory { PROCESS("process", false), BLOCKCHAIN("blockchain"), SYNCHRONIZER("synchronizer"), - NETWORK("network"); + NETWORK("network"), + ROCKSDB("rocksdb"); private final String name; private final boolean pantheonSpecific; diff --git a/pantheon/src/main/java/tech/pegasys/pantheon/cli/PantheonControllerBuilder.java b/pantheon/src/main/java/tech/pegasys/pantheon/cli/PantheonControllerBuilder.java index 01505223da..2aa74284cb 100644 --- a/pantheon/src/main/java/tech/pegasys/pantheon/cli/PantheonControllerBuilder.java +++ b/pantheon/src/main/java/tech/pegasys/pantheon/cli/PantheonControllerBuilder.java @@ -88,7 +88,7 @@ public class PantheonControllerBuilder { final KeyPair nodeKeys = loadKeyPair(nodePrivateKeyFile); final StorageProvider storageProvider = - RocksDbStorageProvider.create(homePath.resolve(DATABASE_PATH)); + RocksDbStorageProvider.create(homePath.resolve(DATABASE_PATH), metricsSystem); if (devMode) { final GenesisConfigFile genesisConfig = GenesisConfigFile.development(); return MainnetPantheonController.init( diff --git a/pantheon/src/test/java/tech/pegasys/pantheon/RunnerTest.java b/pantheon/src/test/java/tech/pegasys/pantheon/RunnerTest.java index 56129232e7..e81c4b4637 100644 --- a/pantheon/src/test/java/tech/pegasys/pantheon/RunnerTest.java +++ b/pantheon/src/test/java/tech/pegasys/pantheon/RunnerTest.java @@ -250,7 +250,7 @@ public final class RunnerTest { } private StorageProvider createKeyValueStorageProvider(final Path dbAhead) throws IOException { - return RocksDbStorageProvider.create(dbAhead); + return RocksDbStorageProvider.create(dbAhead, new NoOpMetricsSystem()); } private JsonRpcConfiguration jsonRpcConfiguration() { diff --git a/services/kvstore/build.gradle b/services/kvstore/build.gradle index fe51c110eb..6ed1fbb02d 100644 --- a/services/kvstore/build.gradle +++ b/services/kvstore/build.gradle @@ -27,6 +27,7 @@ jar { dependencies { api project(':util') + implementation project(':metrics') implementation 'org.apache.logging.log4j:log4j-api' implementation 'com.google.guava:guava' diff --git a/services/kvstore/src/main/java/tech/pegasys/pantheon/services/kvstore/RocksDbKeyValueStorage.java b/services/kvstore/src/main/java/tech/pegasys/pantheon/services/kvstore/RocksDbKeyValueStorage.java index 1205a92c0d..efe09c7ec9 100644 --- a/services/kvstore/src/main/java/tech/pegasys/pantheon/services/kvstore/RocksDbKeyValueStorage.java +++ b/services/kvstore/src/main/java/tech/pegasys/pantheon/services/kvstore/RocksDbKeyValueStorage.java @@ -12,6 +12,10 @@ */ package tech.pegasys.pantheon.services.kvstore; +import tech.pegasys.pantheon.metrics.Counter; +import tech.pegasys.pantheon.metrics.MetricCategory; +import tech.pegasys.pantheon.metrics.MetricsSystem; +import tech.pegasys.pantheon.metrics.OperationTimer; import tech.pegasys.pantheon.util.bytes.BytesValue; import java.io.Closeable; @@ -44,19 +48,47 @@ public class RocksDbKeyValueStorage implements KeyValueStorage, Closeable { private final TransactionDB db; private final AtomicBoolean closed = new AtomicBoolean(false); + private final OperationTimer readLatency; + private final OperationTimer removeLatency; + private final OperationTimer writeLatency; + private final OperationTimer commitLatency; + private final Counter rollbackCount; + static { RocksDB.loadLibrary(); } - public static KeyValueStorage create(final Path storageDirectory) throws StorageException { - return new RocksDbKeyValueStorage(storageDirectory); + public static KeyValueStorage create( + final Path storageDirectory, final MetricsSystem metricsSystem) throws StorageException { + return new RocksDbKeyValueStorage(storageDirectory, metricsSystem); } - private RocksDbKeyValueStorage(final Path storageDirectory) { + private RocksDbKeyValueStorage(final Path storageDirectory, final MetricsSystem metricsSystem) { try { options = new Options().setCreateIfMissing(true); txOptions = new TransactionDBOptions(); db = TransactionDB.open(options, txOptions, storageDirectory.toString()); + + readLatency = + metricsSystem.createTimer( + MetricCategory.ROCKSDB, "read_latency_seconds", "Latency for read from RocksDB."); + removeLatency = + metricsSystem.createTimer( + MetricCategory.ROCKSDB, + "remove_latency_seconds", + "Latency of remove requests from RocksDB."); + writeLatency = + metricsSystem.createTimer( + MetricCategory.ROCKSDB, "write_latency_seconds", "Latency for write to RocksDB."); + commitLatency = + metricsSystem.createTimer( + MetricCategory.ROCKSDB, "commit_latency_seconds", "Latency for commits to RocksDB."); + + rollbackCount = + metricsSystem.createCounter( + MetricCategory.ROCKSDB, + "rollback_count", + "Number of RocksDB transactions rolled back."); } catch (final RocksDBException e) { throw new StorageException(e); } @@ -65,7 +97,8 @@ public class RocksDbKeyValueStorage implements KeyValueStorage, Closeable { @Override public Optional get(final BytesValue key) throws StorageException { throwIfClosed(); - try { + + try (final OperationTimer.TimingContext ignored = readLatency.startTimer()) { return Optional.ofNullable(db.get(key.extractArray())).map(BytesValue::wrap); } catch (final RocksDBException e) { throw new StorageException(e); @@ -143,7 +176,7 @@ public class RocksDbKeyValueStorage implements KeyValueStorage, Closeable { return entry; } - public Stream toStream() { + Stream toStream() { final Spliterator split = Spliterators.spliteratorUnknownSize( this, Spliterator.IMMUTABLE | Spliterator.DISTINCT | Spliterator.NONNULL); @@ -158,7 +191,7 @@ public class RocksDbKeyValueStorage implements KeyValueStorage, Closeable { } } - private static class RocksDbTransaction extends AbstractTransaction { + private class RocksDbTransaction extends AbstractTransaction { private final org.rocksdb.Transaction innerTx; private final WriteOptions options; @@ -169,7 +202,7 @@ public class RocksDbKeyValueStorage implements KeyValueStorage, Closeable { @Override protected void doPut(final BytesValue key, final BytesValue value) { - try { + try (final OperationTimer.TimingContext ignored = writeLatency.startTimer()) { innerTx.put(key.extractArray(), value.extractArray()); } catch (final RocksDBException e) { throw new StorageException(e); @@ -178,7 +211,7 @@ public class RocksDbKeyValueStorage implements KeyValueStorage, Closeable { @Override protected void doRemove(final BytesValue key) { - try { + try (final OperationTimer.TimingContext ignored = removeLatency.startTimer()) { innerTx.delete(key.extractArray()); } catch (final RocksDBException e) { throw new StorageException(e); @@ -187,7 +220,7 @@ public class RocksDbKeyValueStorage implements KeyValueStorage, Closeable { @Override protected void doCommit() throws StorageException { - try { + try (final OperationTimer.TimingContext ignored = commitLatency.startTimer()) { innerTx.commit(); } catch (final RocksDBException e) { throw new StorageException(e); @@ -200,6 +233,7 @@ public class RocksDbKeyValueStorage implements KeyValueStorage, Closeable { protected void doRollback() { try { innerTx.rollback(); + rollbackCount.inc(); } catch (final RocksDBException e) { throw new StorageException(e); } finally { diff --git a/services/kvstore/src/test/java/tech/pegasys/pantheon/services/kvstore/RocksDbKeyValueStorageTest.java b/services/kvstore/src/test/java/tech/pegasys/pantheon/services/kvstore/RocksDbKeyValueStorageTest.java index 2843980522..54fe5d5d43 100644 --- a/services/kvstore/src/test/java/tech/pegasys/pantheon/services/kvstore/RocksDbKeyValueStorageTest.java +++ b/services/kvstore/src/test/java/tech/pegasys/pantheon/services/kvstore/RocksDbKeyValueStorageTest.java @@ -12,6 +12,8 @@ */ package tech.pegasys.pantheon.services.kvstore; +import tech.pegasys.pantheon.metrics.noop.NoOpMetricsSystem; + import org.junit.Rule; import org.junit.rules.TemporaryFolder; @@ -21,6 +23,6 @@ public class RocksDbKeyValueStorageTest extends AbstractKeyValueStorageTest { @Override protected KeyValueStorage createStore() throws Exception { - return RocksDbKeyValueStorage.create(folder.newFolder().toPath()); + return RocksDbKeyValueStorage.create(folder.newFolder().toPath(), new NoOpMetricsSystem()); } }