diff --git a/metrics/src/main/java/tech/pegasys/pantheon/metrics/MetricCategory.java b/metrics/src/main/java/tech/pegasys/pantheon/metrics/MetricCategory.java index cf71690dac..0f6c81b27b 100644 --- a/metrics/src/main/java/tech/pegasys/pantheon/metrics/MetricCategory.java +++ b/metrics/src/main/java/tech/pegasys/pantheon/metrics/MetricCategory.java @@ -23,14 +23,15 @@ public enum MetricCategory { NETWORK("network"), PEERS("peers"), PROCESS("process", false), - ROCKSDB("rocksdb"), + KVSTORE_ROCKSDB("rocksdb"), + KVSTORE_ROCKSDB_STATS("rocksdb", false), RPC("rpc"), SYNCHRONIZER("synchronizer"), TRANSACTION_POOL("transaction_pool"); // Why not BIG_QUEUE and ROCKSDB? They hurt performance under load. public static final Set DEFAULT_METRIC_CATEGORIES = - EnumSet.complementOf(EnumSet.of(BIG_QUEUE, ROCKSDB)); + EnumSet.complementOf(EnumSet.of(BIG_QUEUE, KVSTORE_ROCKSDB, KVSTORE_ROCKSDB_STATS)); private final String name; private final boolean pantheonSpecific; diff --git a/metrics/src/main/java/tech/pegasys/pantheon/metrics/prometheus/PrometheusMetricsSystem.java b/metrics/src/main/java/tech/pegasys/pantheon/metrics/prometheus/PrometheusMetricsSystem.java index 979f8db316..0348699965 100644 --- a/metrics/src/main/java/tech/pegasys/pantheon/metrics/prometheus/PrometheusMetricsSystem.java +++ b/metrics/src/main/java/tech/pegasys/pantheon/metrics/prometheus/PrometheusMetricsSystem.java @@ -96,7 +96,7 @@ public class PrometheusMetricsSystem implements MetricsSystem { (k) -> { if (enabledCategories.contains(category)) { final Counter counter = Counter.build(metricName, help).labelNames(labelNames).create(); - addCollector(category, counter); + addCollectorUnchecked(category, counter); return new PrometheusCounter(counter); } else { return NoOpMetricsSystem.getCounterLabelledMetric(labelNames.length); @@ -125,7 +125,7 @@ public class PrometheusMetricsSystem implements MetricsSystem { .quantile(1.0, 0) .labelNames(labelNames) .create(); - addCollector(category, summary); + addCollectorUnchecked(category, summary); return new PrometheusTimer(summary); } else { return NoOpMetricsSystem.getOperationTimerLabelledMetric(labelNames.length); @@ -142,11 +142,17 @@ public class PrometheusMetricsSystem implements MetricsSystem { final String metricName = convertToPrometheusName(category, name); if (enabledCategories.contains(category)) { final Collector collector = new CurrentValueCollector(metricName, help, valueSupplier); - addCollector(category, collector); + addCollectorUnchecked(category, collector); } } - private void addCollector(final MetricCategory category, final Collector metric) { + public void addCollector(final MetricCategory category, final Collector metric) { + if (enabledCategories.contains(category)) { + addCollectorUnchecked(category, metric); + } + } + + private void addCollectorUnchecked(final MetricCategory category, final Collector metric) { metric.register(registry); collectors .computeIfAbsent(category, key -> Collections.newSetFromMap(new ConcurrentHashMap<>())) @@ -213,7 +219,7 @@ public class PrometheusMetricsSystem implements MetricsSystem { labelValues); } - private String convertToPrometheusName(final MetricCategory category, final String name) { + public static String convertToPrometheusName(final MetricCategory category, final String name) { return prometheusPrefix(category) + name; } @@ -222,7 +228,7 @@ public class PrometheusMetricsSystem implements MetricsSystem { return metricName.startsWith(prefix) ? metricName.substring(prefix.length()) : metricName; } - private String prometheusPrefix(final MetricCategory category) { + private static String prometheusPrefix(final MetricCategory category) { return category.isPantheonSpecific() ? PANTHEON_PREFIX + category.getName() + "_" : category.getName() + "_"; diff --git a/services/kvstore/build.gradle b/services/kvstore/build.gradle index 4a12739449..457280d8d9 100644 --- a/services/kvstore/build.gradle +++ b/services/kvstore/build.gradle @@ -27,13 +27,15 @@ jar { dependencies { api project(':util') + implementation project(':metrics') implementation project(':services:util') - implementation 'org.apache.logging.log4j:log4j-api' implementation 'com.google.guava:guava' - implementation 'org.rocksdb:rocksdbjni' implementation 'info.picocli:picocli' + implementation 'io.prometheus:simpleclient' + implementation 'org.apache.logging.log4j:log4j-api' + implementation 'org.rocksdb:rocksdbjni' runtime 'org.apache.logging.log4j:log4j-core' diff --git a/services/kvstore/src/main/java/tech/pegasys/pantheon/services/kvstore/RocksDBStats.java b/services/kvstore/src/main/java/tech/pegasys/pantheon/services/kvstore/RocksDBStats.java new file mode 100644 index 0000000000..37a6a5c872 --- /dev/null +++ b/services/kvstore/src/main/java/tech/pegasys/pantheon/services/kvstore/RocksDBStats.java @@ -0,0 +1,210 @@ +/* + * Copyright 2019 ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ +package tech.pegasys.pantheon.services.kvstore; + +import static tech.pegasys.pantheon.metrics.MetricCategory.KVSTORE_ROCKSDB_STATS; + +import tech.pegasys.pantheon.metrics.prometheus.PrometheusMetricsSystem; + +import java.util.Arrays; +import java.util.Collections; +import java.util.List; + +import io.prometheus.client.Collector; +import org.rocksdb.HistogramData; +import org.rocksdb.HistogramType; +import org.rocksdb.Statistics; +import org.rocksdb.TickerType; + +class RocksDBStats { + + static final List LABELS = Collections.singletonList("quantile"); + static final List LABEL_50 = Collections.singletonList("0.5"); + static final List LABEL_95 = Collections.singletonList("0.95"); + static final List LABEL_99 = Collections.singletonList("0.99"); + + // Tickers - RocksDB equivalent of counters + static final TickerType[] TICKERS = { + TickerType.BLOCK_CACHE_ADD, + TickerType.BLOCK_CACHE_HIT, + TickerType.BLOCK_CACHE_ADD_FAILURES, + TickerType.BLOCK_CACHE_INDEX_MISS, + TickerType.BLOCK_CACHE_INDEX_HIT, + TickerType.BLOCK_CACHE_INDEX_ADD, + TickerType.BLOCK_CACHE_INDEX_BYTES_INSERT, + TickerType.BLOCK_CACHE_INDEX_BYTES_EVICT, + TickerType.BLOCK_CACHE_FILTER_MISS, + TickerType.BLOCK_CACHE_FILTER_HIT, + TickerType.BLOCK_CACHE_FILTER_ADD, + TickerType.BLOCK_CACHE_FILTER_BYTES_INSERT, + TickerType.BLOCK_CACHE_FILTER_BYTES_EVICT, + TickerType.BLOCK_CACHE_DATA_MISS, + TickerType.BLOCK_CACHE_DATA_HIT, + TickerType.BLOCK_CACHE_DATA_ADD, + TickerType.BLOCK_CACHE_DATA_BYTES_INSERT, + TickerType.BLOCK_CACHE_BYTES_READ, + TickerType.BLOCK_CACHE_BYTES_WRITE, + TickerType.BLOOM_FILTER_USEFUL, + TickerType.PERSISTENT_CACHE_HIT, + TickerType.PERSISTENT_CACHE_MISS, + TickerType.SIM_BLOCK_CACHE_HIT, + TickerType.SIM_BLOCK_CACHE_MISS, + TickerType.MEMTABLE_HIT, + TickerType.MEMTABLE_MISS, + TickerType.GET_HIT_L0, + TickerType.GET_HIT_L1, + TickerType.GET_HIT_L2_AND_UP, + TickerType.COMPACTION_KEY_DROP_NEWER_ENTRY, + TickerType.COMPACTION_KEY_DROP_OBSOLETE, + TickerType.COMPACTION_KEY_DROP_RANGE_DEL, + TickerType.COMPACTION_KEY_DROP_USER, + TickerType.COMPACTION_RANGE_DEL_DROP_OBSOLETE, + TickerType.NUMBER_KEYS_WRITTEN, + TickerType.NUMBER_KEYS_READ, + TickerType.NUMBER_KEYS_UPDATED, + TickerType.BYTES_WRITTEN, + TickerType.BYTES_READ, + TickerType.NUMBER_DB_SEEK, + TickerType.NUMBER_DB_NEXT, + TickerType.NUMBER_DB_PREV, + TickerType.NUMBER_DB_SEEK_FOUND, + TickerType.NUMBER_DB_NEXT_FOUND, + TickerType.NUMBER_DB_PREV_FOUND, + TickerType.ITER_BYTES_READ, + TickerType.NO_FILE_CLOSES, + TickerType.NO_FILE_OPENS, + TickerType.NO_FILE_ERRORS, + // TickerType.STALL_L0_SLOWDOWN_MICROS, + // TickerType.STALL_MEMTABLE_COMPACTION_MICROS, + // TickerType.STALL_L0_NUM_FILES_MICROS, + TickerType.STALL_MICROS, + TickerType.DB_MUTEX_WAIT_MICROS, + TickerType.RATE_LIMIT_DELAY_MILLIS, + TickerType.NO_ITERATORS, + TickerType.NUMBER_MULTIGET_BYTES_READ, + TickerType.NUMBER_MULTIGET_KEYS_READ, + TickerType.NUMBER_MULTIGET_CALLS, + TickerType.NUMBER_FILTERED_DELETES, + TickerType.NUMBER_MERGE_FAILURES, + TickerType.BLOOM_FILTER_PREFIX_CHECKED, + TickerType.BLOOM_FILTER_PREFIX_USEFUL, + TickerType.NUMBER_OF_RESEEKS_IN_ITERATION, + TickerType.GET_UPDATES_SINCE_CALLS, + TickerType.BLOCK_CACHE_COMPRESSED_MISS, + TickerType.BLOCK_CACHE_COMPRESSED_HIT, + TickerType.BLOCK_CACHE_COMPRESSED_ADD, + TickerType.BLOCK_CACHE_COMPRESSED_ADD_FAILURES, + TickerType.WAL_FILE_SYNCED, + TickerType.WAL_FILE_BYTES, + TickerType.WRITE_DONE_BY_SELF, + TickerType.WRITE_DONE_BY_OTHER, + TickerType.WRITE_TIMEDOUT, + TickerType.WRITE_WITH_WAL, + TickerType.COMPACT_READ_BYTES, + TickerType.COMPACT_WRITE_BYTES, + TickerType.FLUSH_WRITE_BYTES, + TickerType.NUMBER_DIRECT_LOAD_TABLE_PROPERTIES, + TickerType.NUMBER_SUPERVERSION_ACQUIRES, + TickerType.NUMBER_SUPERVERSION_RELEASES, + TickerType.NUMBER_SUPERVERSION_CLEANUPS, + TickerType.NUMBER_BLOCK_COMPRESSED, + TickerType.NUMBER_BLOCK_DECOMPRESSED, + TickerType.NUMBER_BLOCK_NOT_COMPRESSED, + TickerType.MERGE_OPERATION_TOTAL_TIME, + TickerType.FILTER_OPERATION_TOTAL_TIME, + TickerType.ROW_CACHE_HIT, + TickerType.ROW_CACHE_MISS, + TickerType.READ_AMP_ESTIMATE_USEFUL_BYTES, + TickerType.READ_AMP_TOTAL_READ_BYTES, + TickerType.NUMBER_RATE_LIMITER_DRAINS, + TickerType.NUMBER_ITER_SKIP, + TickerType.NUMBER_MULTIGET_KEYS_FOUND, + }; + + // Histograms - treated as prometheus summaries + static final HistogramType[] HISTOGRAMS = { + HistogramType.DB_GET, + HistogramType.DB_WRITE, + HistogramType.COMPACTION_TIME, + HistogramType.SUBCOMPACTION_SETUP_TIME, + HistogramType.TABLE_SYNC_MICROS, + HistogramType.COMPACTION_OUTFILE_SYNC_MICROS, + HistogramType.WAL_FILE_SYNC_MICROS, + HistogramType.MANIFEST_FILE_SYNC_MICROS, + HistogramType.TABLE_OPEN_IO_MICROS, + HistogramType.DB_MULTIGET, + HistogramType.READ_BLOCK_COMPACTION_MICROS, + HistogramType.READ_BLOCK_GET_MICROS, + HistogramType.WRITE_RAW_BLOCK_MICROS, + HistogramType.STALL_L0_SLOWDOWN_COUNT, + HistogramType.STALL_MEMTABLE_COMPACTION_COUNT, + HistogramType.STALL_L0_NUM_FILES_COUNT, + HistogramType.HARD_RATE_LIMIT_DELAY_COUNT, + HistogramType.SOFT_RATE_LIMIT_DELAY_COUNT, + HistogramType.NUM_FILES_IN_SINGLE_COMPACTION, + HistogramType.DB_SEEK, + HistogramType.WRITE_STALL, + HistogramType.SST_READ_MICROS, + HistogramType.NUM_SUBCOMPACTIONS_SCHEDULED, + HistogramType.BYTES_PER_READ, + HistogramType.BYTES_PER_WRITE, + HistogramType.BYTES_PER_MULTIGET, + HistogramType.BYTES_COMPRESSED, + HistogramType.BYTES_DECOMPRESSED, + HistogramType.COMPRESSION_TIMES_NANOS, + HistogramType.DECOMPRESSION_TIMES_NANOS, + HistogramType.READ_NUM_MERGE_OPERANDS, + }; + + static void registerRocksDBMetrics( + final Statistics stats, final PrometheusMetricsSystem metricsSystem) { + + for (final TickerType ticker : TICKERS) { + final String promCounterName = ticker.name().toLowerCase(); + metricsSystem.createLongGauge( + KVSTORE_ROCKSDB_STATS, + promCounterName, + "RocksDB reported statistics for " + ticker.name(), + () -> stats.getTickerCount(ticker)); + } + + for (final HistogramType histogram : HISTOGRAMS) { + metricsSystem.addCollector(KVSTORE_ROCKSDB_STATS, histogramToCollector(stats, histogram)); + } + } + + private static Collector histogramToCollector( + final Statistics stats, final HistogramType histogram) { + return new Collector() { + final String metricName = + PrometheusMetricsSystem.convertToPrometheusName( + KVSTORE_ROCKSDB_STATS, histogram.name().toLowerCase()); + + @Override + public List collect() { + final HistogramData data = stats.getHistogramData(histogram); + return Collections.singletonList( + new MetricFamilySamples( + metricName, + Type.SUMMARY, + "RocksDB histogram for " + metricName, + Arrays.asList( + new MetricFamilySamples.Sample(metricName, LABELS, LABEL_50, data.getMedian()), + new MetricFamilySamples.Sample( + metricName, LABELS, LABEL_95, data.getPercentile95()), + new MetricFamilySamples.Sample( + metricName, LABELS, LABEL_99, data.getPercentile99())))); + } + }; + } +} diff --git a/services/kvstore/src/main/java/tech/pegasys/pantheon/services/kvstore/RocksDbKeyValueStorage.java b/services/kvstore/src/main/java/tech/pegasys/pantheon/services/kvstore/RocksDbKeyValueStorage.java index 4e8e8d58e9..38dc8b419b 100644 --- a/services/kvstore/src/main/java/tech/pegasys/pantheon/services/kvstore/RocksDbKeyValueStorage.java +++ b/services/kvstore/src/main/java/tech/pegasys/pantheon/services/kvstore/RocksDbKeyValueStorage.java @@ -16,6 +16,7 @@ import tech.pegasys.pantheon.metrics.Counter; import tech.pegasys.pantheon.metrics.MetricCategory; import tech.pegasys.pantheon.metrics.MetricsSystem; import tech.pegasys.pantheon.metrics.OperationTimer; +import tech.pegasys.pantheon.metrics.prometheus.PrometheusMetricsSystem; import tech.pegasys.pantheon.services.util.RocksDbUtil; import tech.pegasys.pantheon.util.bytes.BytesValue; @@ -27,6 +28,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.rocksdb.Options; import org.rocksdb.RocksDBException; +import org.rocksdb.Statistics; import org.rocksdb.TransactionDB; import org.rocksdb.TransactionDBOptions; import org.rocksdb.WriteOptions; @@ -45,6 +47,7 @@ public class RocksDbKeyValueStorage implements KeyValueStorage, Closeable { private final OperationTimer writeLatency; private final OperationTimer commitLatency; private final Counter rollbackCount; + private final Statistics stats; public static KeyValueStorage create( final RocksDbConfiguration rocksDbConfiguration, final MetricsSystem metricsSystem) @@ -56,33 +59,44 @@ public class RocksDbKeyValueStorage implements KeyValueStorage, Closeable { final RocksDbConfiguration rocksDbConfiguration, final MetricsSystem metricsSystem) { RocksDbUtil.loadNativeLibrary(); try { - + stats = new Statistics(); options = new Options() .setCreateIfMissing(true) .setMaxOpenFiles(rocksDbConfiguration.getMaxOpenFiles()) - .setTableFormatConfig(rocksDbConfiguration.getBlockBasedTableConfig()); + .setTableFormatConfig(rocksDbConfiguration.getBlockBasedTableConfig()) + .setStatistics(stats); txOptions = new TransactionDBOptions(); db = TransactionDB.open(options, txOptions, rocksDbConfiguration.getDatabaseDir().toString()); readLatency = metricsSystem.createTimer( - MetricCategory.ROCKSDB, "read_latency_seconds", "Latency for read from RocksDB."); + MetricCategory.KVSTORE_ROCKSDB, + "read_latency_seconds", + "Latency for read from RocksDB."); removeLatency = metricsSystem.createTimer( - MetricCategory.ROCKSDB, + MetricCategory.KVSTORE_ROCKSDB, "remove_latency_seconds", "Latency of remove requests from RocksDB."); writeLatency = metricsSystem.createTimer( - MetricCategory.ROCKSDB, "write_latency_seconds", "Latency for write to RocksDB."); + MetricCategory.KVSTORE_ROCKSDB, + "write_latency_seconds", + "Latency for write to RocksDB."); commitLatency = metricsSystem.createTimer( - MetricCategory.ROCKSDB, "commit_latency_seconds", "Latency for commits to RocksDB."); + MetricCategory.KVSTORE_ROCKSDB, + "commit_latency_seconds", + "Latency for commits to RocksDB."); + + if (metricsSystem instanceof PrometheusMetricsSystem) { + RocksDBStats.registerRocksDBMetrics(stats, (PrometheusMetricsSystem) metricsSystem); + } metricsSystem.createLongGauge( - MetricCategory.ROCKSDB, + MetricCategory.KVSTORE_ROCKSDB, "rocks_db_table_readers_memory_bytes", "Estimated memory used for RocksDB index and filter blocks in bytes", () -> { @@ -96,7 +110,7 @@ public class RocksDbKeyValueStorage implements KeyValueStorage, Closeable { rollbackCount = metricsSystem.createCounter( - MetricCategory.ROCKSDB, + MetricCategory.KVSTORE_ROCKSDB, "rollback_count", "Number of RocksDB transactions rolled back."); } catch (final RocksDBException e) {