Refactor RocksDBStats (#1266)

Move RocksDBStats to it's own module.  This also brings metrics to
metrics:core since none of our other module have nested modules but
they have peer modules.


Signed-off-by: Adrian Sutton <adrian.sutton@consensys.net>

acceptance-tests/build.gradle

@@ -27,7 +27,7 @@ dependencies {
   testImplementation project(':ethereum:jsonrpc')
   testImplementation project(':ethereum:permissioning')
   testImplementation project(':ethereum:rlp')
-  testImplementation project(':metrics')
+  testImplementation project(':metrics:core')
   testImplementation project(':pantheon')
   testImplementation project(':services:kvstore')
   testImplementation project(':testutil')

consensus/clique/build.gradle

@@ -47,7 +47,7 @@ dependencies {
   testImplementation project(path: ':ethereum:core', configuration: 'testSupportArtifacts')
   testImplementation project(path: ':consensus:common', configuration: 'testArtifacts')
   testImplementation project(':testutil')
-  testImplementation project(':metrics')
+  testImplementation project(':metrics:core')
 
   testImplementation 'junit:junit'
   testImplementation 'org.assertj:assertj-core'

consensus/ibft/build.gradle

@@ -47,9 +47,9 @@ dependencies {
   testImplementation project(path: ':config:', configuration: 'testSupportArtifacts')
   testImplementation project(path: ':consensus:common', configuration: 'testArtifacts')
   testImplementation project(':testutil')
-  testImplementation project(':metrics')
+  testImplementation project(':metrics:core')
 
-  integrationTestImplementation project(':metrics')
+  integrationTestImplementation project(':metrics:core')
   integrationTestImplementation 'junit:junit'
   integrationTestImplementation 'org.assertj:assertj-core'
   integrationTestImplementation 'org.mockito:mockito-core'

consensus/ibftlegacy/build.gradle

@@ -23,7 +23,7 @@ dependencies {
   implementation project(':ethereum:jsonrpc')
   implementation project(':ethereum:rlp')
   implementation project(':ethereum:p2p')
-  implementation project(':metrics')
+  implementation project(':metrics:core')
   implementation project(':services:kvstore')
 
   implementation 'com.google.guava:guava'
@@ -32,7 +32,7 @@ dependencies {
   testImplementation project(path: ':ethereum:core', configuration: 'testSupportArtifacts')
   testImplementation project(path: ':consensus:ibft', configuration: 'testSupportArtifacts')
   testImplementation project(':testutil')
-  testImplementation project(':metrics')
+  testImplementation project(':metrics:core')
 
   testImplementation 'junit:junit'
   testImplementation 'org.assertj:assertj-core'

ethereum/blockcreation/build.gradle

@@ -27,7 +27,7 @@ dependencies {
   testImplementation project(path: ':ethereum:core', configuration: 'testSupportArtifacts')
   testImplementation project(path: ':ethereum:core', configuration: 'testArtifacts')
   testImplementation project(':testutil')
-  testImplementation project(':metrics')
+  testImplementation project(':metrics:core')
   testImplementation 'junit:junit'
   testImplementation 'org.assertj:assertj-core'
   testImplementation 'org.awaitility:awaitility'

ethereum/core/build.gradle

@@ -31,7 +31,7 @@ dependencies {
   implementation project(':enclave')
   implementation project(':ethereum:rlp')
   implementation project(':ethereum:trie')
-  implementation project(':metrics')
+  implementation project(':metrics:core')
   implementation project(':services:kvstore')
 
   implementation 'com.fasterxml.jackson.core:jackson-databind'
@@ -71,7 +71,7 @@ dependencies {
   jmhImplementation project(path: ':ethereum:core', configuration: 'testSupportArtifacts')
   jmhImplementation project(':ethereum:rlp')
   jmhImplementation project(':ethereum:trie')
-  jmhImplementation project(':metrics')
+  jmhImplementation project(':metrics:core')
   jmhImplementation project(':services:kvstore')
   jmhImplementation project(':util')
 

ethereum/eth/build.gradle

@@ -31,7 +31,7 @@ dependencies {
   implementation project(':ethereum:rlp')
   implementation project(':ethereum:trie')
   implementation project(':ethereum:permissioning')
-  implementation project(':metrics')
+  implementation project(':metrics:core')
   implementation project(':services:kvstore')
   implementation project(':services:pipeline')
   implementation project(':services:tasks')

ethereum/jsonrpc/build.gradle

@@ -35,7 +35,7 @@ dependencies {
   implementation project(':ethereum:p2p')
   implementation project(':ethereum:rlp')
   implementation project(':ethereum:permissioning')
-  implementation project(':metrics')
+  implementation project(':metrics:core')
 
   implementation 'com.google.guava:guava'
   implementation 'io.vertx:vertx-core'

ethereum/p2p/build.gradle

@@ -30,7 +30,7 @@ dependencies {
   implementation project(':ethereum:core')
   implementation project(':ethereum:permissioning')
   implementation project(':ethereum:rlp')
-  implementation project(':metrics')
+  implementation project(':metrics:core')
 
   implementation 'com.google.guava:guava'
   implementation 'io.prometheus:simpleclient'

gradle/check-licenses.gradle

@@ -127,6 +127,7 @@ downloadLicenses {
       (group('pantheon.ethereum'))   : apache,
       (group('pantheon.services'))   : apache,
       (group('pantheon.consensus'))  : apache,
+      (group('pantheon.metrics'))  : apache,
 
       // https://checkerframework.org/manual/#license
       // The more permissive MIT License applies to code that you might want

metrics/build.gradle

@@ -11,37 +11,4 @@
  * specific language governing permissions and limitations under the License.
  */
 
-apply plugin: 'java-library'
+jar { enabled = false }
-
-jar {
-  baseName 'pantheon-metrics'
-  manifest {
-    attributes(
-      'Specification-Title': baseName,
-      'Specification-Version': project.version,
-      'Implementation-Title': baseName,
-      'Implementation-Version': calculateVersion()
-      )
-  }
-}
-
-dependencies {
-  implementation project(':util')
-
-  implementation 'com.google.guava:guava'
-  implementation 'io.prometheus:simpleclient'
-  implementation 'io.prometheus:simpleclient_common'
-  implementation 'io.prometheus:simpleclient_hotspot'
-  implementation 'io.prometheus:simpleclient_pushgateway'
-  implementation 'io.vertx:vertx-core'
-  implementation 'io.vertx:vertx-web'
-  implementation 'org.apache.logging.log4j:log4j-api'
-
-  runtime 'org.apache.logging.log4j:log4j-core'
-
-  // test dependencies.
-  testImplementation 'junit:junit'
-  testImplementation 'org.assertj:assertj-core'
-  testImplementation 'org.mockito:mockito-core'
-  testImplementation 'com.squareup.okhttp3:okhttp'
-}

metrics/core/build.gradle

@@ -0,0 +1,47 @@
+/*
+ * Copyright 2018 ConsenSys AG.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+ * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations under the License.
+ */
+
+apply plugin: 'java-library'
+
+jar {
+  baseName 'pantheon-metrics-core'
+  manifest {
+    attributes(
+      'Specification-Title': baseName,
+      'Specification-Version': project.version,
+      'Implementation-Title': baseName,
+      'Implementation-Version': calculateVersion()
+      )
+  }
+}
+
+dependencies {
+  implementation project(':util')
+
+  implementation 'com.google.guava:guava'
+  implementation 'io.prometheus:simpleclient'
+  implementation 'io.prometheus:simpleclient_common'
+  implementation 'io.prometheus:simpleclient_hotspot'
+  implementation 'io.prometheus:simpleclient_pushgateway'
+  implementation 'io.vertx:vertx-core'
+  implementation 'io.vertx:vertx-web'
+  implementation 'org.apache.logging.log4j:log4j-api'
+
+  runtime 'org.apache.logging.log4j:log4j-core'
+
+  // test dependencies.
+  testImplementation 'junit:junit'
+  testImplementation 'org.assertj:assertj-core'
+  testImplementation 'org.mockito:mockito-core'
+  testImplementation 'com.squareup.okhttp3:okhttp'
+}

metrics/rocksdb/build.gradle

@@ -0,0 +1,37 @@
+/*
+ * Copyright 2018 ConsenSys AG.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+ * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations under the License.
+ */
+
+apply plugin: 'java-library'
+
+jar {
+  baseName 'pantheon-metrics-rocksdb'
+  manifest {
+    attributes(
+      'Specification-Title': baseName,
+      'Specification-Version': project.version,
+      'Implementation-Title': baseName,
+      'Implementation-Version': calculateVersion()
+      )
+  }
+}
+
+dependencies {
+  implementation project(':metrics:core')
+  implementation project(':services:util')
+
+  implementation 'com.google.guava:guava'
+  implementation 'io.prometheus:simpleclient'
+  implementation 'info.picocli:picocli'
+  implementation 'org.apache.logging.log4j:log4j-api'
+  implementation 'org.rocksdb:rocksdbjni'
+}

metrics/rocksdb/src/main/java/tech/pegasys/pantheon/metrics/rocksdb/RocksDBStats.java

@@ -10,7 +10,7 @@
  * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
  * specific language governing permissions and limitations under the License.
  */
-package tech.pegasys.pantheon.services.kvstore;
+package tech.pegasys.pantheon.metrics.rocksdb;
 
 import static tech.pegasys.pantheon.metrics.MetricCategory.KVSTORE_ROCKSDB_STATS;
 
@@ -26,7 +26,7 @@ import org.rocksdb.HistogramType;
 import org.rocksdb.Statistics;
 import org.rocksdb.TickerType;
 
-class RocksDBStats {
+public class RocksDBStats {
 
   static final List<String> LABELS = Collections.singletonList("quantile");
   static final List<String> LABEL_50 = Collections.singletonList("0.5");
@@ -166,7 +166,7 @@ class RocksDBStats {
     HistogramType.READ_NUM_MERGE_OPERANDS,
   };
 
-  static void registerRocksDBMetrics(
+  public static void registerRocksDBMetrics(
       final Statistics stats, final PrometheusMetricsSystem metricsSystem) {
 
     for (final TickerType ticker : TICKERS) {

pantheon/build.gradle

@@ -39,7 +39,7 @@ dependencies {
   implementation project(':ethereum:permissioning')
   implementation project(':ethereum:p2p')
   implementation project(':ethereum:rlp')
-  implementation project(':metrics')
+  implementation project(':metrics:core')
   implementation project(':enclave')
   implementation project(':services:kvstore')
 

services/kvstore/build.gradle

@@ -28,7 +28,8 @@ jar {
 dependencies {
   api project(':util')
 
-  implementation project(':metrics')
+  implementation project(':metrics:core')
+  implementation project(':metrics:rocksdb')
   implementation project(':services:util')
 
   implementation 'com.google.guava:guava'

services/kvstore/src/main/java/tech/pegasys/pantheon/services/kvstore/RocksDbKeyValueStorage.java

@@ -16,6 +16,8 @@ import tech.pegasys.pantheon.metrics.Counter;
 import tech.pegasys.pantheon.metrics.MetricCategory;
 import tech.pegasys.pantheon.metrics.MetricsSystem;
 import tech.pegasys.pantheon.metrics.OperationTimer;
+import tech.pegasys.pantheon.metrics.prometheus.PrometheusMetricsSystem;
+import tech.pegasys.pantheon.metrics.rocksdb.RocksDBStats;
 import tech.pegasys.pantheon.services.util.RocksDbUtil;
 import tech.pegasys.pantheon.util.bytes.BytesValue;
 
@@ -75,7 +77,7 @@ public class RocksDbKeyValueStorage implements KeyValueStorage, Closeable {
                   MetricCategory.KVSTORE_ROCKSDB,
                   "read_latency_seconds",
                   "Latency for read from RocksDB.",
-                  rocksDbConfiguration.getLabel())
+                  "database")
               .labels(rocksDbConfiguration.getLabel());
       removeLatency =
           metricsSystem
@@ -83,7 +85,7 @@ public class RocksDbKeyValueStorage implements KeyValueStorage, Closeable {
                   MetricCategory.KVSTORE_ROCKSDB,
                   "remove_latency_seconds",
                   "Latency of remove requests from RocksDB.",
-                  rocksDbConfiguration.getLabel())
+                  "database")
               .labels(rocksDbConfiguration.getLabel());
       writeLatency =
           metricsSystem
@@ -91,7 +93,7 @@ public class RocksDbKeyValueStorage implements KeyValueStorage, Closeable {
                   MetricCategory.KVSTORE_ROCKSDB,
                   "write_latency_seconds",
                   "Latency for write to RocksDB.",
-                  rocksDbConfiguration.getLabel())
+                  "database")
               .labels(rocksDbConfiguration.getLabel());
       commitLatency =
           metricsSystem
@@ -99,9 +101,13 @@ public class RocksDbKeyValueStorage implements KeyValueStorage, Closeable {
                   MetricCategory.KVSTORE_ROCKSDB,
                   "commit_latency_seconds",
                   "Latency for commits to RocksDB.",
-                  rocksDbConfiguration.getLabel())
+                  "database")
               .labels(rocksDbConfiguration.getLabel());
 
+      if (metricsSystem instanceof PrometheusMetricsSystem) {
+        RocksDBStats.registerRocksDBMetrics(stats, (PrometheusMetricsSystem) metricsSystem);
+      }
+
       metricsSystem.createLongGauge(
           MetricCategory.KVSTORE_ROCKSDB,
           "rocks_db_table_readers_memory_bytes",
@@ -121,7 +127,7 @@ public class RocksDbKeyValueStorage implements KeyValueStorage, Closeable {
                   MetricCategory.KVSTORE_ROCKSDB,
                   "rollback_count",
                   "Number of RocksDB transactions rolled back.",
-                  rocksDbConfiguration.getLabel())
+                  "database")
               .labels(rocksDbConfiguration.getLabel());
     } catch (final RocksDBException e) {
       throw new StorageException(e);

services/pipeline/build.gradle

@@ -27,7 +27,7 @@ jar {
 
 dependencies {
   api project(':util')
-  implementation project(':metrics')
+  implementation project(':metrics:core')
 
   implementation 'org.apache.logging.log4j:log4j-api'
   implementation 'com.google.guava:guava'

services/tasks/build.gradle

@@ -30,7 +30,7 @@ dependencies {
 
   compileOnly 'org.openjdk.jmh:jmh-generator-annprocess'
 
-  implementation project(':metrics')
+  implementation project(':metrics:core')
   implementation project(':services:util')
 
   implementation 'org.apache.logging.log4j:log4j-api'

services/util/build.gradle

@@ -27,7 +27,7 @@ jar {
 
 dependencies {
   api project(':util')
-  implementation project(':metrics')
+  implementation project(':metrics:core')
 
   implementation 'org.apache.logging.log4j:log4j-api'
   implementation 'com.google.guava:guava'

settings.gradle

@@ -14,7 +14,6 @@
 rootProject.name='pantheon'
 include 'acceptance-tests'
 include 'config'
-include 'consensus'
 include 'consensus:clique'
 include 'consensus:common'
 include 'consensus:ibft'
@@ -32,7 +31,8 @@ include 'ethereum:permissioning'
 include 'ethereum:referencetests'
 include 'ethereum:rlp'
 include 'ethereum:trie'
-include 'metrics'
+include 'metrics:core'
+include 'metrics:rocksdb'
 include 'pantheon'
 include 'services:kvstore'
 include 'services:pipeline'