[PRIV-44] Use single instance of Rocksdb for privacy (#1247)

* Use single Db for private states

* Update Rocksdb options to use a common cache

Signed-off-by: Adrian Sutton <adrian.sutton@consensys.net>

ethereum/core/src/main/java/tech/pegasys/pantheon/ethereum/core/PrivacyParameters.java

@@ -136,7 +136,6 @@ public class PrivacyParameters {
 
   public static class Builder {
     private final String PRIVATE_DATABASE_PATH = "private";
-    private final String PRIVATE_STATE_DATABASE_PATH = "privateState";
 
     private boolean enabled;
     private URI enclaveUrl;
@@ -184,15 +183,10 @@ public class PrivacyParameters {
         WorldStateArchive privateWorldStateArchive =
             new WorldStateArchive(privateWorldStateStorage);
 
-        Path privateStateDbPath = dataDir.resolve(PRIVATE_STATE_DATABASE_PATH);
-        StorageProvider privateStateStorageProvider =
-            RocksDbStorageProvider.create(
-                new RocksDbConfiguration.Builder().databaseDir(privateStateDbPath).build(),
-                metricsSystem);
         PrivateTransactionStorage privateTransactionStorage =
-            privateStateStorageProvider.createPrivateTransactionStorage();
+            privateStorageProvider.createPrivateTransactionStorage();
         PrivateStateStorage privateStateStorage =
-            privateStateStorageProvider.createPrivateStateStorage();
+            privateStorageProvider.createPrivateStateStorage();
 
         config.setPrivateWorldStateArchive(privateWorldStateArchive);
         config.setEnclavePublicKey(enclavePublicKey);

services/kvstore/src/main/java/tech/pegasys/pantheon/services/kvstore/RocksDbConfiguration.java

@@ -12,18 +12,26 @@
  */
 package tech.pegasys.pantheon.services.kvstore;
 
+import tech.pegasys.pantheon.services.util.RocksDbUtil;
+
 import java.nio.file.Path;
 
+import org.rocksdb.BlockBasedTableConfig;
+import org.rocksdb.LRUCache;
 import picocli.CommandLine;
 
 public class RocksDbConfiguration {
 
   private final Path databaseDir;
   private final int maxOpenFiles;
+  private final BlockBasedTableConfig blockBasedTableConfig;
 
-  public RocksDbConfiguration(final Path databaseDir, final int maxOpenFiles) {
+  public RocksDbConfiguration(
+      final Path databaseDir, final int maxOpenFiles, final LRUCache cache) {
+    RocksDbUtil.loadNativeLibrary();
     this.databaseDir = databaseDir;
     this.maxOpenFiles = maxOpenFiles;
+    this.blockBasedTableConfig = new BlockBasedTableConfig().setBlockCache(cache);
   }
 
   public Path getDatabaseDir() {
@@ -34,9 +42,14 @@ public class RocksDbConfiguration {
     return maxOpenFiles;
   }
 
+  public BlockBasedTableConfig getBlockBasedTableConfig() {
+    return blockBasedTableConfig;
+  }
+
   public static class Builder {
 
     Path databaseDir;
+    LRUCache cache = null;
 
     @CommandLine.Option(
         names = {"--Xrocksdb-max-open-files"},
@@ -46,6 +59,14 @@ public class RocksDbConfiguration {
         description = "Max number of files RocksDB will open (default: ${DEFAULT-VALUE})")
     int maxOpenFiles;
 
+    @CommandLine.Option(
+        names = {"--Xrocksdb-cache-capacity"},
+        hidden = true,
+        defaultValue = "8388608",
+        paramLabel = "<LONG>",
+        description = "Cache capacity of RocksDB (default: ${DEFAULT-VALUE})")
+    long cacheCapacity;
+
     public Builder databaseDir(final Path databaseDir) {
       this.databaseDir = databaseDir;
       return this;
@@ -56,8 +77,21 @@ public class RocksDbConfiguration {
       return this;
     }
 
+    public Builder cacheCapacity(final long cacheCapacity) {
+      this.cacheCapacity = cacheCapacity;
+      return this;
+    }
+
+    private LRUCache createCache(final long cacheCapacity) {
+      RocksDbUtil.loadNativeLibrary();
+      return new LRUCache(cacheCapacity);
+    }
+
     public RocksDbConfiguration build() {
-      return new RocksDbConfiguration(databaseDir, maxOpenFiles);
+      if (cache == null) {
+        cache = createCache(cacheCapacity);
+      }
+      return new RocksDbConfiguration(databaseDir, maxOpenFiles, cache);
     }
   }
 }

services/kvstore/src/main/java/tech/pegasys/pantheon/services/kvstore/RocksDbKeyValueStorage.java

@@ -60,7 +60,9 @@ public class RocksDbKeyValueStorage implements KeyValueStorage, Closeable {
       options =
           new Options()
               .setCreateIfMissing(true)
-              .setMaxOpenFiles(rocksDbConfiguration.getMaxOpenFiles());
+              .setMaxOpenFiles(rocksDbConfiguration.getMaxOpenFiles())
+              .setTableFormatConfig(rocksDbConfiguration.getBlockBasedTableConfig());
+
       txOptions = new TransactionDBOptions();
       db = TransactionDB.open(options, txOptions, rocksDbConfiguration.getDatabaseDir().toString());