rocksdb usage - Replace Column Size with more accurate Total Size (#6540)

Add rocksdb x-stats command to print various stats for each column family

---------

Signed-off-by: Simon Dudley <simon.dudley@consensys.net>

CHANGELOG.md

@@ -30,6 +30,8 @@
 - Support for `shanghaiTime` fork and Shanghai EVM smart contracts in QBFT/IBFT chains [#6353](https://github.com/hyperledger/besu/pull/6353)
 - Change ExecutionHaltReason for contract creation collision case to return ILLEGAL_STATE_CHANGE [#6518](https://github.com/hyperledger/besu/pull/6518) 
 - Experimental feature `--Xbonsai-code-using-code-hash-enabled` for storing Bonsai code storage by code hash [#6505](https://github.com/hyperledger/besu/pull/6505)
+- More accurate column size `storage rocksdb usage` subcommand [#6540](https://github.com/hyperledger/besu/pull/6540)
+- Adds `storage rocksdb x-stats` subcommand [#6540](https://github.com/hyperledger/besu/pull/6540)
 
 ### Bug fixes
 - Fix the way an advertised host configured with `--p2p-host` is treated when communicating with the originator of a PING packet [#6225](https://github.com/hyperledger/besu/pull/6225)

besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/RocksDbHelper.java

@@ -21,6 +21,7 @@ import java.io.PrintWriter;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.function.BiConsumer;
+import java.util.stream.Stream;
 
 import org.bouncycastle.util.Arrays;
 import org.rocksdb.ColumnFamilyDescriptor;
@@ -66,36 +67,210 @@ public class RocksDbHelper {
     }
   }
 
-  static void printUsageForColumnFamily(
+  static void printStatsForColumnFamily(
       final RocksDB rocksdb, final ColumnFamilyHandle cfHandle, final PrintWriter out)
-      throws RocksDBException, NumberFormatException {
+      throws RocksDBException {
     final String size = rocksdb.getProperty(cfHandle, "rocksdb.estimate-live-data-size");
     final String numberOfKeys = rocksdb.getProperty(cfHandle, "rocksdb.estimate-num-keys");
-    boolean emptyColumnFamily = false;
-    if (!size.isBlank() && !numberOfKeys.isBlank()) {
+    final long sizeLong = Long.parseLong(size);
+    final long numberOfKeysLong = Long.parseLong(numberOfKeys);
+    if (!size.isBlank()
+        && !numberOfKeys.isBlank()
+        && isPopulatedColumnFamily(sizeLong, numberOfKeysLong)) {
+      out.println("=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=");
+      out.println("Column Family: " + getNameById(cfHandle.getName()));
+
+      final String prefix = "rocksdb.";
+      final String cfstats = "cfstats";
+      final String cfstats_no_file_histogram = "cfstats-no-file-histogram";
+      final String cf_file_histogram = "cf-file-histogram";
+      final String cf_write_stall_stats = "cf-write-stall-stats";
+      final String dbstats = "dbstats";
+      final String db_write_stall_stats = "db-write-stall-stats";
+      final String levelstats = "levelstats";
+      final String block_cache_entry_stats = "block-cache-entry-stats";
+      final String fast_block_cache_entry_stats = "fast-block-cache-entry-stats";
+      final String num_immutable_mem_table = "num-immutable-mem-table";
+      final String num_immutable_mem_table_flushed = "num-immutable-mem-table-flushed";
+      final String mem_table_flush_pending = "mem-table-flush-pending";
+      final String compaction_pending = "compaction-pending";
+      final String background_errors = "background-errors";
+      final String cur_size_active_mem_table = "cur-size-active-mem-table";
+      final String cur_size_all_mem_tables = "cur-size-all-mem-tables";
+      final String size_all_mem_tables = "size-all-mem-tables";
+      final String num_entries_active_mem_table = "num-entries-active-mem-table";
+      final String num_entries_imm_mem_tables = "num-entries-imm-mem-tables";
+      final String num_deletes_active_mem_table = "num-deletes-active-mem-table";
+      final String num_deletes_imm_mem_tables = "num-deletes-imm-mem-tables";
+      final String estimate_num_keys = "estimate-num-keys";
+      final String estimate_table_readers_mem = "estimate-table-readers-mem";
+      final String is_file_deletions_enabled = "is-file-deletions-enabled";
+      final String num_snapshots = "num-snapshots";
+      final String oldest_snapshot_time = "oldest-snapshot-time";
+      final String oldest_snapshot_sequence = "oldest-snapshot-sequence";
+      final String num_live_versions = "num-live-versions";
+      final String current_version_number = "current-super-version-number";
+      final String estimate_live_data_size = "estimate-live-data-size";
+      final String min_log_number_to_keep_str = "min-log-number-to-keep";
+      final String min_obsolete_sst_number_to_keep_str = "min-obsolete-sst-number-to-keep";
+      final String base_level_str = "base-level";
+      final String total_sst_files_size = "total-sst-files-size";
+      final String live_sst_files_size = "live-sst-files-size";
+      final String obsolete_sst_files_size = "obsolete-sst-files-size";
+      final String live_sst_files_size_at_temperature = "live-sst-files-size-at-temperature";
+      final String estimate_pending_comp_bytes = "estimate-pending-compaction-bytes";
+      final String aggregated_table_properties = "aggregated-table-properties";
+      final String num_running_compactions = "num-running-compactions";
+      final String num_running_flushes = "num-running-flushes";
+      final String actual_delayed_write_rate = "actual-delayed-write-rate";
+      final String is_write_stopped = "is-write-stopped";
+      final String estimate_oldest_key_time = "estimate-oldest-key-time";
+      final String block_cache_capacity = "block-cache-capacity";
+      final String block_cache_usage = "block-cache-usage";
+      final String block_cache_pinned_usage = "block-cache-pinned-usage";
+      final String options_statistics = "options-statistics";
+      final String num_blob_files = "num-blob-files";
+      final String blob_stats = "blob-stats";
+      final String total_blob_file_size = "total-blob-file-size";
+      final String live_blob_file_size = "live-blob-file-size";
+      final String live_blob_file_garbage_size = "live-blob-file-garbage-size";
+      final String blob_cache_capacity = "blob-cache-capacity";
+      final String blob_cache_usage = "blob-cache-usage";
+      final String blob_cache_pinned_usage = "blob-cache-pinned-usage";
+      Stream.of(
+              cfstats,
+              cfstats_no_file_histogram,
+              cf_file_histogram,
+              cf_write_stall_stats,
+              dbstats,
+              db_write_stall_stats,
+              levelstats,
+              block_cache_entry_stats,
+              fast_block_cache_entry_stats,
+              num_immutable_mem_table,
+              num_immutable_mem_table_flushed,
+              mem_table_flush_pending,
+              compaction_pending,
+              background_errors,
+              cur_size_active_mem_table,
+              cur_size_all_mem_tables,
+              size_all_mem_tables,
+              num_entries_active_mem_table,
+              num_entries_imm_mem_tables,
+              num_deletes_active_mem_table,
+              num_deletes_imm_mem_tables,
+              estimate_num_keys,
+              estimate_table_readers_mem,
+              is_file_deletions_enabled,
+              num_snapshots,
+              oldest_snapshot_time,
+              oldest_snapshot_sequence,
+              num_live_versions,
+              current_version_number,
+              estimate_live_data_size,
+              min_log_number_to_keep_str,
+              min_obsolete_sst_number_to_keep_str,
+              base_level_str,
+              total_sst_files_size,
+              live_sst_files_size,
+              obsolete_sst_files_size,
+              live_sst_files_size_at_temperature,
+              estimate_pending_comp_bytes,
+              aggregated_table_properties,
+              num_running_compactions,
+              num_running_flushes,
+              actual_delayed_write_rate,
+              is_write_stopped,
+              estimate_oldest_key_time,
+              block_cache_capacity,
+              block_cache_usage,
+              block_cache_pinned_usage,
+              options_statistics,
+              num_blob_files,
+              blob_stats,
+              total_blob_file_size,
+              live_blob_file_size,
+              live_blob_file_garbage_size,
+              blob_cache_capacity,
+              blob_cache_usage,
+              blob_cache_pinned_usage)
+          .forEach(
+              prop -> {
+                try {
+                  final String value = rocksdb.getProperty(cfHandle, prefix + prop);
+                  if (!value.isBlank()) {
+                    out.println(prop + ": " + value);
+                  }
+                } catch (RocksDBException e) {
+                  LOG.debug("couldn't get property {}", prop);
+                }
+              });
+      out.println("=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=");
+    }
+  }
+
+  static ColumnFamilyUsage getAndPrintUsageForColumnFamily(
+      final RocksDB rocksdb, final ColumnFamilyHandle cfHandle, final PrintWriter out)
+      throws RocksDBException, NumberFormatException {
+    final String numberOfKeys = rocksdb.getProperty(cfHandle, "rocksdb.estimate-num-keys");
+    if (!numberOfKeys.isBlank()) {
       try {
-        final long sizeLong = Long.parseLong(size);
         final long numberOfKeysLong = Long.parseLong(numberOfKeys);
         final String totalSstFilesSize =
             rocksdb.getProperty(cfHandle, "rocksdb.total-sst-files-size");
         final long totalSstFilesSizeLong =
             !totalSstFilesSize.isBlank() ? Long.parseLong(totalSstFilesSize) : 0;
-        if (sizeLong == 0 && numberOfKeysLong == 0) {
-          emptyColumnFamily = true;
-        }
 
-        if (!emptyColumnFamily) {
+        final String totalBlobFilesSize =
+            rocksdb.getProperty(cfHandle, "rocksdb.total-blob-file-size");
+        final long totalBlobFilesSizeLong =
+            !totalBlobFilesSize.isBlank() ? Long.parseLong(totalBlobFilesSize) : 0;
+
+        final long totalFilesSize = totalSstFilesSizeLong + totalBlobFilesSizeLong;
+        if (isPopulatedColumnFamily(0, numberOfKeysLong)) {
           printLine(
               out,
               getNameById(cfHandle.getName()),
               rocksdb.getProperty(cfHandle, "rocksdb.estimate-num-keys"),
-              formatOutputSize(sizeLong),
-              formatOutputSize(totalSstFilesSizeLong));
+              formatOutputSize(totalFilesSize),
+              formatOutputSize(totalSstFilesSizeLong),
+              formatOutputSize(totalBlobFilesSizeLong));
         }
+        return new ColumnFamilyUsage(
+            getNameById(cfHandle.getName()),
+            numberOfKeysLong,
+            totalFilesSize,
+            totalSstFilesSizeLong,
+            totalBlobFilesSizeLong);
       } catch (NumberFormatException e) {
         LOG.error("Failed to parse string into long: " + e.getMessage());
       }
     }
+    // return empty usage on error
+    return new ColumnFamilyUsage(getNameById(cfHandle.getName()), 0, 0, 0, 0);
+  }
+
+  static void printTotals(final PrintWriter out, final List<ColumnFamilyUsage> columnFamilyUsages) {
+    final long totalKeys = columnFamilyUsages.stream().mapToLong(ColumnFamilyUsage::keys).sum();
+    final long totalSize =
+        columnFamilyUsages.stream().mapToLong(ColumnFamilyUsage::totalSize).sum();
+    final long totalSsts =
+        columnFamilyUsages.stream().mapToLong(ColumnFamilyUsage::sstFilesSize).sum();
+    final long totalBlobs =
+        columnFamilyUsages.stream().mapToLong(ColumnFamilyUsage::blobFilesSize).sum();
+    printSeparator(out);
+    printLine(
+        out,
+        "ESTIMATED TOTAL",
+        String.valueOf(totalKeys),
+        formatOutputSize(totalSize),
+        formatOutputSize(totalSsts),
+        formatOutputSize(totalBlobs));
+    printSeparator(out);
+  }
+
+  private static boolean isPopulatedColumnFamily(final long size, final long numberOfKeys) {
+    return size != 0 || numberOfKeys != 0;
   }
 
   static String formatOutputSize(final long size) {
@@ -123,19 +298,28 @@ public class RocksDbHelper {
   }
 
   static void printTableHeader(final PrintWriter out) {
+    printSeparator(out);
     out.format(
-        "| Column Family                  | Keys            | Column Size  | SST Files Size  |\n");
+        "| Column Family                  | Keys            | Total Size  | SST Files Size  | Blob Files Size  | \n");
+    printSeparator(out);
+  }
+
+  private static void printSeparator(final PrintWriter out) {
     out.format(
-        "|--------------------------------|-----------------|--------------|-----------------|\n");
+        "|--------------------------------|-----------------|-------------|-----------------|------------------|\n");
   }
 
   static void printLine(
       final PrintWriter out,
       final String cfName,
       final String keys,
-      final String columnSize,
-      final String sstFilesSize) {
-    final String format = "| %-30s | %-15s | %-12s | %-15s |\n";
-    out.format(format, cfName, keys, columnSize, sstFilesSize);
+      final String totalFilesSize,
+      final String sstFilesSize,
+      final String blobFilesSize) {
+    final String format = "| %-30s | %-15s | %-11s | %-15s | %-16s |\n";
+    out.format(format, cfName, keys, totalFilesSize, sstFilesSize, blobFilesSize);
   }
+
+  record ColumnFamilyUsage(
+      String name, long keys, long totalSize, long sstFilesSize, long blobFilesSize) {}
 }

besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/RocksDbSubCommand.java

@@ -19,6 +19,8 @@ import static org.hyperledger.besu.controller.BesuController.DATABASE_PATH;
 import org.hyperledger.besu.cli.util.VersionProvider;
 
 import java.io.PrintWriter;
+import java.util.ArrayList;
+import java.util.List;
 
 import org.rocksdb.RocksDBException;
 import picocli.CommandLine;
@@ -31,12 +33,12 @@ import picocli.CommandLine.ParentCommand;
     description = "Print RocksDB information",
     mixinStandardHelpOptions = true,
     versionProvider = VersionProvider.class,
-    subcommands = {RocksDbSubCommand.RocksDbUsage.class})
+    subcommands = {RocksDbSubCommand.RocksDbUsage.class, RocksDbSubCommand.RocksDbStats.class})
 public class RocksDbSubCommand implements Runnable {
 
   @SuppressWarnings("unused")
   @ParentCommand
-  private StorageSubCommand parentCommand;
+  private StorageSubCommand storageSubCommand;
 
   @SuppressWarnings("unused")
   @CommandLine.Spec
@@ -60,7 +62,7 @@ public class RocksDbSubCommand implements Runnable {
 
     @SuppressWarnings("unused")
     @ParentCommand
-    private RocksDbSubCommand parentCommand;
+    private RocksDbSubCommand rocksDbSubCommand;
 
     @Override
     public void run() {
@@ -68,9 +70,9 @@ public class RocksDbSubCommand implements Runnable {
       final PrintWriter out = spec.commandLine().getOut();
 
       final String dbPath =
-          parentCommand
-              .parentCommand
-              .parentCommand
+          rocksDbSubCommand
+              .storageSubCommand
+              .besuCommand
               .dataDir()
               .toString()
               .concat("/")
@@ -78,11 +80,56 @@ public class RocksDbSubCommand implements Runnable {
 
       RocksDbHelper.printTableHeader(out);
 
+      final List<RocksDbHelper.ColumnFamilyUsage> columnFamilyUsages = new ArrayList<>();
       RocksDbHelper.forEachColumnFamily(
           dbPath,
           (rocksdb, cfHandle) -> {
             try {
-              RocksDbHelper.printUsageForColumnFamily(rocksdb, cfHandle, out);
+              columnFamilyUsages.add(
+                  RocksDbHelper.getAndPrintUsageForColumnFamily(rocksdb, cfHandle, out));
+            } catch (RocksDBException e) {
+              throw new RuntimeException(e);
+            }
+          });
+      RocksDbHelper.printTotals(out, columnFamilyUsages);
+    }
+  }
+
+  @Command(
+      name = "x-stats",
+      description = "Print rocksdb stats",
+      mixinStandardHelpOptions = true,
+      versionProvider = VersionProvider.class)
+  static class RocksDbStats implements Runnable {
+
+    @SuppressWarnings("unused")
+    @CommandLine.Spec
+    private CommandLine.Model.CommandSpec spec;
+
+    @SuppressWarnings("unused")
+    @ParentCommand
+    private RocksDbSubCommand rocksDbSubCommand;
+
+    @Override
+    public void run() {
+
+      final PrintWriter out = spec.commandLine().getOut();
+
+      final String dbPath =
+          rocksDbSubCommand
+              .storageSubCommand
+              .besuCommand
+              .dataDir()
+              .toString()
+              .concat("/")
+              .concat(DATABASE_PATH);
+
+      out.println("Column Family Stats...");
+      RocksDbHelper.forEachColumnFamily(
+          dbPath,
+          (rocksdb, cfHandle) -> {
+            try {
+              RocksDbHelper.printStatsForColumnFamily(rocksdb, cfHandle, out);
             } catch (RocksDBException e) {
               throw new RuntimeException(e);
             }

besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/StorageSubCommand.java

@@ -57,7 +57,7 @@ public class StorageSubCommand implements Runnable {
 
   @SuppressWarnings("unused")
   @ParentCommand
-  BesuCommand parentCommand;
+  BesuCommand besuCommand;
 
   @SuppressWarnings("unused")
   @Spec
@@ -104,8 +104,8 @@ public class StorageSubCommand implements Runnable {
 
     private StorageProvider getStorageProvider() {
       // init collection of ignorable segments
-      parentCommand.parentCommand.setIgnorableStorageSegments();
-      return parentCommand.parentCommand.getStorageProvider();
+      parentCommand.besuCommand.setIgnorableStorageSegments();
+      return parentCommand.besuCommand.getStorageProvider();
     }
 
     private void revert(final StorageProvider storageProvider) {

besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogSubCommand.java

@@ -78,7 +78,7 @@ public class TrieLogSubCommand implements Runnable {
   }
 
   private static BesuController createBesuController() {
-    return parentCommand.parentCommand.buildController();
+    return parentCommand.besuCommand.buildController();
   }
 
   @Command(
@@ -132,7 +132,7 @@ public class TrieLogSubCommand implements Runnable {
       final TrieLogContext context = getTrieLogContext();
       final Path dataDirectoryPath =
           Paths.get(
-              TrieLogSubCommand.parentCommand.parentCommand.dataDir().toAbsolutePath().toString());
+              TrieLogSubCommand.parentCommand.besuCommand.dataDir().toAbsolutePath().toString());
 
       LOG.info("Estimating trie logs size before pruning...");
       long sizeBefore = estimatedSizeOfTrieLogs();
@@ -167,7 +167,7 @@ public class TrieLogSubCommand implements Runnable {
     private long estimatedSizeOfTrieLogs() {
       final String dbPath =
           TrieLogSubCommand.parentCommand
-              .parentCommand
+              .besuCommand
               .dataDir()
               .toString()
               .concat("/")
@@ -180,6 +180,7 @@ public class TrieLogSubCommand implements Runnable {
             (rocksdb, cfHandle) -> {
               try {
                 if (Arrays.equals(cfHandle.getName(), TRIE_LOG_STORAGE.getId())) {
+                  // TODO SLD use sst + blob?
                   estimatedSaving.set(
                       Long.parseLong(
                           rocksdb.getProperty(cfHandle, "rocksdb.estimate-live-data-size")));
@@ -233,7 +234,7 @@ public class TrieLogSubCommand implements Runnable {
         trieLogFilePath =
             Paths.get(
                 TrieLogSubCommand.parentCommand
-                    .parentCommand
+                    .besuCommand
                     .dataDir()
                     .resolve("trie-logs.bin")
                     .toAbsolutePath()
@@ -283,7 +284,7 @@ public class TrieLogSubCommand implements Runnable {
         trieLogFilePath =
             Paths.get(
                 TrieLogSubCommand.parentCommand
-                    .parentCommand
+                    .besuCommand
                     .dataDir()
                     .resolve("trie-logs.bin")
                     .toAbsolutePath()