|
|
@ -55,6 +55,7 @@ import org.rocksdb.DBOptions; |
|
|
|
import org.rocksdb.Env; |
|
|
|
import org.rocksdb.Env; |
|
|
|
import org.rocksdb.LRUCache; |
|
|
|
import org.rocksdb.LRUCache; |
|
|
|
import org.rocksdb.Options; |
|
|
|
import org.rocksdb.Options; |
|
|
|
|
|
|
|
import org.rocksdb.PlainTableConfig; |
|
|
|
import org.rocksdb.ReadOptions; |
|
|
|
import org.rocksdb.ReadOptions; |
|
|
|
import org.rocksdb.RocksDB; |
|
|
|
import org.rocksdb.RocksDB; |
|
|
|
import org.rocksdb.RocksDBException; |
|
|
|
import org.rocksdb.RocksDBException; |
|
|
@ -70,8 +71,6 @@ import org.slf4j.LoggerFactory; |
|
|
|
public abstract class RocksDBColumnarKeyValueStorage implements SegmentedKeyValueStorage { |
|
|
|
public abstract class RocksDBColumnarKeyValueStorage implements SegmentedKeyValueStorage { |
|
|
|
|
|
|
|
|
|
|
|
private static final Logger LOG = LoggerFactory.getLogger(RocksDBColumnarKeyValueStorage.class); |
|
|
|
private static final Logger LOG = LoggerFactory.getLogger(RocksDBColumnarKeyValueStorage.class); |
|
|
|
private static final int ROCKSDB_FORMAT_VERSION = 5; |
|
|
|
|
|
|
|
private static final long ROCKSDB_BLOCK_SIZE = 32768; |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
/** RocksDb blockcache size when using the high spec option */ |
|
|
|
/** RocksDb blockcache size when using the high spec option */ |
|
|
|
protected static final long ROCKSDB_BLOCKCACHE_SIZE_HIGH_SPEC = 1_073_741_824L; |
|
|
|
protected static final long ROCKSDB_BLOCKCACHE_SIZE_HIGH_SPEC = 1_073_741_824L; |
|
|
@ -164,7 +163,7 @@ public abstract class RocksDBColumnarKeyValueStorage implements SegmentedKeyValu |
|
|
|
.forEach(trimmedSegments::remove); |
|
|
|
.forEach(trimmedSegments::remove); |
|
|
|
columnDescriptors = |
|
|
|
columnDescriptors = |
|
|
|
trimmedSegments.stream() |
|
|
|
trimmedSegments.stream() |
|
|
|
.map(segment -> createColumnDescriptor(segment, configuration)) |
|
|
|
.map(segment -> createColumnDescriptor(segment)) |
|
|
|
.collect(Collectors.toList()); |
|
|
|
.collect(Collectors.toList()); |
|
|
|
|
|
|
|
|
|
|
|
setGlobalOptions(configuration, stats); |
|
|
|
setGlobalOptions(configuration, stats); |
|
|
@ -181,19 +180,18 @@ public abstract class RocksDBColumnarKeyValueStorage implements SegmentedKeyValu |
|
|
|
* options to apply to the corresponding Column Family |
|
|
|
* options to apply to the corresponding Column Family |
|
|
|
* |
|
|
|
* |
|
|
|
* @param segment the segment identifier |
|
|
|
* @param segment the segment identifier |
|
|
|
* @param configuration RocksDB configuration |
|
|
|
|
|
|
|
* @return a column family descriptor |
|
|
|
* @return a column family descriptor |
|
|
|
*/ |
|
|
|
*/ |
|
|
|
private ColumnFamilyDescriptor createColumnDescriptor( |
|
|
|
private ColumnFamilyDescriptor createColumnDescriptor( |
|
|
|
final SegmentIdentifier segment, final RocksDBConfiguration configuration) { |
|
|
|
final SegmentIdentifier segment) { |
|
|
|
|
|
|
|
|
|
|
|
BlockBasedTableConfig basedTableConfig = createBlockBasedTableConfig(segment, configuration); |
|
|
|
PlainTableConfig plainTableConfig = createPlainTableConfig(); |
|
|
|
|
|
|
|
|
|
|
|
final var options = |
|
|
|
final var options = |
|
|
|
new ColumnFamilyOptions() |
|
|
|
new ColumnFamilyOptions() |
|
|
|
.setTtl(0) |
|
|
|
.setTtl(0) |
|
|
|
.setCompressionType(CompressionType.LZ4_COMPRESSION) |
|
|
|
.setCompressionType(CompressionType.LZ4_COMPRESSION) |
|
|
|
.setTableFormatConfig(basedTableConfig); |
|
|
|
.setTableFormatConfig(plainTableConfig); |
|
|
|
|
|
|
|
|
|
|
|
if (segment.containsStaticData()) { |
|
|
|
if (segment.containsStaticData()) { |
|
|
|
options |
|
|
|
options |
|
|
@ -206,30 +204,16 @@ public abstract class RocksDBColumnarKeyValueStorage implements SegmentedKeyValu |
|
|
|
return new ColumnFamilyDescriptor(segment.getId(), options); |
|
|
|
return new ColumnFamilyDescriptor(segment.getId(), options); |
|
|
|
} |
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
/*** |
|
|
|
private PlainTableConfig createPlainTableConfig() { |
|
|
|
* Create a Block Base Table configuration for each segment, depending on the configuration in place |
|
|
|
return new PlainTableConfig() |
|
|
|
* and the segment itself |
|
|
|
.setBloomBitsPerKey(10) // Optional: can use bloom filters to speed up key lookups
|
|
|
|
* |
|
|
|
.setHashTableRatio(0.75) // Default hash table ratio
|
|
|
|
* @param segment The segment related to the column family |
|
|
|
.setIndexSparseness(16) // Adjust the density of the index
|
|
|
|
* @param config RocksDB configuration |
|
|
|
.setEncodingType(PlainTableConfig.DEFAULT_ENCODING_TYPE) // Use prefix encoding for efficient memory use
|
|
|
|
* @return Block Base Table configuration |
|
|
|
.setFullScanMode(false); // Optimized for in-memory full scan
|
|
|
|
*/ |
|
|
|
|
|
|
|
private BlockBasedTableConfig createBlockBasedTableConfig( |
|
|
|
|
|
|
|
final SegmentIdentifier segment, final RocksDBConfiguration config) { |
|
|
|
|
|
|
|
final LRUCache cache = |
|
|
|
|
|
|
|
new LRUCache( |
|
|
|
|
|
|
|
config.isHighSpec() && segment.isEligibleToHighSpecFlag() |
|
|
|
|
|
|
|
? ROCKSDB_BLOCKCACHE_SIZE_HIGH_SPEC |
|
|
|
|
|
|
|
: config.getCacheCapacity()); |
|
|
|
|
|
|
|
return new BlockBasedTableConfig() |
|
|
|
|
|
|
|
.setFormatVersion(ROCKSDB_FORMAT_VERSION) |
|
|
|
|
|
|
|
.setBlockCache(cache) |
|
|
|
|
|
|
|
.setFilterPolicy(new BloomFilter(10, false)) |
|
|
|
|
|
|
|
.setPartitionFilters(true) |
|
|
|
|
|
|
|
.setCacheIndexAndFilterBlocks(false) |
|
|
|
|
|
|
|
.setBlockSize(ROCKSDB_BLOCK_SIZE); |
|
|
|
|
|
|
|
} |
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
/*** |
|
|
|
/*** |
|
|
|
* Set Global options (DBOptions) |
|
|
|
* Set Global options (DBOptions) |
|
|
|
* |
|
|
|
* |
|
|
|