From fe673431fab8c6439e6eb0e83f6545a1583afaeb Mon Sep 17 00:00:00 2001 From: Pedro Novais Date: Fri, 8 Apr 2022 02:13:20 +0100 Subject: [PATCH] update rocksdbjni to v6.29.4 for compatibility with arm64 CPU architecture (#3634) * update rocksdbjni to v6.29.5 for compatibility with arm64 CPU Signed-off-by: Pedro Novais Co-authored-by: garyschulte --- gradle/versions.gradle | 2 +- .../RocksDBColumnarKeyValueStorage.java | 39 ++++++++++++------- .../RocksDBColumnarKeyValueStorageTest.java | 34 ++++++++++++++++ .../kvstore/SegmentedKeyValueStorage.java | 2 +- .../SegmentedKeyValueStorageAdapter.java | 4 +- 5 files changed, 62 insertions(+), 19 deletions(-) diff --git a/gradle/versions.gradle b/gradle/versions.gradle index 389192d044..ca75305738 100644 --- a/gradle/versions.gradle +++ b/gradle/versions.gradle @@ -162,7 +162,7 @@ dependencyManagement { dependency 'org.openjdk.jmh:jmh-core:1.34' dependency 'org.openjdk.jmh:jmh-generator-annprocess:1.34' - dependency 'org.rocksdb:rocksdbjni:6.15.2' + dependency 'org.rocksdb:rocksdbjni:6.29.5' dependency 'org.slf4j:slf4j-api:1.7.25' diff --git a/plugins/rocksdb/src/main/java/org/hyperledger/besu/plugin/services/storage/rocksdb/segmented/RocksDBColumnarKeyValueStorage.java b/plugins/rocksdb/src/main/java/org/hyperledger/besu/plugin/services/storage/rocksdb/segmented/RocksDBColumnarKeyValueStorage.java index 68d4eb3141..63148e17a7 100644 --- a/plugins/rocksdb/src/main/java/org/hyperledger/besu/plugin/services/storage/rocksdb/segmented/RocksDBColumnarKeyValueStorage.java +++ b/plugins/rocksdb/src/main/java/org/hyperledger/besu/plugin/services/storage/rocksdb/segmented/RocksDBColumnarKeyValueStorage.java @@ -31,6 +31,7 @@ import org.hyperledger.besu.services.kvstore.SegmentedKeyValueStorageTransaction import java.nio.charset.StandardCharsets; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Optional; @@ -40,7 +41,6 @@ import java.util.function.Predicate; import java.util.stream.Collectors; import java.util.stream.Stream; -import com.google.common.collect.ImmutableMap; import org.apache.tuweni.bytes.Bytes; import org.rocksdb.BlockBasedTableConfig; import org.rocksdb.ColumnFamilyDescriptor; @@ -123,15 +123,14 @@ public class RocksDBColumnarKeyValueStorage Collectors.toMap( segment -> Bytes.wrap(segment.getId()), SegmentIdentifier::getName)); - final ImmutableMap.Builder builder = ImmutableMap.builder(); + columnHandlesByName = new HashMap<>(); for (ColumnFamilyHandle columnHandle : columnHandles) { final String segmentName = requireNonNullElse( segmentsById.get(Bytes.wrap(columnHandle.getName())), DEFAULT_COLUMN); - builder.put(segmentName, columnHandle); + columnHandlesByName.put(segmentName, columnHandle); } - columnHandlesByName = builder.build(); } catch (final RocksDBException e) { throw new StorageException(e); } @@ -195,18 +194,28 @@ public class RocksDBColumnarKeyValueStorage } @Override - public void clear(final ColumnFamilyHandle segmentHandle) { - try (final RocksIterator rocksIterator = db.newIterator(segmentHandle)) { - rocksIterator.seekToFirst(); - if (rocksIterator.isValid()) { - final byte[] firstKey = rocksIterator.key(); - rocksIterator.seekToLast(); - if (rocksIterator.isValid()) { - final byte[] lastKey = rocksIterator.key(); - db.deleteRange(segmentHandle, firstKey, lastKey); - db.delete(segmentHandle, lastKey); - } + public ColumnFamilyHandle clear(final ColumnFamilyHandle segmentHandle) { + try { + + var entry = + columnHandlesByName.entrySet().stream() + .filter(e -> e.getValue().equals(segmentHandle)) + .findAny(); + + if (entry.isPresent()) { + String segmentName = entry.get().getKey(); + ColumnFamilyDescriptor descriptor = + new ColumnFamilyDescriptor( + segmentHandle.getName(), segmentHandle.getDescriptor().getOptions()); + db.dropColumnFamily(segmentHandle); + segmentHandle.close(); + ColumnFamilyHandle newHandle = db.createColumnFamily(descriptor); + columnHandlesByName.put(segmentName, newHandle); + return newHandle; } + + return segmentHandle; + } catch (final RocksDBException e) { throw new StorageException(e); } diff --git a/plugins/rocksdb/src/test/java/org/hyperledger/besu/plugin/services/storage/rocksdb/unsegmented/RocksDBColumnarKeyValueStorageTest.java b/plugins/rocksdb/src/test/java/org/hyperledger/besu/plugin/services/storage/rocksdb/unsegmented/RocksDBColumnarKeyValueStorageTest.java index 662f150daf..6e27cb8f4d 100644 --- a/plugins/rocksdb/src/test/java/org/hyperledger/besu/plugin/services/storage/rocksdb/unsegmented/RocksDBColumnarKeyValueStorageTest.java +++ b/plugins/rocksdb/src/test/java/org/hyperledger/besu/plugin/services/storage/rocksdb/unsegmented/RocksDBColumnarKeyValueStorageTest.java @@ -31,6 +31,8 @@ import java.nio.charset.StandardCharsets; import java.util.Arrays; import java.util.Optional; import java.util.Set; +import java.util.function.Consumer; +import java.util.function.Supplier; import org.junit.Rule; import org.junit.Test; @@ -41,6 +43,38 @@ public class RocksDBColumnarKeyValueStorageTest extends AbstractKeyValueStorageT @Rule public final TemporaryFolder folder = new TemporaryFolder(); + @Test + public void assertClear() throws Exception { + final byte[] key = bytesFromHexString("0001"); + final byte[] val1 = bytesFromHexString("0FFF"); + final byte[] val2 = bytesFromHexString("1337"); + final SegmentedKeyValueStorage store = createSegmentedStore(); + Supplier segment = () -> store.getSegmentIdentifierByName(TestSegment.FOO); + final Consumer insert = + value -> { + final Transaction tx = store.startTransaction(); + tx.put(segment.get(), key, value); + tx.commit(); + }; + + // insert val: + insert.accept(val1); + final Optional result = store.get(segment.get(), key); + assertThat(result.orElse(null)).isEqualTo(val1); + + // clear and assert empty: + store.clear(segment.get()); + final Optional truncResult = store.get(segment.get(), key); + assertThat(truncResult).isEmpty(); + + // insert into empty: + insert.accept(val2); + final Optional nextResult = store.get(segment.get(), key); + assertThat(nextResult.orElse(null)).isEqualTo(val2); + + store.close(); + } + @Test public void twoSegmentsAreIndependent() throws Exception { final SegmentedKeyValueStorage store = createSegmentedStore(); diff --git a/services/kvstore/src/main/java/org/hyperledger/besu/services/kvstore/SegmentedKeyValueStorage.java b/services/kvstore/src/main/java/org/hyperledger/besu/services/kvstore/SegmentedKeyValueStorage.java index 7c75f9e9c7..aa7f022474 100644 --- a/services/kvstore/src/main/java/org/hyperledger/besu/services/kvstore/SegmentedKeyValueStorage.java +++ b/services/kvstore/src/main/java/org/hyperledger/besu/services/kvstore/SegmentedKeyValueStorage.java @@ -74,7 +74,7 @@ public interface SegmentedKeyValueStorage extends Closeable { Set getAllKeysThat(S segmentHandle, Predicate returnCondition); - void clear(S segmentHandle); + S clear(S segmentHandle); /** * Represents a set of changes to be committed atomically. A single transaction is not diff --git a/services/kvstore/src/main/java/org/hyperledger/besu/services/kvstore/SegmentedKeyValueStorageAdapter.java b/services/kvstore/src/main/java/org/hyperledger/besu/services/kvstore/SegmentedKeyValueStorageAdapter.java index e0b71a513c..f2d7c2d072 100644 --- a/services/kvstore/src/main/java/org/hyperledger/besu/services/kvstore/SegmentedKeyValueStorageAdapter.java +++ b/services/kvstore/src/main/java/org/hyperledger/besu/services/kvstore/SegmentedKeyValueStorageAdapter.java @@ -26,7 +26,7 @@ import java.util.function.Predicate; import java.util.stream.Stream; public class SegmentedKeyValueStorageAdapter implements KeyValueStorage { - private final S segmentHandle; + private S segmentHandle; private final SegmentedKeyValueStorage storage; public SegmentedKeyValueStorageAdapter( @@ -37,7 +37,7 @@ public class SegmentedKeyValueStorageAdapter implements KeyValueStorage { @Override public void clear() { - storage.clear(segmentHandle); + segmentHandle = storage.clear(segmentHandle); } @Override