refactor-privacy-storage (#7)

* refactor-privacy-storage

Signed-off-by: Ivaylo Kirilov <iikirilov@gmail.com>
pull/62/head
Ivaylo Kirilov 5 years ago committed by MadelineMurray
parent f84bd55f2f
commit c09145e490
  1. 26
      ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/privacy/methods/priv/PrivGetTransactionReceipt.java
  2. 11
      ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/privacy/methods/priv/PrivGetTransactionReceiptTest.java
  3. 49
      ethereum/core/src/integration-test/java/org/hyperledger/besu/ethereum/mainnet/precompiles/privacy/PrivacyPrecompiledContractIntegrationTest.java
  4. 19
      ethereum/core/src/main/java/org/hyperledger/besu/ethereum/core/PrivacyParameters.java
  5. 21
      ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/precompiles/privacy/PrivacyPrecompiledContract.java
  6. 74
      ethereum/core/src/main/java/org/hyperledger/besu/ethereum/privacy/PrivateStateKeyValueStorage.java
  7. 37
      ethereum/core/src/main/java/org/hyperledger/besu/ethereum/privacy/PrivateStateStorage.java
  8. 3
      ethereum/core/src/main/java/org/hyperledger/besu/ethereum/privacy/PrivateTransactionHandler.java
  9. 110
      ethereum/core/src/main/java/org/hyperledger/besu/ethereum/privacy/PrivateTransactionKeyValueStorage.java
  10. 160
      ethereum/core/src/main/java/org/hyperledger/besu/ethereum/privacy/storage/PrivateStateKeyValueStorage.java
  11. 23
      ethereum/core/src/main/java/org/hyperledger/besu/ethereum/privacy/storage/PrivateStateStorage.java
  12. 50
      ethereum/core/src/main/java/org/hyperledger/besu/ethereum/privacy/storage/PrivateTransactionMetadata.java
  13. 5
      ethereum/core/src/main/java/org/hyperledger/besu/ethereum/storage/StorageProvider.java
  14. 11
      ethereum/core/src/main/java/org/hyperledger/besu/ethereum/storage/keyvalue/KeyValueStorageProvider.java
  15. 11
      ethereum/core/src/test-support/java/org/hyperledger/besu/ethereum/core/InMemoryStorageProvider.java
  16. 38
      ethereum/core/src/test/java/org/hyperledger/besu/ethereum/mainnet/precompiles/privacy/PrivacyPrecompiledContractTest.java
  17. 3
      ethereum/core/src/test/java/org/hyperledger/besu/ethereum/privacy/PrivateTransactionHandlerTest.java
  18. 2
      plugin-api/build.gradle
  19. 10
      plugin-api/src/main/java/org/hyperledger/besu/plugin/services/storage/KeyValueStorage.java
  20. 21
      plugins/rocksdb/src/main/java/org/hyperledger/besu/plugin/services/storage/rocksdb/segmented/RocksDBColumnarKeyValueStorage.java
  21. 18
      plugins/rocksdb/src/main/java/org/hyperledger/besu/plugin/services/storage/rocksdb/unsegmented/RocksDBKeyValueStorage.java
  22. 42
      plugins/rocksdb/src/test/java/org/hyperledger/besu/plugin/services/storage/rocksdb/unsegmented/RocksDBColumnarKeyValueStorageTest.java
  23. 9
      services/kvstore/src/main/java/org/hyperledger/besu/services/kvstore/InMemoryKeyValueStorage.java
  24. 9
      services/kvstore/src/main/java/org/hyperledger/besu/services/kvstore/LimitedInMemoryKeyValueStorage.java
  25. 5
      services/kvstore/src/main/java/org/hyperledger/besu/services/kvstore/SegmentedKeyValueStorage.java
  26. 8
      services/kvstore/src/main/java/org/hyperledger/besu/services/kvstore/SegmentedKeyValueStorageAdapter.java
  27. 17
      testutil/src/main/java/org/hyperledger/besu/kvstore/AbstractKeyValueStorageTest.java

@ -93,8 +93,8 @@ public class PrivGetTransactionReceipt implements JsonRpcMethod {
final long blockNumber = blockchain.getBlockchain().getBlockHeader(blockhash).get().getNumber();
final String publicKey = privacyParameters.getEnclavePublicKey();
PrivateTransaction privateTransaction;
String privacyGroupId;
final PrivateTransaction privateTransaction;
final String privacyGroupId;
try {
final ReceiveResponse receiveResponse = getReceiveResponseFromEnclave(transaction, publicKey);
LOG.trace("Received transaction information from Enclave");
@ -123,34 +123,34 @@ public class PrivGetTransactionReceipt implements JsonRpcMethod {
LOG.trace("Calculated contractAddress: {}", contractAddress);
BytesValue rlpEncoded = RLP.encode(privateTransaction::writeTo);
Bytes32 txHash = org.hyperledger.besu.crypto.Hash.keccak256(rlpEncoded);
final BytesValue rlpEncoded = RLP.encode(privateTransaction::writeTo);
final Bytes32 txHash = org.hyperledger.besu.crypto.Hash.keccak256(rlpEncoded);
LOG.trace("Calculated private transaction hash: {}", txHash);
List<Log> events =
final List<Log> transactionLogs =
privacyParameters
.getPrivateTransactionStorage()
.getEvents(txHash)
.getPrivateStateStorage()
.getTransactionLogs(txHash)
.orElse(Collections.emptyList());
LOG.trace("Processed private transaction events");
BytesValue output =
final BytesValue transactionOutput =
privacyParameters
.getPrivateTransactionStorage()
.getOutput(txHash)
.getPrivateStateStorage()
.getTransactionOutput(txHash)
.orElse(BytesValue.wrap(new byte[0]));
LOG.trace("Processed private transaction output");
PrivateTransactionReceiptResult result =
final PrivateTransactionReceiptResult result =
new PrivateTransactionReceiptResult(
contractAddress,
privateTransaction.getSender().toString(),
privateTransaction.getTo().map(Address::toString).orElse(null),
events,
output,
transactionLogs,
transactionOutput,
blockhash,
transactionHash,
blockNumber,

@ -42,8 +42,8 @@ import org.hyperledger.besu.ethereum.core.PrivacyParameters;
import org.hyperledger.besu.ethereum.core.Transaction;
import org.hyperledger.besu.ethereum.core.Wei;
import org.hyperledger.besu.ethereum.privacy.PrivateTransaction;
import org.hyperledger.besu.ethereum.privacy.PrivateTransactionStorage;
import org.hyperledger.besu.ethereum.privacy.Restriction;
import org.hyperledger.besu.ethereum.privacy.storage.PrivateStateStorage;
import org.hyperledger.besu.ethereum.rlp.RLP;
import org.hyperledger.besu.util.bytes.Bytes32;
import org.hyperledger.besu.util.bytes.BytesValue;
@ -154,11 +154,10 @@ public class PrivGetTransactionReceiptTest {
final BlockHeader mockBlockHeader = mock(BlockHeader.class);
when(blockchain.getBlockHeader(any(Hash.class))).thenReturn(Optional.of(mockBlockHeader));
when(mockBlockHeader.getNumber()).thenReturn(0L);
final PrivateTransactionStorage privateTransactionStorage =
mock(PrivateTransactionStorage.class);
when(privacyParameters.getPrivateTransactionStorage()).thenReturn(privateTransactionStorage);
when(privateTransactionStorage.getEvents(any(Bytes32.class))).thenReturn(Optional.empty());
when(privateTransactionStorage.getOutput(any(Bytes32.class))).thenReturn(Optional.empty());
final PrivateStateStorage privateStateStorage = mock(PrivateStateStorage.class);
when(privacyParameters.getPrivateStateStorage()).thenReturn(privateStateStorage);
when(privateStateStorage.getTransactionLogs(any(Bytes32.class))).thenReturn(Optional.empty());
when(privateStateStorage.getTransactionOutput(any(Bytes32.class))).thenReturn(Optional.empty());
}
@Test

@ -32,10 +32,9 @@ import org.hyperledger.besu.ethereum.core.MutableWorldState;
import org.hyperledger.besu.ethereum.core.ProcessableBlockHeader;
import org.hyperledger.besu.ethereum.core.WorldUpdater;
import org.hyperledger.besu.ethereum.mainnet.SpuriousDragonGasCalculator;
import org.hyperledger.besu.ethereum.privacy.PrivateStateStorage;
import org.hyperledger.besu.ethereum.privacy.PrivateTransaction;
import org.hyperledger.besu.ethereum.privacy.PrivateTransactionProcessor;
import org.hyperledger.besu.ethereum.privacy.PrivateTransactionStorage;
import org.hyperledger.besu.ethereum.privacy.storage.PrivateStateStorage;
import org.hyperledger.besu.ethereum.vm.BlockHashLookup;
import org.hyperledger.besu.ethereum.vm.MessageFrame;
import org.hyperledger.besu.ethereum.vm.OperationTracer;
@ -85,15 +84,13 @@ public class PrivacyPrecompiledContractIntegrationTest {
private static OrionTestHarness testHarness;
private static WorldStateArchive worldStateArchive;
private static PrivateTransactionStorage privateTransactionStorage;
private static PrivateTransactionStorage.Updater updater;
private static PrivateStateStorage privateStateStorage;
private static PrivateStateStorage.Updater storageUpdater;
private PrivateTransactionProcessor mockPrivateTxProcessor() {
PrivateTransactionProcessor mockPrivateTransactionProcessor =
final PrivateTransactionProcessor mockPrivateTransactionProcessor =
mock(PrivateTransactionProcessor.class);
PrivateTransactionProcessor.Result result =
final PrivateTransactionProcessor.Result result =
PrivateTransactionProcessor.Result.successful(
null, 0, BytesValue.fromHexString(DEFAULT_OUTPUT), null);
when(mockPrivateTransactionProcessor.processTransaction(
@ -126,19 +123,18 @@ public class PrivacyPrecompiledContractIntegrationTest {
messageFrame = mock(MessageFrame.class);
worldStateArchive = mock(WorldStateArchive.class);
MutableWorldState mutableWorldState = mock(MutableWorldState.class);
final MutableWorldState mutableWorldState = mock(MutableWorldState.class);
when(mutableWorldState.updater()).thenReturn(mock(WorldUpdater.class));
when(worldStateArchive.getMutable()).thenReturn(mutableWorldState);
when(worldStateArchive.getMutable(any())).thenReturn(Optional.of(mutableWorldState));
privateTransactionStorage = mock(PrivateTransactionStorage.class);
updater = mock(PrivateTransactionStorage.Updater.class);
when(updater.putTransactionLogs(nullable(Bytes32.class), any())).thenReturn(updater);
when(updater.putTransactionResult(nullable(Bytes32.class), any())).thenReturn(updater);
when(privateTransactionStorage.updater()).thenReturn(updater);
privateStateStorage = mock(PrivateStateStorage.class);
storageUpdater = mock(PrivateStateStorage.Updater.class);
when(storageUpdater.putPrivateAccountState(nullable(Bytes32.class), any()))
when(storageUpdater.putLatestStateRoot(nullable(Bytes32.class), any()))
.thenReturn(storageUpdater);
when(storageUpdater.putTransactionLogs(nullable(Bytes32.class), any()))
.thenReturn(storageUpdater);
when(storageUpdater.putTransactionResult(nullable(Bytes32.class), any()))
.thenReturn(storageUpdater);
when(privateStateStorage.updater()).thenReturn(storageUpdater);
}
@ -154,26 +150,25 @@ public class PrivacyPrecompiledContractIntegrationTest {
}
@Test
public void testSendAndReceive() throws Exception {
List<String> publicKeys = testHarness.getPublicKeys();
public void testSendAndReceive() {
final List<String> publicKeys = testHarness.getPublicKeys();
String s = new String(VALID_PRIVATE_TRANSACTION_RLP_BASE64, UTF_8);
SendRequest sc =
final String s = new String(VALID_PRIVATE_TRANSACTION_RLP_BASE64, UTF_8);
final SendRequest sc =
new SendRequestLegacy(s, publicKeys.get(0), Lists.newArrayList(publicKeys.get(0)));
SendResponse sr = enclave.send(sc);
final SendResponse sr = enclave.send(sc);
PrivacyPrecompiledContract privacyPrecompiledContract =
final PrivacyPrecompiledContract privacyPrecompiledContract =
new PrivacyPrecompiledContract(
new SpuriousDragonGasCalculator(),
publicKeys.get(0),
enclave,
worldStateArchive,
privateTransactionStorage,
privateStateStorage);
privacyPrecompiledContract.setPrivateTransactionProcessor(mockPrivateTxProcessor());
BytesValue actual =
final BytesValue actual =
privacyPrecompiledContract.compute(BytesValues.fromBase64(sr.getKey()), messageFrame);
assertThat(actual).isEqualTo(BytesValue.fromHexString(DEFAULT_OUTPUT));
@ -181,11 +176,11 @@ public class PrivacyPrecompiledContractIntegrationTest {
@Test
public void testNoPrivateKeyError() throws RuntimeException {
List<String> publicKeys = testHarness.getPublicKeys();
final List<String> publicKeys = testHarness.getPublicKeys();
publicKeys.add("noPrivateKey");
String s = new String(VALID_PRIVATE_TRANSACTION_RLP_BASE64, UTF_8);
SendRequest sc = new SendRequestLegacy(s, publicKeys.get(0), publicKeys);
final String s = new String(VALID_PRIVATE_TRANSACTION_RLP_BASE64, UTF_8);
final SendRequest sc = new SendRequestLegacy(s, publicKeys.get(0), publicKeys);
final Throwable thrown = catchThrowable(() -> enclave.send(sc));
@ -194,11 +189,11 @@ public class PrivacyPrecompiledContractIntegrationTest {
@Test
public void testWrongPrivateKeyError() throws RuntimeException {
List<String> publicKeys = testHarness.getPublicKeys();
final List<String> publicKeys = testHarness.getPublicKeys();
publicKeys.add("noPrivateKenoPrivateKenoPrivateKenoPrivateK");
String s = new String(VALID_PRIVATE_TRANSACTION_RLP_BASE64, UTF_8);
SendRequest sc = new SendRequestLegacy(s, publicKeys.get(0), publicKeys);
final String s = new String(VALID_PRIVATE_TRANSACTION_RLP_BASE64, UTF_8);
final SendRequest sc = new SendRequestLegacy(s, publicKeys.get(0), publicKeys);
final Throwable thrown = catchThrowable(() -> enclave.send(sc));

@ -18,8 +18,7 @@ import static java.nio.charset.StandardCharsets.UTF_8;
import org.hyperledger.besu.crypto.SECP256K1;
import org.hyperledger.besu.crypto.SECP256K1.KeyPair;
import org.hyperledger.besu.ethereum.privacy.PrivateStateStorage;
import org.hyperledger.besu.ethereum.privacy.PrivateTransactionStorage;
import org.hyperledger.besu.ethereum.privacy.storage.PrivateStateStorage;
import org.hyperledger.besu.ethereum.storage.StorageProvider;
import org.hyperledger.besu.ethereum.worldstate.WorldStateArchive;
import org.hyperledger.besu.ethereum.worldstate.WorldStatePreimageStorage;
@ -44,10 +43,9 @@ public class PrivacyParameters {
private String enclavePublicKey;
private File enclavePublicKeyFile;
private Optional<SECP256K1.KeyPair> signingKeyPair = Optional.empty();
private WorldStateArchive privateWorldStateArchive;
private StorageProvider privateStorageProvider;
private PrivateTransactionStorage privateTransactionStorage;
private PrivateStateStorage privateStateStorage;
public Integer getPrivacyAddress() {
@ -114,15 +112,6 @@ public class PrivacyParameters {
this.privateStorageProvider = privateStorageProvider;
}
public PrivateTransactionStorage getPrivateTransactionStorage() {
return privateTransactionStorage;
}
public void setPrivateTransactionStorage(
final PrivateTransactionStorage privateTransactionStorage) {
this.privateTransactionStorage = privateTransactionStorage;
}
public PrivateStateStorage getPrivateStateStorage() {
return privateStateStorage;
}
@ -181,16 +170,14 @@ public class PrivacyParameters {
final WorldStateArchive privateWorldStateArchive =
new WorldStateArchive(privateWorldStateStorage, privatePreimageStorage);
final PrivateTransactionStorage privateTransactionStorage =
storageProvider.createPrivateTransactionStorage();
final PrivateStateStorage privateStateStorage = storageProvider.createPrivateStateStorage();
config.setPrivateWorldStateArchive(privateWorldStateArchive);
config.setEnclavePublicKey(enclavePublicKey);
config.setEnclavePublicKeyFile(enclavePublicKeyFile);
config.setPrivateStorageProvider(storageProvider);
config.setPrivateTransactionStorage(privateTransactionStorage);
config.setPrivateStateStorage(privateStateStorage);
if (privateKeyPath != null) {
config.setSigningKeyPair(KeyPair.load(privateKeyPath.toFile()));
}

@ -27,10 +27,9 @@ import org.hyperledger.besu.ethereum.core.PrivacyParameters;
import org.hyperledger.besu.ethereum.core.WorldUpdater;
import org.hyperledger.besu.ethereum.debug.TraceOptions;
import org.hyperledger.besu.ethereum.mainnet.AbstractPrecompiledContract;
import org.hyperledger.besu.ethereum.privacy.PrivateStateStorage;
import org.hyperledger.besu.ethereum.privacy.PrivateTransaction;
import org.hyperledger.besu.ethereum.privacy.PrivateTransactionProcessor;
import org.hyperledger.besu.ethereum.privacy.PrivateTransactionStorage;
import org.hyperledger.besu.ethereum.privacy.storage.PrivateStateStorage;
import org.hyperledger.besu.ethereum.rlp.BytesValueRLPInput;
import org.hyperledger.besu.ethereum.rlp.RLP;
import org.hyperledger.besu.ethereum.trie.MerklePatriciaTrie;
@ -49,7 +48,6 @@ public class PrivacyPrecompiledContract extends AbstractPrecompiledContract {
private final Enclave enclave;
private final String enclavePublicKey;
private final WorldStateArchive privateWorldStateArchive;
private final PrivateTransactionStorage privateTransactionStorage;
private final PrivateStateStorage privateStateStorage;
private PrivateTransactionProcessor privateTransactionProcessor;
private static final Hash EMPTY_ROOT_HASH = Hash.wrap(MerklePatriciaTrie.EMPTY_TRIE_NODE_HASH);
@ -63,7 +61,6 @@ public class PrivacyPrecompiledContract extends AbstractPrecompiledContract {
privacyParameters.getEnclavePublicKey(),
new Enclave(privacyParameters.getEnclaveUri()),
privacyParameters.getPrivateWorldStateArchive(),
privacyParameters.getPrivateTransactionStorage(),
privacyParameters.getPrivateStateStorage());
}
@ -72,13 +69,11 @@ public class PrivacyPrecompiledContract extends AbstractPrecompiledContract {
final String publicKey,
final Enclave enclave,
final WorldStateArchive worldStateArchive,
final PrivateTransactionStorage privateTransactionStorage,
final PrivateStateStorage privateStateStorage) {
super("Privacy", gasCalculator);
this.enclave = enclave;
this.enclavePublicKey = publicKey;
this.privateWorldStateArchive = worldStateArchive;
this.privateTransactionStorage = privateTransactionStorage;
this.privateStateStorage = privateStateStorage;
}
@ -97,7 +92,7 @@ public class PrivacyPrecompiledContract extends AbstractPrecompiledContract {
final String key = BytesValues.asBase64String(input);
final ReceiveRequest receiveRequest = new ReceiveRequest(key, enclavePublicKey);
ReceiveResponse receiveResponse;
final ReceiveResponse receiveResponse;
try {
receiveResponse = enclave.receive(receiveRequest);
} catch (Exception e) {
@ -116,7 +111,7 @@ public class PrivacyPrecompiledContract extends AbstractPrecompiledContract {
// get the last world state root hash - or create a new one
final Hash lastRootHash =
privateStateStorage.getPrivateAccountState(privacyGroupId).orElse(EMPTY_ROOT_HASH);
privateStateStorage.getLatestStateRoot(privacyGroupId).orElse(EMPTY_ROOT_HASH);
final MutableWorldState disposablePrivateState =
privateWorldStateArchive.getMutable(lastRootHash).get();
@ -150,17 +145,15 @@ public class PrivacyPrecompiledContract extends AbstractPrecompiledContract {
disposablePrivateState.persist();
final PrivateStateStorage.Updater privateStateUpdater = privateStateStorage.updater();
privateStateUpdater.putPrivateAccountState(privacyGroupId, disposablePrivateState.rootHash());
privateStateUpdater.commit();
privateStateUpdater.putLatestStateRoot(privacyGroupId, disposablePrivateState.rootHash());
final Bytes32 txHash = keccak256(RLP.encode(privateTransaction::writeTo));
final PrivateTransactionStorage.Updater privateUpdater = privateTransactionStorage.updater();
final LogSeries logs = result.getLogs();
if (!logs.isEmpty()) {
privateUpdater.putTransactionLogs(txHash, result.getLogs());
privateStateUpdater.putTransactionLogs(txHash, result.getLogs());
}
privateUpdater.putTransactionResult(txHash, result.getOutput());
privateUpdater.commit();
privateStateUpdater.putTransactionResult(txHash, result.getOutput());
privateStateUpdater.commit();
}
return result.getOutput();

@ -1,74 +0,0 @@
/*
* Copyright ConsenSys AG.
*
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
* an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
* specific language governing permissions and limitations under the License.
*
* SPDX-License-Identifier: Apache-2.0
*/
package org.hyperledger.besu.ethereum.privacy;
import org.hyperledger.besu.ethereum.core.Hash;
import org.hyperledger.besu.plugin.services.storage.KeyValueStorage;
import org.hyperledger.besu.plugin.services.storage.KeyValueStorageTransaction;
import org.hyperledger.besu.util.bytes.Bytes32;
import org.hyperledger.besu.util.bytes.BytesValue;
import java.util.Optional;
public class PrivateStateKeyValueStorage implements PrivateStateStorage {
private final KeyValueStorage keyValueStorage;
public PrivateStateKeyValueStorage(final KeyValueStorage keyValueStorage) {
this.keyValueStorage = keyValueStorage;
}
@Override
public Optional<Hash> getPrivateAccountState(final BytesValue privacyId) {
final byte[] id = privacyId.getArrayUnsafe();
if (keyValueStorage.get(id).isPresent()) {
return Optional.of(Hash.wrap(Bytes32.wrap(keyValueStorage.get(id).get())));
} else {
return Optional.empty();
}
}
@Override
public boolean isWorldStateAvailable(final Bytes32 rootHash) {
return false;
}
@Override
public PrivateStateStorage.Updater updater() {
return new PrivateStateKeyValueStorage.Updater(keyValueStorage.startTransaction());
}
public static class Updater implements PrivateStateStorage.Updater {
private final KeyValueStorageTransaction transaction;
private Updater(final KeyValueStorageTransaction transaction) {
this.transaction = transaction;
}
@Override
public PrivateStateStorage.Updater putPrivateAccountState(
final BytesValue privacyId, final Hash privateStateHash) {
transaction.put(privacyId.getArrayUnsafe(), privateStateHash.extractArray());
return this;
}
@Override
public void commit() {
transaction.commit();
}
}
}

@ -1,37 +0,0 @@
/*
* Copyright ConsenSys AG.
*
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
* an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
* specific language governing permissions and limitations under the License.
*
* SPDX-License-Identifier: Apache-2.0
*/
package org.hyperledger.besu.ethereum.privacy;
import org.hyperledger.besu.ethereum.core.Hash;
import org.hyperledger.besu.util.bytes.Bytes32;
import org.hyperledger.besu.util.bytes.BytesValue;
import java.util.Optional;
public interface PrivateStateStorage {
Optional<Hash> getPrivateAccountState(BytesValue privacyId);
boolean isWorldStateAvailable(Bytes32 rootHash);
PrivateStateStorage.Updater updater();
interface Updater {
PrivateStateStorage.Updater putPrivateAccountState(BytesValue privacyId, Hash privateStateHash);
void commit();
}
}

@ -28,6 +28,7 @@ import org.hyperledger.besu.ethereum.core.Transaction;
import org.hyperledger.besu.ethereum.mainnet.TransactionValidator;
import org.hyperledger.besu.ethereum.mainnet.ValidationResult;
import org.hyperledger.besu.ethereum.privacy.markertransaction.PrivateMarkerTransactionFactory;
import org.hyperledger.besu.ethereum.privacy.storage.PrivateStateStorage;
import org.hyperledger.besu.ethereum.rlp.BytesValueRLPOutput;
import org.hyperledger.besu.ethereum.worldstate.WorldStateArchive;
import org.hyperledger.besu.util.bytes.BytesValues;
@ -152,7 +153,7 @@ public class PrivateTransactionHandler {
public long getSenderNonce(final Address sender, final String privacyGroupId) {
return privateStateStorage
.getPrivateAccountState(BytesValues.fromBase64(privacyGroupId))
.getLatestStateRoot(BytesValues.fromBase64(privacyGroupId))
.map(
lastRootHash ->
privateWorldStateArchive

@ -1,110 +0,0 @@
/*
* Copyright ConsenSys AG.
*
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
* an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
* specific language governing permissions and limitations under the License.
*
* SPDX-License-Identifier: Apache-2.0
*/
package org.hyperledger.besu.ethereum.privacy;
import static java.nio.charset.StandardCharsets.UTF_8;
import org.hyperledger.besu.ethereum.core.Log;
import org.hyperledger.besu.ethereum.core.LogSeries;
import org.hyperledger.besu.ethereum.rlp.RLP;
import org.hyperledger.besu.plugin.services.storage.KeyValueStorage;
import org.hyperledger.besu.plugin.services.storage.KeyValueStorageTransaction;
import org.hyperledger.besu.util.bytes.Bytes32;
import org.hyperledger.besu.util.bytes.BytesValue;
import org.hyperledger.besu.util.bytes.BytesValues;
import java.util.List;
import java.util.Optional;
public class PrivateTransactionKeyValueStorage implements PrivateTransactionStorage {
private final KeyValueStorage keyValueStorage;
private static final BytesValue EVENTS_KEY_SUFFIX = BytesValue.of("EVENTS".getBytes(UTF_8));
private static final BytesValue OUTPUT_KEY_SUFFIX = BytesValue.of("OUTPUT".getBytes(UTF_8));
public PrivateTransactionKeyValueStorage(final KeyValueStorage keyValueStorage) {
this.keyValueStorage = keyValueStorage;
}
@Override
public Optional<List<Log>> getEvents(final Bytes32 transactionHash) {
return get(transactionHash, EVENTS_KEY_SUFFIX).map(this::rlpDecodeLog);
}
@Override
public Optional<BytesValue> getOutput(final Bytes32 transactionHash) {
return get(transactionHash, OUTPUT_KEY_SUFFIX);
}
@Override
public boolean isPrivateStateAvailable(final Bytes32 transactionHash) {
return get(transactionHash, EVENTS_KEY_SUFFIX).isPresent()
|| get(transactionHash, OUTPUT_KEY_SUFFIX).isPresent();
}
private List<Log> rlpDecodeLog(final BytesValue bytes) {
return RLP.input(bytes).readList(Log::readFrom);
}
private Optional<BytesValue> get(final BytesValue key, final BytesValue keySuffix) {
return keyValueStorage
.get(BytesValues.concatenate(key, keySuffix).getArrayUnsafe())
.map(BytesValue::wrap);
}
@Override
public Updater updater() {
return new Updater(keyValueStorage.startTransaction());
}
public static class Updater implements PrivateTransactionStorage.Updater {
private final KeyValueStorageTransaction transaction;
private Updater(final KeyValueStorageTransaction transaction) {
this.transaction = transaction;
}
@Override
public PrivateTransactionStorage.Updater putTransactionLogs(
final Bytes32 transactionHash, final LogSeries logs) {
set(transactionHash, EVENTS_KEY_SUFFIX, RLP.encode(logs::writeTo));
return this;
}
@Override
public PrivateTransactionStorage.Updater putTransactionResult(
final Bytes32 transactionHash, final BytesValue events) {
set(transactionHash, OUTPUT_KEY_SUFFIX, events);
return this;
}
private void set(final BytesValue key, final BytesValue keySuffix, final BytesValue value) {
transaction.put(
BytesValues.concatenate(key, keySuffix).getArrayUnsafe(), value.getArrayUnsafe());
}
@Override
public void commit() {
transaction.commit();
}
@Override
public void rollback() {
transaction.rollback();
}
}
}

@ -0,0 +1,160 @@
/*
* Copyright ConsenSys AG.
*
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
* an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
* specific language governing permissions and limitations under the License.
*
* SPDX-License-Identifier: Apache-2.0
*/
package org.hyperledger.besu.ethereum.privacy.storage;
import static java.nio.charset.StandardCharsets.UTF_8;
import org.hyperledger.besu.ethereum.core.Hash;
import org.hyperledger.besu.ethereum.core.Log;
import org.hyperledger.besu.ethereum.core.LogSeries;
import org.hyperledger.besu.ethereum.rlp.BytesValueRLPInput;
import org.hyperledger.besu.ethereum.rlp.RLP;
import org.hyperledger.besu.plugin.services.storage.KeyValueStorage;
import org.hyperledger.besu.plugin.services.storage.KeyValueStorageTransaction;
import org.hyperledger.besu.util.bytes.Bytes32;
import org.hyperledger.besu.util.bytes.BytesValue;
import org.hyperledger.besu.util.bytes.BytesValues;
import java.util.List;
import java.util.Optional;
public class PrivateStateKeyValueStorage implements PrivateStateStorage {
@Deprecated
private static final BytesValue EVENTS_KEY_SUFFIX = BytesValue.of("EVENTS".getBytes(UTF_8));
private static final BytesValue LOGS_KEY_SUFFIX = BytesValue.of("LOGS".getBytes(UTF_8));
private static final BytesValue OUTPUT_KEY_SUFFIX = BytesValue.of("OUTPUT".getBytes(UTF_8));
private static final BytesValue METADATA_KEY_SUFFIX = BytesValue.of("METADATA".getBytes(UTF_8));
private final KeyValueStorage keyValueStorage;
public PrivateStateKeyValueStorage(final KeyValueStorage keyValueStorage) {
this.keyValueStorage = keyValueStorage;
}
@Override
public Optional<Hash> getLatestStateRoot(final BytesValue privacyId) {
final byte[] id = privacyId.getArrayUnsafe();
if (keyValueStorage.get(id).isPresent()) {
return Optional.of(Hash.wrap(Bytes32.wrap(keyValueStorage.get(id).get())));
} else {
return Optional.empty();
}
}
@Override
public Optional<List<Log>> getTransactionLogs(final Bytes32 transactionHash) {
final Optional<List<Log>> logs = get(transactionHash, LOGS_KEY_SUFFIX).map(this::rlpDecodeLog);
if (logs.isEmpty()) {
return get(transactionHash, EVENTS_KEY_SUFFIX).map(this::rlpDecodeLog);
}
return logs;
}
@Override
public Optional<BytesValue> getTransactionOutput(final Bytes32 transactionHash) {
return get(transactionHash, OUTPUT_KEY_SUFFIX);
}
@Override
public Optional<PrivateTransactionMetadata> getTransactionMetadata(
final Bytes32 blockHash, final Bytes32 transactionHash) {
return get(BytesValues.concatenate(blockHash, transactionHash), METADATA_KEY_SUFFIX)
.map(
bytesValue ->
PrivateTransactionMetadata.readFrom(new BytesValueRLPInput(bytesValue, false)));
}
@Override
public boolean isPrivateStateAvailable(final Bytes32 transactionHash) {
return false;
}
@Override
public boolean isWorldStateAvailable(final Bytes32 rootHash) {
return false;
}
private Optional<BytesValue> get(final BytesValue key, final BytesValue keySuffix) {
return keyValueStorage
.get(BytesValues.concatenate(key, keySuffix).getArrayUnsafe())
.map(BytesValue::wrap);
}
private List<Log> rlpDecodeLog(final BytesValue bytes) {
return RLP.input(bytes).readList(Log::readFrom);
}
@Override
public PrivateStateStorage.Updater updater() {
return new PrivateStateKeyValueStorage.Updater(keyValueStorage.startTransaction());
}
public static class Updater implements PrivateStateStorage.Updater {
private final KeyValueStorageTransaction transaction;
private Updater(final KeyValueStorageTransaction transaction) {
this.transaction = transaction;
}
@Override
public Updater putLatestStateRoot(final BytesValue privacyId, final Hash privateStateHash) {
transaction.put(privacyId.getArrayUnsafe(), privateStateHash.extractArray());
return this;
}
@Override
public Updater putTransactionLogs(final Bytes32 transactionHash, final LogSeries logs) {
set(transactionHash, LOGS_KEY_SUFFIX, RLP.encode(logs::writeTo));
return this;
}
@Override
public Updater putTransactionResult(final Bytes32 transactionHash, final BytesValue events) {
set(transactionHash, OUTPUT_KEY_SUFFIX, events);
return this;
}
@Override
public Updater putTransactionMetadata(
final Bytes32 blockHash,
final Bytes32 transactionHash,
final PrivateTransactionMetadata metadata) {
set(
BytesValues.concatenate(blockHash, transactionHash),
METADATA_KEY_SUFFIX,
RLP.encode(metadata::writeTo));
return this;
}
@Override
public void commit() {
transaction.commit();
}
@Override
public void rollback() {
transaction.rollback();
}
private void set(final BytesValue key, final BytesValue keySuffix, final BytesValue value) {
transaction.put(
BytesValues.concatenate(key, keySuffix).getArrayUnsafe(), value.getArrayUnsafe());
}
}
}

@ -12,8 +12,9 @@
*
* SPDX-License-Identifier: Apache-2.0
*/
package org.hyperledger.besu.ethereum.privacy;
package org.hyperledger.besu.ethereum.privacy.storage;
import org.hyperledger.besu.ethereum.core.Hash;
import org.hyperledger.besu.ethereum.core.Log;
import org.hyperledger.besu.ethereum.core.LogSeries;
import org.hyperledger.besu.util.bytes.Bytes32;
@ -22,22 +23,36 @@ import org.hyperledger.besu.util.bytes.BytesValue;
import java.util.List;
import java.util.Optional;
public interface PrivateTransactionStorage {
public interface PrivateStateStorage {
Optional<List<Log>> getEvents(Bytes32 transactionHash);
@Deprecated
Optional<Hash> getLatestStateRoot(BytesValue privacyId);
Optional<BytesValue> getOutput(Bytes32 transactionHash);
Optional<List<Log>> getTransactionLogs(Bytes32 transactionHash);
Optional<BytesValue> getTransactionOutput(Bytes32 transactionHash);
Optional<PrivateTransactionMetadata> getTransactionMetadata(
Bytes32 blockHash, Bytes32 transactionHash);
boolean isPrivateStateAvailable(Bytes32 transactionHash);
boolean isWorldStateAvailable(Bytes32 rootHash);
Updater updater();
interface Updater {
@Deprecated
Updater putLatestStateRoot(BytesValue privacyId, Hash privateStateHash);
Updater putTransactionLogs(Bytes32 transactionHash, LogSeries logs);
Updater putTransactionResult(Bytes32 transactionHash, BytesValue events);
Updater putTransactionMetadata(
Bytes32 blockHash, Bytes32 transactionHash, PrivateTransactionMetadata metadata);
void commit();
void rollback();

@ -0,0 +1,50 @@
/*
* Copyright 2019 ConsenSys AG.
*
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
* an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
* specific language governing permissions and limitations under the License.
*
* SPDX-License-Identifier: Apache-2.0
*/
package org.hyperledger.besu.ethereum.privacy.storage;
import org.hyperledger.besu.ethereum.core.Hash;
import org.hyperledger.besu.ethereum.rlp.RLPInput;
import org.hyperledger.besu.ethereum.rlp.RLPOutput;
/** Mined private transaction metadata. */
public class PrivateTransactionMetadata {
private final Hash stateRoot;
public PrivateTransactionMetadata(final Hash stateRoot) {
this.stateRoot = stateRoot;
}
public Hash getStateRoot() {
return stateRoot;
}
public void writeTo(final RLPOutput out) {
out.startList();
out.writeBytesValue(stateRoot);
out.endList();
}
public static PrivateTransactionMetadata readFrom(final RLPInput input) {
input.enterList();
final PrivateTransactionMetadata privateTransactionMetadata =
new PrivateTransactionMetadata(Hash.wrap(input.readBytes32()));
input.leaveList();
return privateTransactionMetadata;
}
}

@ -16,8 +16,7 @@ package org.hyperledger.besu.ethereum.storage;
import org.hyperledger.besu.ethereum.chain.BlockchainStorage;
import org.hyperledger.besu.ethereum.mainnet.ProtocolSchedule;
import org.hyperledger.besu.ethereum.privacy.PrivateStateStorage;
import org.hyperledger.besu.ethereum.privacy.PrivateTransactionStorage;
import org.hyperledger.besu.ethereum.privacy.storage.PrivateStateStorage;
import org.hyperledger.besu.ethereum.worldstate.WorldStatePreimageStorage;
import org.hyperledger.besu.ethereum.worldstate.WorldStateStorage;
import org.hyperledger.besu.plugin.services.storage.KeyValueStorage;
@ -32,8 +31,6 @@ public interface StorageProvider extends Closeable {
WorldStatePreimageStorage createWorldStatePreimageStorage();
PrivateTransactionStorage createPrivateTransactionStorage();
PrivateStateStorage createPrivateStateStorage();
KeyValueStorage createPruningStorage();

@ -17,10 +17,8 @@ package org.hyperledger.besu.ethereum.storage.keyvalue;
import org.hyperledger.besu.ethereum.chain.BlockchainStorage;
import org.hyperledger.besu.ethereum.mainnet.ProtocolSchedule;
import org.hyperledger.besu.ethereum.mainnet.ScheduleBasedBlockHeaderFunctions;
import org.hyperledger.besu.ethereum.privacy.PrivateStateKeyValueStorage;
import org.hyperledger.besu.ethereum.privacy.PrivateStateStorage;
import org.hyperledger.besu.ethereum.privacy.PrivateTransactionKeyValueStorage;
import org.hyperledger.besu.ethereum.privacy.PrivateTransactionStorage;
import org.hyperledger.besu.ethereum.privacy.storage.PrivateStateKeyValueStorage;
import org.hyperledger.besu.ethereum.privacy.storage.PrivateStateStorage;
import org.hyperledger.besu.ethereum.storage.StorageProvider;
import org.hyperledger.besu.ethereum.worldstate.WorldStatePreimageStorage;
import org.hyperledger.besu.ethereum.worldstate.WorldStateStorage;
@ -71,11 +69,6 @@ public class KeyValueStorageProvider implements StorageProvider {
return new WorldStatePreimageKeyValueStorage(worldStatePreimageStorage);
}
@Override
public PrivateTransactionStorage createPrivateTransactionStorage() {
return new PrivateTransactionKeyValueStorage(privateTransactionStorage);
}
@Override
public PrivateStateStorage createPrivateStateStorage() {
return new PrivateStateKeyValueStorage(privateStateStorage);

@ -20,10 +20,8 @@ import org.hyperledger.besu.ethereum.chain.MutableBlockchain;
import org.hyperledger.besu.ethereum.mainnet.MainnetBlockHeaderFunctions;
import org.hyperledger.besu.ethereum.mainnet.ProtocolSchedule;
import org.hyperledger.besu.ethereum.mainnet.ScheduleBasedBlockHeaderFunctions;
import org.hyperledger.besu.ethereum.privacy.PrivateStateKeyValueStorage;
import org.hyperledger.besu.ethereum.privacy.PrivateStateStorage;
import org.hyperledger.besu.ethereum.privacy.PrivateTransactionKeyValueStorage;
import org.hyperledger.besu.ethereum.privacy.PrivateTransactionStorage;
import org.hyperledger.besu.ethereum.privacy.storage.PrivateStateKeyValueStorage;
import org.hyperledger.besu.ethereum.privacy.storage.PrivateStateStorage;
import org.hyperledger.besu.ethereum.storage.StorageProvider;
import org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueStoragePrefixedKeyBlockchainStorage;
import org.hyperledger.besu.ethereum.storage.keyvalue.WorldStateKeyValueStorage;
@ -79,11 +77,6 @@ public class InMemoryStorageProvider implements StorageProvider {
return new WorldStatePreimageKeyValueStorage(new InMemoryKeyValueStorage());
}
@Override
public PrivateTransactionStorage createPrivateTransactionStorage() {
return new PrivateTransactionKeyValueStorage(new InMemoryKeyValueStorage());
}
@Override
public PrivateStateStorage createPrivateStateStorage() {
return new PrivateStateKeyValueStorage(new InMemoryKeyValueStorage());

@ -32,10 +32,9 @@ import org.hyperledger.besu.ethereum.core.MutableWorldState;
import org.hyperledger.besu.ethereum.core.ProcessableBlockHeader;
import org.hyperledger.besu.ethereum.core.WorldUpdater;
import org.hyperledger.besu.ethereum.mainnet.SpuriousDragonGasCalculator;
import org.hyperledger.besu.ethereum.privacy.PrivateStateStorage;
import org.hyperledger.besu.ethereum.privacy.PrivateTransaction;
import org.hyperledger.besu.ethereum.privacy.PrivateTransactionProcessor;
import org.hyperledger.besu.ethereum.privacy.PrivateTransactionStorage;
import org.hyperledger.besu.ethereum.privacy.storage.PrivateStateStorage;
import org.hyperledger.besu.ethereum.vm.BlockHashLookup;
import org.hyperledger.besu.ethereum.vm.MessageFrame;
import org.hyperledger.besu.ethereum.vm.OperationTracer;
@ -79,17 +78,17 @@ public class PrivacyPrecompiledContractTest {
.extractArray());
private Enclave mockEnclave() {
Enclave mockEnclave = mock(Enclave.class);
ReceiveResponse response = new ReceiveResponse(VALID_PRIVATE_TRANSACTION_RLP_BASE64, "");
final Enclave mockEnclave = mock(Enclave.class);
final ReceiveResponse response = new ReceiveResponse(VALID_PRIVATE_TRANSACTION_RLP_BASE64, "");
when(mockEnclave.receive(any(ReceiveRequest.class))).thenReturn(response);
return mockEnclave;
}
private PrivateTransactionProcessor mockPrivateTxProcessor() {
PrivateTransactionProcessor mockPrivateTransactionProcessor =
final PrivateTransactionProcessor mockPrivateTransactionProcessor =
mock(PrivateTransactionProcessor.class);
LogSeries logs = mock(LogSeries.class);
PrivateTransactionProcessor.Result result =
final LogSeries logs = mock(LogSeries.class);
final PrivateTransactionProcessor.Result result =
PrivateTransactionProcessor.Result.successful(
logs, 0, BytesValue.fromHexString(DEFAULT_OUTPUT), null);
when(mockPrivateTransactionProcessor.processTransaction(
@ -108,29 +107,27 @@ public class PrivacyPrecompiledContractTest {
}
private Enclave brokenMockEnclave() {
Enclave mockEnclave = mock(Enclave.class);
final Enclave mockEnclave = mock(Enclave.class);
when(mockEnclave.receive(any(ReceiveRequest.class))).thenThrow(EnclaveException.class);
return mockEnclave;
}
@Before
public void setUp() {
WorldStateArchive worldStateArchive;
final WorldStateArchive worldStateArchive;
worldStateArchive = mock(WorldStateArchive.class);
MutableWorldState mutableWorldState = mock(MutableWorldState.class);
final MutableWorldState mutableWorldState = mock(MutableWorldState.class);
when(mutableWorldState.updater()).thenReturn(mock(WorldUpdater.class));
when(worldStateArchive.getMutable()).thenReturn(mutableWorldState);
when(worldStateArchive.getMutable(any())).thenReturn(Optional.of(mutableWorldState));
PrivateTransactionStorage privateTransactionStorage = mock(PrivateTransactionStorage.class);
PrivateTransactionStorage.Updater updater = mock(PrivateTransactionStorage.Updater.class);
when(updater.putTransactionLogs(nullable(Bytes32.class), any())).thenReturn(updater);
when(updater.putTransactionResult(nullable(Bytes32.class), any())).thenReturn(updater);
when(privateTransactionStorage.updater()).thenReturn(updater);
PrivateStateStorage privateStateStorage = mock(PrivateStateStorage.class);
PrivateStateStorage.Updater storageUpdater = mock(PrivateStateStorage.Updater.class);
when(storageUpdater.putPrivateAccountState(nullable(Bytes32.class), any()))
final PrivateStateStorage privateStateStorage = mock(PrivateStateStorage.class);
final PrivateStateStorage.Updater storageUpdater = mock(PrivateStateStorage.Updater.class);
when(storageUpdater.putLatestStateRoot(nullable(Bytes32.class), any()))
.thenReturn(storageUpdater);
when(storageUpdater.putTransactionLogs(nullable(Bytes32.class), any()))
.thenReturn(storageUpdater);
when(storageUpdater.putTransactionResult(nullable(Bytes32.class), any()))
.thenReturn(storageUpdater);
when(privateStateStorage.updater()).thenReturn(storageUpdater);
@ -140,7 +137,6 @@ public class PrivacyPrecompiledContractTest {
publicKey,
mockEnclave(),
worldStateArchive,
privateTransactionStorage,
privateStateStorage);
privacyPrecompiledContract.setPrivateTransactionProcessor(mockPrivateTxProcessor());
brokenPrivateTransactionHandler =
@ -149,14 +145,12 @@ public class PrivacyPrecompiledContractTest {
publicKey,
brokenMockEnclave(),
worldStateArchive,
privateTransactionStorage,
privateStateStorage);
messageFrame = mock(MessageFrame.class);
}
@Test
public void testPrivacyPrecompiledContract() {
final BytesValue actual = privacyPrecompiledContract.compute(key, messageFrame);
assertThat(actual).isEqualTo(BytesValue.fromHexString(DEFAULT_OUTPUT));

@ -39,6 +39,7 @@ import org.hyperledger.besu.ethereum.core.Wei;
import org.hyperledger.besu.ethereum.mainnet.TransactionValidator.TransactionInvalidReason;
import org.hyperledger.besu.ethereum.mainnet.ValidationResult;
import org.hyperledger.besu.ethereum.privacy.markertransaction.FixedKeySigningPrivateMarkerTransactionFactory;
import org.hyperledger.besu.ethereum.privacy.storage.PrivateStateStorage;
import org.hyperledger.besu.ethereum.worldstate.WorldStateArchive;
import org.hyperledger.besu.util.bytes.BytesValue;
import org.hyperledger.besu.util.bytes.BytesValues;
@ -104,7 +105,7 @@ public class PrivateTransactionHandlerTest {
public void setUp() throws Exception {
PrivateStateStorage privateStateStorage = mock(PrivateStateStorage.class);
Hash mockHash = mock(Hash.class);
when(privateStateStorage.getPrivateAccountState(any(BytesValue.class)))
when(privateStateStorage.getLatestStateRoot(any(BytesValue.class)))
.thenReturn(Optional.of(mockHash));
WorldStateArchive worldStateArchive = mock(WorldStateArchive.class);
Account account = mock(Account.class);

@ -56,7 +56,7 @@ Calculated : ${currentHash}
tasks.register('checkAPIChanges', FileStateChecker) {
description = "Checks that the API for the Plugin-API project does not change without deliberate thought"
files = sourceSets.main.allJava.files
knownHash = 'f0SPUj4/aLZKyjAGcDYai021dO8pg5xLaNvALEWxoIg='
knownHash = 'azykDVSB2DOA/rgZyedO8Uwe1eGDGRCMUAZN4FMB3R4='
}
check.dependsOn('checkAPIChanges')

@ -19,6 +19,7 @@ import org.hyperledger.besu.plugin.services.exception.StorageException;
import java.io.Closeable;
import java.util.Optional;
import java.util.Set;
import java.util.function.Predicate;
/**
@ -70,6 +71,15 @@ public interface KeyValueStorage extends Closeable {
*/
long removeAllKeysUnless(Predicate<byte[]> retainCondition) throws StorageException;
/**
* Performs an evaluation against each key in the store, returning the set of entries that pass.
*
* @param returnCondition predicate to evaluate each key against, unless the result is {@code
* null}, the key is added to the returned list of keys.
* @return the set of keys that pass the condition.
*/
Set<byte[]> getAllKeysThat(Predicate<byte[]> returnCondition);
/**
* Begins a fresh transaction, for sequencing operations for later atomic execution.
*

@ -33,11 +33,13 @@ import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.Set;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.function.Predicate;
import java.util.stream.Collectors;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Sets;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.rocksdb.BlockBasedTableConfig;
@ -161,7 +163,7 @@ public class RocksDBColumnarKeyValueStorage
}
@Override
public long removeUnless(
public long removeAllEntriesUnless(
final ColumnFamilyHandle segmentHandle, final Predicate<byte[]> inUseCheck) {
long removedNodeCounter = 0;
try (final RocksIterator rocksIterator = db.newIterator(segmentHandle)) {
@ -180,6 +182,23 @@ public class RocksDBColumnarKeyValueStorage
return removedNodeCounter;
}
@Override
public Set<byte[]> getAllKeysThat(
final ColumnFamilyHandle segmentHandle, final Predicate<byte[]> returnCondition) {
final Set<byte[]> returnedKeys = Sets.newIdentityHashSet();
try (final RocksIterator rocksIterator = db.newIterator(segmentHandle)) {
rocksIterator.seekToFirst();
while (rocksIterator.isValid()) {
final byte[] key = rocksIterator.key();
if (returnCondition.test(key)) {
returnedKeys.add(key);
}
rocksIterator.next();
}
}
return returnedKeys;
}
@Override
public void clear(final ColumnFamilyHandle segmentHandle) {
try (final RocksIterator rocksIterator = db.newIterator(segmentHandle)) {

@ -25,9 +25,11 @@ import org.hyperledger.besu.plugin.services.storage.rocksdb.configuration.RocksD
import org.hyperledger.besu.services.kvstore.KeyValueStorageTransactionTransitionValidatorDecorator;
import java.util.Optional;
import java.util.Set;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.function.Predicate;
import com.google.common.collect.Sets;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.rocksdb.BlockBasedTableConfig;
@ -130,6 +132,22 @@ public class RocksDBKeyValueStorage implements KeyValueStorage {
return removedNodeCounter;
}
@Override
public Set<byte[]> getAllKeysThat(final Predicate<byte[]> returnCondition) {
final Set<byte[]> returnedKeys = Sets.newIdentityHashSet();
try (final RocksIterator rocksIterator = db.newIterator()) {
rocksIterator.seekToFirst();
while (rocksIterator.isValid()) {
final byte[] key = rocksIterator.key();
if (returnCondition.test(key)) {
returnedKeys.add(key);
}
rocksIterator.next();
}
}
return returnedKeys;
}
@Override
public KeyValueStorageTransaction startTransaction() throws StorageException {
throwIfClosed();

@ -29,6 +29,7 @@ import org.hyperledger.besu.services.kvstore.SegmentedKeyValueStorageAdapter;
import java.nio.charset.StandardCharsets;
import java.util.Arrays;
import java.util.Optional;
import java.util.Set;
import org.junit.Rule;
import org.junit.Test;
@ -43,7 +44,7 @@ public class RocksDBColumnarKeyValueStorageTest extends AbstractKeyValueStorageT
public void twoSegmentsAreIndependent() throws Exception {
final SegmentedKeyValueStorage<ColumnFamilyHandle> store = createSegmentedStore();
Transaction<ColumnFamilyHandle> tx = store.startTransaction();
final Transaction<ColumnFamilyHandle> tx = store.startTransaction();
tx.put(
store.getSegmentIdentifierByName(TestSegment.BAR),
bytesFromHexString("0001"),
@ -62,7 +63,7 @@ public class RocksDBColumnarKeyValueStorageTest extends AbstractKeyValueStorageT
final ColumnFamilyHandle fooSegment = store.getSegmentIdentifierByName(TestSegment.FOO);
final ColumnFamilyHandle barSegment = store.getSegmentIdentifierByName(TestSegment.BAR);
Transaction<ColumnFamilyHandle> tx = store.startTransaction();
final Transaction<ColumnFamilyHandle> tx = store.startTransaction();
tx.put(fooSegment, bytesOf(1), bytesOf(1));
tx.put(fooSegment, bytesOf(2), bytesOf(2));
tx.put(fooSegment, bytesOf(3), bytesOf(3));
@ -71,8 +72,10 @@ public class RocksDBColumnarKeyValueStorageTest extends AbstractKeyValueStorageT
tx.put(barSegment, bytesOf(6), bytesOf(6));
tx.commit();
final long removedFromFoo = store.removeUnless(fooSegment, x -> Arrays.equals(x, bytesOf(3)));
final long removedFromBar = store.removeUnless(barSegment, x -> Arrays.equals(x, bytesOf(4)));
final long removedFromFoo =
store.removeAllEntriesUnless(fooSegment, x -> Arrays.equals(x, bytesOf(3)));
final long removedFromBar =
store.removeAllEntriesUnless(barSegment, x -> Arrays.equals(x, bytesOf(4)));
assertThat(removedFromFoo).isEqualTo(2);
assertThat(removedFromBar).isEqualTo(2);
@ -86,6 +89,37 @@ public class RocksDBColumnarKeyValueStorageTest extends AbstractKeyValueStorageT
assertThat(store.get(barSegment, bytesOf(6))).isEmpty();
}
@Test
public void canGetThroughSegmentIteration() throws Exception {
final SegmentedKeyValueStorage<ColumnFamilyHandle> store = createSegmentedStore();
final ColumnFamilyHandle fooSegment = store.getSegmentIdentifierByName(TestSegment.FOO);
final ColumnFamilyHandle barSegment = store.getSegmentIdentifierByName(TestSegment.BAR);
final Transaction<ColumnFamilyHandle> tx = store.startTransaction();
tx.put(fooSegment, bytesOf(1), bytesOf(1));
tx.put(fooSegment, bytesOf(2), bytesOf(2));
tx.put(fooSegment, bytesOf(3), bytesOf(3));
tx.put(barSegment, bytesOf(4), bytesOf(4));
tx.put(barSegment, bytesOf(5), bytesOf(5));
tx.put(barSegment, bytesOf(6), bytesOf(6));
tx.commit();
final Set<byte[]> gotFromFoo =
store.getAllKeysThat(fooSegment, x -> Arrays.equals(x, bytesOf(3)));
final Set<byte[]> gotFromBar =
store.getAllKeysThat(
barSegment, x -> Arrays.equals(x, bytesOf(4)) || Arrays.equals(x, bytesOf(5)));
final Set<byte[]> gotEmpty =
store.getAllKeysThat(fooSegment, x -> Arrays.equals(x, bytesOf(0)));
assertThat(gotFromFoo.size()).isEqualTo(1);
assertThat(gotFromBar.size()).isEqualTo(2);
assertThat(gotEmpty).isEmpty();
assertThat(gotFromFoo).containsExactlyInAnyOrder(bytesOf(3));
assertThat(gotFromBar).containsExactlyInAnyOrder(bytesOf(4), bytesOf(5));
}
public enum TestSegment implements SegmentIdentifier {
FOO(new byte[] {1}),
BAR(new byte[] {2});

@ -28,6 +28,7 @@ import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReadWriteLock;
import java.util.concurrent.locks.ReentrantReadWriteLock;
import java.util.function.Predicate;
import java.util.stream.Collectors;
public class InMemoryKeyValueStorage implements KeyValueStorage {
@ -88,6 +89,14 @@ public class InMemoryKeyValueStorage implements KeyValueStorage {
}
}
@Override
public Set<byte[]> getAllKeysThat(final Predicate<byte[]> returnCondition) {
return hashValueStore.keySet().stream()
.map(BytesValue::getArrayUnsafe)
.filter(returnCondition)
.collect(Collectors.toSet());
}
@Override
public void close() {}

@ -28,6 +28,7 @@ import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReadWriteLock;
import java.util.concurrent.locks.ReentrantReadWriteLock;
import java.util.function.Predicate;
import java.util.stream.Collectors;
import com.google.common.cache.Cache;
import com.google.common.cache.CacheBuilder;
@ -88,6 +89,14 @@ public class LimitedInMemoryKeyValueStorage implements KeyValueStorage {
return initialSize - storage.size();
}
@Override
public Set<byte[]> getAllKeysThat(final Predicate<byte[]> returnCondition) {
return storage.asMap().keySet().stream()
.map(BytesValue::getArrayUnsafe)
.filter(returnCondition)
.collect(Collectors.toSet());
}
@Override
public KeyValueStorageTransaction startTransaction() throws StorageException {
return new KeyValueStorageTransactionTransitionValidatorDecorator(new MemoryTransaction());

@ -19,6 +19,7 @@ import org.hyperledger.besu.plugin.services.storage.SegmentIdentifier;
import java.io.Closeable;
import java.util.Optional;
import java.util.Set;
import java.util.function.Predicate;
/**
@ -48,7 +49,9 @@ public interface SegmentedKeyValueStorage<S> extends Closeable {
*/
Transaction<S> startTransaction() throws StorageException;
long removeUnless(S segmentHandle, Predicate<byte[]> inUseCheck);
long removeAllEntriesUnless(S segmentHandle, Predicate<byte[]> inUseCheck);
Set<byte[]> getAllKeysThat(S segmentHandle, Predicate<byte[]> returnCondition);
void clear(S segmentHandle);

@ -21,6 +21,7 @@ import org.hyperledger.besu.plugin.services.storage.SegmentIdentifier;
import java.io.IOException;
import java.util.Optional;
import java.util.Set;
import java.util.function.Predicate;
public class SegmentedKeyValueStorageAdapter<S> implements KeyValueStorage {
@ -51,7 +52,12 @@ public class SegmentedKeyValueStorageAdapter<S> implements KeyValueStorage {
@Override
public long removeAllKeysUnless(final Predicate<byte[]> retainCondition) throws StorageException {
return storage.removeUnless(segmentHandle, retainCondition);
return storage.removeAllEntriesUnless(segmentHandle, retainCondition);
}
@Override
public Set<byte[]> getAllKeysThat(final Predicate<byte[]> returnCondition) {
return storage.getAllKeysThat(segmentHandle, returnCondition);
}
@Override

@ -25,6 +25,7 @@ import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.Optional;
import java.util.Set;
import java.util.concurrent.CountDownLatch;
import java.util.function.Function;
@ -86,6 +87,22 @@ public abstract class AbstractKeyValueStorageTest {
assertThat(store.containsKey(bytesFromHexString("12"))).isTrue();
}
@Test
public void getAllKeysThat() throws Exception {
final KeyValueStorage store = createStore();
final KeyValueStorageTransaction tx = store.startTransaction();
tx.put(bytesFromHexString("0F"), bytesFromHexString("0ABC"));
tx.put(bytesFromHexString("10"), bytesFromHexString("0ABC"));
tx.put(bytesFromHexString("11"), bytesFromHexString("0ABC"));
tx.put(bytesFromHexString("12"), bytesFromHexString("0ABC"));
tx.commit();
Set<byte[]> keys = store.getAllKeysThat(bv -> BytesValue.wrap(bv).toString().contains("1"));
assertThat(keys.size()).isEqualTo(3);
assertThat(keys)
.containsExactlyInAnyOrder(
bytesFromHexString("10"), bytesFromHexString("11"), bytesFromHexString("12"));
}
@Test
public void containsKey() throws Exception {
final KeyValueStorage store = createStore();

Loading…
Cancel
Save