mirror of https://github.com/hyperledger/besu
[Pruning Bugfix] Prevent race condition in key deletion. (#760)
* add doomed key check (busy-waiting for now) Signed-off-by: Ratan Rai Sur <ratan.r.sur@gmail.com> * optional and logging Signed-off-by: Ratan Rai Sur <ratan.r.sur@gmail.com> * remove logging Signed-off-by: Ratan Rai Sur <ratan.r.sur@gmail.com> * sleeping and hardening Signed-off-by: Ratan Rai Sur <ratan.r.sur@gmail.com> * rename segments Signed-off-by: Ratan Rai Sur <ratan.r.sur@gmail.com> * move away from atomic references to regular vars Signed-off-by: Ratan Rai Sur <ratan.r.sur@gmail.com> * remove hardened segment parameter Signed-off-by: Ratan Rai Sur <ratan.r.sur@gmail.com> * increase sleep Signed-off-by: Ratan Rai Sur <ratan.r.sur@gmail.com> * spotless Signed-off-by: Ratan Rai Sur <ratan.r.sur@gmail.com> * remove unnecessary interface Signed-off-by: Ratan Rai Sur <ratan.r.sur@gmail.com> * rename Signed-off-by: Ratan Rai Sur <ratan.r.sur@gmail.com> * move commit waiting outside of timer Signed-off-by: Ratan Rai Sur <ratan.r.sur@gmail.com> * set default lock timeout to 1ms Signed-off-by: Ratan Rai Sur <ratan.r.sur@gmail.com> * add default lock timeout to tests Signed-off-by: Ratan Rai Sur <ratan.r.sur@gmail.com> * Revert "rename segments" This reverts commit 184eefaaa0ccc857b0caff2b382f8338ff225d5d. Signed-off-by: Ratan Rai Sur <ratan.r.sur@gmail.com> * fix jmh compilation error Signed-off-by: Ratan Rai Sur <ratan.r.sur@gmail.com> * add documentation Signed-off-by: Ratan Rai Sur <ratan.r.sur@gmail.com> * bump up sleep to 1ms Signed-off-by: Ratan Rai Sur <ratan.r.sur@gmail.com> * (POC) Add lock to ensure that we don't prune while comitting Signed-off-by: Ratan Rai Sur <ratan.r.sur@gmail.com> * remove unnecessary persist (#569) Signed-off-by: Ratan Rai Sur <ratan.r.sur@gmail.com> * flesh out @mbaxter's idea and remove my code Signed-off-by: Ratan Rai Sur <ratan.r.sur@gmail.com> * iterator changes Signed-off-by: Ratan Rai Sur <ratan.r.sur@gmail.com> * hybridize with doomed key Signed-off-by: Ratan Rai Sur <ratan.r.sur@gmail.com> * comment Signed-off-by: Ratan Rai Sur <ratan.r.sur@gmail.com> * move doomed key unset to after node added listener Signed-off-by: Ratan Rai Sur <ratan.r.sur@gmail.com> * update instead of getting and setting doomedKeyRef in commit Signed-off-by: Ratan Rai Sur <ratan.r.sur@gmail.com> * comment Signed-off-by: Ratan Rai Sur <ratan.r.sur@gmail.com> * invert condition Signed-off-by: Ratan Rai Sur <ratan.r.sur@gmail.com> * remove locks Signed-off-by: Ratan Rai Sur <ratan.r.sur@gmail.com> * remove `removeAllKeysUnless` Signed-off-by: Ratan Rai Sur <ratan.r.sur@gmail.com> * more remove removeAllKeysUnless Signed-off-by: Ratan Rai Sur <ratan.r.sur@gmail.com> * reuse streamKeys Signed-off-by: Ratan Rai Sur <ratan.r.sur@gmail.com> * remove test Signed-off-by: Ratan Rai Sur <ratan.r.sur@gmail.com> * set default lock timeout to 1ms Signed-off-by: Ratan Rai Sur <ratan.r.sur@gmail.com> * add default lock timeout to tests Signed-off-by: Ratan Rai Sur <ratan.r.sur@gmail.com> * fix jmh compilation error Signed-off-by: Ratan Rai Sur <ratan.r.sur@gmail.com> * revert back to locks instead of doomedkey Signed-off-by: Ratan Rai Sur <ratan.r.sur@gmail.com> * change delete to not guarantee deletion Signed-off-by: Ratan Rai Sur <ratan.r.sur@gmail.com> * plugin hash Signed-off-by: Ratan Rai Sur <ratan.r.sur@gmail.com> * javadoc Signed-off-by: Ratan Rai Sur <ratan.r.sur@gmail.com> * Revert "change delete to not guarantee deletion" This reverts commitpull/857/head2289bb34cf. Signed-off-by: Ratan Rai Sur <ratan.r.sur@gmail.com> * skip key deletion on timeout Signed-off-by: Ratan Rai Sur <ratan.r.sur@gmail.com> * clear in rollback Signed-off-by: Ratan Rai Sur <ratan.r.sur@gmail.com> * Revert "fix jmh compilation error" This reverts commitb64ecf8656. Signed-off-by: Ratan Rai Sur <ratan.r.sur@gmail.com> * Revert "add default lock timeout to tests" This reverts commitaff6aa6065. Signed-off-by: Ratan Rai Sur <ratan.r.sur@gmail.com> * Revert "set default lock timeout to 1ms" This reverts commit267fe0a642. Signed-off-by: Ratan Rai Sur <ratan.r.sur@gmail.com> * use noSlowDown write option instead of global timeout Signed-off-by: Ratan Rai Sur <ratan.r.sur@gmail.com> * add back tests Signed-off-by: Ratan Rai Sur <ratan.r.sur@gmail.com> * close tryDeleteOptions Signed-off-by: Ratan Rai Sur <ratan.r.sur@gmail.com> * remove unnecessary lock Signed-off-by: Ratan Rai Sur <ratan.r.sur@gmail.com> * move increment inside try Signed-off-by: Ratan Rai Sur <ratan.r.sur@gmail.com> * use StorageException subclass instead of field Signed-off-by: Ratan Rai Sur <ratan.r.sur@gmail.com> * revert accidental deletion in javadoc Signed-off-by: Ratan Rai Sur <ratan.r.sur@gmail.com> * tryDelete javadoc Signed-off-by: Ratan Rai Sur <ratan.r.sur@gmail.com> * add trace for skipping key deletion Signed-off-by: Ratan Rai Sur <ratan.r.sur@gmail.com> * merge catch and finally try blocks Signed-off-by: Ratan Rai Sur <ratan.r.sur@gmail.com> * switch from exception to boolean return value Signed-off-by: Ratan Rai Sur <ratan.r.sur@gmail.com> * tweak Signed-off-by: Ratan Rai Sur <ratan.r.sur@gmail.com> * changelog changes Signed-off-by: Ratan Rai Sur <ratan.r.sur@gmail.com> * add api back Signed-off-by: Ratan Rai Sur <ratan.r.sur@gmail.com> * add back throws javadoc Signed-off-by: Ratan Rai Sur <ratan.r.sur@gmail.com> Co-authored-by: Meredith Baxter <meredith.baxter@consensys.net> Co-authored-by: MadelineMurray <43356962+MadelineMurray@users.noreply.github.com>
Ratan Rai Sur
5 years ago
committed by
GitHub
parent
69f6493f91
commit
43eccbbb67
@ -0,0 +1,97 @@ |
||||
/* |
||||
* Copyright ConsenSys AG. |
||||
* |
||||
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with |
||||
* the License. You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on |
||||
* an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the |
||||
* specific language governing permissions and limitations under the License. |
||||
* |
||||
* SPDX-License-Identifier: Apache-2.0 |
||||
*/ |
||||
|
||||
package org.hyperledger.besu.plugin.services.storage.rocksdb; |
||||
|
||||
import static com.google.common.base.Preconditions.checkState; |
||||
|
||||
import java.util.Iterator; |
||||
import java.util.NoSuchElementException; |
||||
import java.util.Spliterator; |
||||
import java.util.Spliterators; |
||||
import java.util.concurrent.atomic.AtomicBoolean; |
||||
import java.util.stream.Stream; |
||||
import java.util.stream.StreamSupport; |
||||
|
||||
import org.apache.logging.log4j.LogManager; |
||||
import org.apache.logging.log4j.Logger; |
||||
import org.rocksdb.RocksDBException; |
||||
import org.rocksdb.RocksIterator; |
||||
|
||||
public class RocksDbKeyIterator implements Iterator<byte[]>, AutoCloseable { |
||||
private static final Logger LOG = LogManager.getLogger(); |
||||
|
||||
private final RocksIterator rocksIterator; |
||||
private final AtomicBoolean closed = new AtomicBoolean(false); |
||||
|
||||
private RocksDbKeyIterator(final RocksIterator rocksIterator) { |
||||
this.rocksIterator = rocksIterator; |
||||
} |
||||
|
||||
public static RocksDbKeyIterator create(final RocksIterator rocksIterator) { |
||||
return new RocksDbKeyIterator(rocksIterator); |
||||
} |
||||
|
||||
@Override |
||||
public boolean hasNext() { |
||||
assertOpen(); |
||||
return rocksIterator.isValid(); |
||||
} |
||||
|
||||
@Override |
||||
public byte[] next() { |
||||
assertOpen(); |
||||
try { |
||||
rocksIterator.status(); |
||||
} catch (final RocksDBException e) { |
||||
LOG.error( |
||||
String.format("%s encountered a problem while iterating.", getClass().getSimpleName()), |
||||
e); |
||||
} |
||||
if (!hasNext()) { |
||||
throw new NoSuchElementException(); |
||||
} |
||||
final byte[] key = rocksIterator.key(); |
||||
rocksIterator.next(); |
||||
return key; |
||||
} |
||||
|
||||
public Stream<byte[]> toStream() { |
||||
assertOpen(); |
||||
final Spliterator<byte[]> spliterator = |
||||
Spliterators.spliteratorUnknownSize( |
||||
this, |
||||
Spliterator.IMMUTABLE |
||||
| Spliterator.DISTINCT |
||||
| Spliterator.NONNULL |
||||
| Spliterator.ORDERED |
||||
| Spliterator.SORTED); |
||||
|
||||
return StreamSupport.stream(spliterator, false).onClose(this::close); |
||||
} |
||||
|
||||
private void assertOpen() { |
||||
checkState( |
||||
!closed.get(), |
||||
String.format("Attempt to read from a closed %s", getClass().getSimpleName())); |
||||
} |
||||
|
||||
@Override |
||||
public void close() { |
||||
if (closed.compareAndSet(false, true)) { |
||||
rocksIterator.close(); |
||||
} |
||||
} |
||||
} |
||||
Loading…
Reference in new issue