CipherLabs
/
besu
mirror of https://github.com/hyperledger/besu
2
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Reduce Trace Memory Consumption (#344)

Browse Source 
Switch the EVM Memory to a copy on write structure, so when the trace
reads the memory they don't have to copy mutable bytes but can instead
simply use the value as immutable.

Also, fix a state diff bug when an account is deleted that was also just
created in the TX don't report it as an all equal diff, as well as new accounts storage diffs.

Signed-off-by: Danno Ferrin <danno.ferrin@gmail.com>
pull/346/head
Danno Ferrin 5 years ago
parent b802fa75ce
commit 76f1493b27
5 changed files with 82 additions and 49 deletions
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Show Stats Download Patch File Download Diff File
  1. 5
      ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/results/StructLog.java
  2. 20
      ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/results/tracing/diff/StateDiffGenerator.java
  3. 12
      ethereum/core/src/main/java/org/hyperledger/besu/ethereum/debug/TraceFrame.java
  4. 15
      ethereum/core/src/main/java/org/hyperledger/besu/ethereum/vm/DebugOperationTracer.java
  5. 79
      ethereum/core/src/main/java/org/hyperledger/besu/ethereum/vm/Memory.java

5
ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/results/StructLog.java
Unescape View File

@ -25,7 +25,6 @@ import java.util.TreeMap;
import com.fasterxml.jackson.annotation.JsonGetter;
import com.fasterxml.jackson.annotation.JsonPropertyOrder;
import org.apache.tuweni.bytes.Bytes;
import org.apache.tuweni.bytes.Bytes32;
import org.apache.tuweni.units.bigints.UInt256;
@JsonPropertyOrder({"pc", "op", "gas", "gasCost", "depth", "stack", "memory", "storage"})
@ -48,14 +47,14 @@ public class StructLog {
memory =
traceFrame
.getMemory()
.map(a -> Arrays.stream(a).map(Bytes32::toShortHexString).toArray(String[]::new))
.map(a -> Arrays.stream(a).map(Bytes::toShortHexString).toArray(String[]::new))
.orElse(null);
op = traceFrame.getOpcode();
pc = traceFrame.getPc();
stack =
traceFrame
.getStack()
.map(a -> Arrays.stream(a).map(Bytes32::toShortHexString).toArray(String[]::new))
.map(a -> Arrays.stream(a).map(Bytes::toShortHexString).toArray(String[]::new))
.orElse(null);
storage = traceFrame.getStorage().map(StructLog::formatStorage).orElse(null);
reason = traceFrame.getRevertReason().map(Bytes::toShortHexString).orElse(null);

20
ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/results/tracing/diff/StateDiffGenerator.java
Unescape View File

@ -61,12 +61,19 @@ public class StateDiffGenerator {
final Map<String, DiffNode> storageDiff = new TreeMap<>();
for (final Map.Entry<UInt256, UInt256> entry :
updatedAccount.getUpdatedStorage().entrySet()) {
final UInt256 originalValue = rootAccount.getStorageValue(entry.getKey());
final UInt256 newValue = entry.getValue();
if (!originalValue.equals(newValue)) {
storageDiff.put(
entry.getKey().toHexString(),
new DiffNode(originalValue.toHexString(), newValue.toHexString()));
if (rootAccount == null) {
if (!UInt256.ZERO.equals(newValue)) {
storageDiff.put(
entry.getKey().toHexString(), new DiffNode(null, newValue.toHexString()));
}
} else {
final UInt256 originalValue = rootAccount.getStorageValue(entry.getKey());
if (!originalValue.equals(newValue)) {
storageDiff.put(
entry.getKey().toHexString(),
new DiffNode(originalValue.toHexString(), newValue.toHexString()));
}
}
}
@ -86,6 +93,9 @@ public class StateDiffGenerator {
// Add deleted accounts
for (final Address accountAddress : transactionUpdater.getDeletedAccountAddresses()) {
final Account deletedAccount = previousUpdater.get(accountAddress);
if (deletedAccount == null) {
continue;
}
final AccountDiff accountDiff =
new AccountDiff(
createDiffNode(deletedAccount, null, StateDiffGenerator::balanceAsHex),

12
ethereum/core/src/main/java/org/hyperledger/besu/ethereum/debug/TraceFrame.java
Unescape View File

@ -43,7 +43,7 @@ public class TraceFrame {
private final Bytes inputData;
private final Supplier<Bytes> outputData;
private final Optional<Bytes32[]> stack;
private final Optional<Bytes32[]> memory;
private final Optional<Bytes[]> memory;
private final Optional<Map<UInt256, UInt256>> storage;
private final WorldUpdater worldUpdater;
private final Optional<Bytes> revertReason;
@ -51,7 +51,7 @@ public class TraceFrame {
private final Optional<Code> maybeCode;
private final int stackItemsProduced;
private final Optional<Bytes32[]> stackPostExecution;
private final Optional<Bytes32[]> memoryPostExecution;
private final Optional<Bytes[]> memoryPostExecution;
private Optional<Integer> maybeNextDepth;
private Gas gasRemainingPostExecution;
@ -69,7 +69,7 @@ public class TraceFrame {
final Bytes inputData,
final Supplier<Bytes> outputData,
final Optional<Bytes32[]> stack,
final Optional<Bytes32[]> memory,
final Optional<Bytes[]> memory,
final Optional<Map<UInt256, UInt256>> storage,
final WorldUpdater worldUpdater,
final Optional<Bytes> revertReason,
@ -77,7 +77,7 @@ public class TraceFrame {
final Optional<Code> maybeCode,
final int stackItemsProduced,
final Optional<Bytes32[]> stackPostExecution,
final Optional<Bytes32[]> memoryPostExecution,
final Optional<Bytes[]> memoryPostExecution,
final Optional<Map<UInt256, UInt256>> storagePreExecution,
final boolean virtualOperation,
final Optional<MemoryEntry> maybeUpdatedMemory) {
@ -141,7 +141,7 @@ public class TraceFrame {
return stack;
}
public Optional<Bytes32[]> getMemory() {
public Optional<Bytes[]> getMemory() {
return memory;
}
@ -188,7 +188,7 @@ public class TraceFrame {
return stackPostExecution;
}
public Optional<Bytes32[]> getMemoryPostExecution() {
public Optional<Bytes[]> getMemoryPostExecution() {
return memoryPostExecution;
}

15
ethereum/core/src/main/java/org/hyperledger/besu/ethereum/vm/DebugOperationTracer.java
Unescape View File

@ -37,6 +37,8 @@ import org.apache.tuweni.units.bigints.UInt256;
public class DebugOperationTracer implements OperationTracer {
private static final UInt256 UINT256_32 = UInt256.valueOf(32);
private final TraceOptions options;
private List<TraceFrame> traceFrames = new ArrayList<>();
private TraceFrame lastFrame;
@ -61,11 +63,11 @@ public class DebugOperationTracer implements OperationTracer {
final Bytes inputData = frame.getInputData();
final Supplier<Bytes> outputData = frame::getOutputData;
final Optional<Bytes32[]> stack = captureStack(frame);
final Optional<Bytes32[]> memory = captureMemory(frame);
final Optional<Bytes[]> memory = captureMemory(frame);
final Optional<Map<UInt256, UInt256>> storagePreExecution = captureStorage(frame);
final WorldUpdater worldUpdater = frame.getWorldState();
final Optional<Bytes32[]> stackPostExecution;
final Optional<Bytes32[]> memoryPostExecution;
final Optional<Bytes[]> memoryPostExecution;
try {
executeOperation.execute();
} finally {
@ -119,19 +121,18 @@ public class DebugOperationTracer implements OperationTracer {
.getMutable()
.getUpdatedStorage());
return Optional.of(storageContents);
} catch (ModificationNotAllowedException e) {
} catch (final ModificationNotAllowedException e) {
return Optional.of(new TreeMap<>());
}
}
private Optional<Bytes32[]> captureMemory(final MessageFrame frame) {
private Optional<Bytes[]> captureMemory(final MessageFrame frame) {
if (!options.isMemoryEnabled()) {
return Optional.empty();
}
final Bytes32[] memoryContents = new Bytes32[frame.memoryWordSize().intValue()];
final Bytes[] memoryContents = new Bytes32[frame.memoryWordSize().intValue()];
for (int i = 0; i < memoryContents.length; i++) {
memoryContents[i] =
Bytes32.wrap(frame.readMemory(UInt256.valueOf(i).multiply(32), UInt256.valueOf(32)), 0);
memoryContents[i] = frame.readMemory(UInt256.valueOf(i * 32L), UINT256_32);
}
return Optional.of(memoryContents);
}

79
ethereum/core/src/main/java/org/hyperledger/besu/ethereum/vm/Memory.java
Unescape View File

@ -67,16 +67,16 @@ public class Memory {
* stack is really just growable memory that is grown/accessed from the end and on by fully word,
* but the same page-based design probably make sense.
*/
private final ArrayList<MutableBytes32> data;
private final ArrayList<Bytes32> data;
// Really data.size(), but cached as a UInt256 to avoid recomputing it each time.
private UInt256 activeWords = UInt256.ZERO;
private UInt256 activeWords;
public Memory() {
this(new ArrayList<>());
}
private Memory(final ArrayList<MutableBytes32> data) {
private Memory(final ArrayList<Bytes32> data) {
this.data = data;
this.activeWords = UInt256.valueOf(data.size());
}
@ -141,7 +141,7 @@ public class Memory {
* @param numBytes The minimum number of bytes in memory.
* @return The number of active words that accommodate at least the number of specified bytes.
*/
public UInt256 calculateNewActiveWords(final UInt256 location, final UInt256 numBytes) {
UInt256 calculateNewActiveWords(final UInt256 location, final UInt256 numBytes) {
if (numBytes.isZero()) {
return activeWords;
}
@ -174,7 +174,7 @@ public class Memory {
* @param address The location in memory to start with.
* @param numBytes The number of bytes to get.
*/
public void ensureCapacityForBytes(final long address, final int numBytes) {
void ensureCapacityForBytes(final long address, final int numBytes) {
// Do not increase the memory capacity if no bytes are being written
// regardless of what the address may be.
if (numBytes == 0) {
@ -227,7 +227,7 @@ public class Memory {
*
* @return The current number of active bytes stored in memory.
*/
public long getActiveBytes() {
long getActiveBytes() {
return (long) data.size() * Bytes32.SIZE;
}
@ -236,7 +236,7 @@ public class Memory {
*
* @return The current number of active words stored in memory.
*/
public UInt256 getActiveWords() {
UInt256 getActiveWords() {
return activeWords;
}
@ -272,10 +272,12 @@ public class Memory {
if (startWord == endWord) {
// Bytes within a word, fast-path.
final MutableBytes bytes = data.get(startWord);
final Bytes bytes = data.get(startWord);
return idxInStart == 0 && length == Bytes32.SIZE
? bytes.copy()
: bytes.slice(idxInStart, length).copy();
// ? bytes.copy()
// : bytes.slice(idxInStart, length).copy();
? bytes
: bytes.slice(idxInStart, length);
}
// Spans multiple word, slower path.
@ -381,7 +383,10 @@ public class Memory {
if (startWord == endWord) {
// Bytes within a word, fast-path.
value.copyTo(data.get(startWord), idxInStart);
final MutableBytes mb = data.get(startWord).mutableCopy();
value.copyTo(mb, idxInStart);
data.set(startWord, (Bytes32) mb.copy());
return;
}
@ -389,13 +394,19 @@ public class Memory {
final int bytesInStartWord = Bytes32.SIZE - idxInStart;
int valueIdx = 0;
value.slice(valueIdx, bytesInStartWord).copyTo(data.get(startWord), idxInStart);
final MutableBytes startMutable = data.get(startWord).mutableCopy();
value.slice(valueIdx, bytesInStartWord).copyTo(startMutable, idxInStart);
data.set(startWord, (Bytes32) startMutable.copy());
valueIdx += bytesInStartWord;
for (int i = startWord + 1; i < endWord; i++) {
value.slice(valueIdx, Bytes32.SIZE).copyTo(data.get(i));
final MutableBytes mb = data.get(i).mutableCopy();
value.slice(valueIdx, Bytes32.SIZE).copyTo(mb);
data.set(i, (Bytes32) mb.copy());
valueIdx += Bytes32.SIZE;
}
value.slice(valueIdx).copyTo(data.get(endWord), 0);
final MutableBytes endMutable = data.get(endWord).mutableCopy();
value.slice(valueIdx).copyTo(endMutable, 0);
data.set(endWord, (Bytes32) endMutable.copy());
}
/**
@ -404,7 +415,7 @@ public class Memory {
* @param location The location in memory from which to start clearing the bytes.
* @param numBytes The number of bytes to clear.
*/
public void clearBytes(final UInt256 location, final UInt256 numBytes) {
private void clearBytes(final UInt256 location, final UInt256 numBytes) {
// See getBytes for why we checki length == 0 first, before calling asByteIndex(location).
final int length = asByteLength(numBytes);
if (length == 0) {
@ -419,7 +430,7 @@ public class Memory {
* @param location The location in memory from which to start clearing the bytes.
* @param numBytes The number of bytes to clear.
*/
public void clearBytes(final long location, final int numBytes) {
private void clearBytes(final long location, final int numBytes) {
if (numBytes == 0) {
return;
}
@ -436,11 +447,13 @@ public class Memory {
if (startWord == endWord) {
// Bytes within a word, fast-path.
MutableBytes bytes = data.get(startWord);
if (idxInStart != 0 || numBytes != Bytes32.SIZE) {
bytes = bytes.mutableSlice(idxInStart, numBytes);
}
final MutableBytes storingBytes = data.get(startWord).mutableCopy();
final MutableBytes bytes =
(idxInStart != 0 || numBytes != Bytes32.SIZE)
? storingBytes.mutableSlice(idxInStart, numBytes)
: storingBytes;
bytes.clear();
data.set(startWord, (Bytes32) storingBytes);
return;
}
@ -448,11 +461,15 @@ public class Memory {
final int bytesInStartWord = Bytes32.SIZE - idxInStart;
final int bytesInEndWord = idxInEnd + 1;
data.get(startWord).mutableSlice(idxInStart, bytesInStartWord).clear();
final MutableBytes startMutable = data.get(startWord).mutableCopy();
startMutable.mutableSlice(idxInStart, bytesInStartWord).clear();
data.set(startWord, (Bytes32) startMutable.copy());
for (int i = startWord + 1; i < endWord; i++) {
data.get(i).clear();
data.set(i, Bytes32.ZERO);
}
data.get(endWord).mutableSlice(0, bytesInEndWord).clear();
final MutableBytes endMutable = data.get(endWord).mutableCopy();
endMutable.mutableSlice(0, bytesInEndWord).clear();
data.set(endWord, (Bytes32) endMutable.copy());
}
/**
@ -461,14 +478,16 @@ public class Memory {
* @param location the location of the byte to set.
* @param value the value to set for the byte at {@code location}.
*/
public void setByte(final UInt256 location, final byte value) {
void setByte(final UInt256 location, final byte value) {
final long start = asByteIndex(location);
ensureCapacityForBytes(start, 1);
final int word = wordForByte(start);
final int idxInWord = indexInWord(start);
data.get(word).set(idxInWord, value);
final MutableBytes mb = data.get(word).mutableCopy();
mb.set(idxInWord, value);
data.set(word, (Bytes32) mb.copy());
}
/**
@ -517,14 +536,18 @@ public class Memory {
if (idxInStart == 0) {
// Word-aligned. Fast-path.
bytes.copyTo(data.get(startWord));
data.set(startWord, bytes);
return;
}
// Spans 2 memory word, slower path.
final int sizeInFirstWord = Bytes32.SIZE - idxInStart;
bytes.slice(0, sizeInFirstWord).copyTo(data.get(startWord), idxInStart);
bytes.slice(sizeInFirstWord).copyTo(data.get(startWord + 1), 0);
final MutableBytes firstWord = data.get(startWord).mutableCopy();
bytes.slice(0, sizeInFirstWord).copyTo(firstWord, idxInStart);
final MutableBytes secondWord = data.get(startWord + 1).mutableCopy();
bytes.slice(sizeInFirstWord).copyTo(secondWord, 0);
data.set(startWord, (Bytes32) firstWord);
data.set(startWord + 1, (Bytes32) secondWord);
}
@Override

Write Preview
Loading…
Cancel
Save