Fix operator generator command excluding alloc field when generating config (#7684)

Signed-off-by: Jason Frame <jason.frame@consensys.net>

besu/src/test/java/org/hyperledger/besu/cli/subcommands/operator/OperatorSubCommandTest.java

@@ -282,8 +282,10 @@ public class OperatorSubCommandTest extends CommandTestAbstract {
         false,
         singletonList("key.pub"),
         Optional.empty(),
-        Optional.of(
+        List.of(
-            "0xf853a00000000000000000000000000000000000000000000000000000000000000000ea94d5feb0fc5a54a89f97aeb34c3df15397c19f6dd294d6a9a4c886eb008ac307abdc1f38745c1dd13a88808400000000c0"));
+            new Field(
+                "extraData",
+                "0xf853a00000000000000000000000000000000000000000000000000000000000000000ea94d5feb0fc5a54a89f97aeb34c3df15397c19f6dd294d6a9a4c886eb008ac307abdc1f38745c1dd13a88808400000000c0")));
   }
 
   @Test
@@ -296,8 +298,49 @@ public class OperatorSubCommandTest extends CommandTestAbstract {
         false,
         singletonList("key.pub"),
         Optional.empty(),
-        Optional.of(
+        List.of(
-            "0xf84fa00000000000000000000000000000000000000000000000000000000000000000ea94d5feb0fc5a54a89f97aeb34c3df15397c19f6dd294d6a9a4c886eb008ac307abdc1f38745c1dd13a88c080c0"));
+            new Field(
+                "extraData",
+                "0xf84fa00000000000000000000000000000000000000000000000000000000000000000ea94d5feb0fc5a54a89f97aeb34c3df15397c19f6dd294d6a9a4c886eb008ac307abdc1f38745c1dd13a88c080c0")));
+  }
+
+  @Test
+  public void generatedGenesisFileShouldContainAllOriginalFieldsExcludingExtraData()
+      throws IOException {
+    final JsonObject alloc =
+        new JsonObject(
+            """
+    {
+      "24defc2d149861d3d245749b81fe0e6b28e04f31": {
+        "balance": "0x446c3b15f9926687d2c40534fdb564000000000000"
+      },
+      "2a813d7db3de19b07f92268b6d4125ed295cbe00": {
+        "balance": "0x446c3b15f9926687d2c40534fdb542000000000000"
+      }
+    }""");
+    final List<Field> fields =
+        List.of(
+            new Field("nonce", "0x0"),
+            new Field("timestamp", "0x5b3c3d18"),
+            new Field("gasUsed", "0x0"),
+            new Field(
+                "parentHash", "0x0000000000000000000000000000000000000000000000000000000000000000"),
+            new Field("gasLimit", "0x47b760"),
+            new Field("difficulty", "0x1"),
+            new Field(
+                "mixHash", "0x63746963616c2062797a616e74696e65206661756c7420746f6c6572616e6365"),
+            new Field("coinbase", "0x0000000000000000000000000000000000000000"),
+            new Field("alloc", alloc.getMap().toString()));
+
+    runCmdAndCheckOutput(
+        cmd(),
+        "/operator/config_generate_keys.json",
+        tmpOutputDirectoryPath,
+        "genesis.json",
+        false,
+        singletonList("key.pub"),
+        Optional.empty(),
+        fields);
   }
 
   private void runCmdAndCheckOutput(
@@ -316,7 +359,7 @@ public class OperatorSubCommandTest extends CommandTestAbstract {
         generate,
         expectedKeyFiles,
         Optional.empty(),
-        Optional.empty());
+        List.of());
   }
 
   private void runCmdAndCheckOutput(
@@ -336,9 +379,11 @@ public class OperatorSubCommandTest extends CommandTestAbstract {
         generate,
         expectedKeyFiles,
         signatureAlgorithm,
-        Optional.empty());
+        List.of());
   }
 
+  private record Field(String key, String value) {}
+
   private void runCmdAndCheckOutput(
       final Cmd cmd,
       final String configFile,
@@ -347,7 +392,7 @@ public class OperatorSubCommandTest extends CommandTestAbstract {
       final boolean generate,
       final Collection<String> expectedKeyFiles,
       final Optional<SignatureAlgorithm> signatureAlgorithm,
-      final Optional<String> expectedExtraData)
+      final List<Field> expectedFields)
       throws IOException {
     final URL configFilePath = this.getClass().getResource(configFile);
     parseCommand(
@@ -368,8 +413,9 @@ public class OperatorSubCommandTest extends CommandTestAbstract {
     final String genesisString = contentOf(outputGenesisFile, UTF_8);
     final JsonObject genesisContent = new JsonObject(genesisString);
     assertThat(genesisContent.containsKey("extraData")).isTrue();
-    expectedExtraData.ifPresent(
+
-        extraData -> assertThat(genesisContent.getString("extraData")).isEqualTo(extraData));
+    expectedFields.forEach(
+        field -> assertThat(genesisContent.getString(field.key)).isEqualTo(field.value));
 
     final Path expectedKeysPath = outputDirectoryPath.resolve("keys");
     final File keysDirectory = new File(expectedKeysPath.toUri());

besu/src/test/resources/operator/config_generate_keys.json

@@ -4,7 +4,9 @@
       "chainId": 2017,
       "eip150Block": 0,
       "ibft2": {
-
+        "blockperiodseconds": 2,
+        "epochlength": 30000,
+        "requesttimeoutseconds": 10
       }
     },
     "nonce": "0x0",
@@ -16,11 +18,6 @@
     "difficulty": "0x1",
     "mixHash": "0x63746963616c2062797a616e74696e65206661756c7420746f6c6572616e6365",
     "coinbase": "0x0000000000000000000000000000000000000000",
-    "ibft2": {
-      "blockperiodseconds": 2,
-      "epochlength": 30000,
-      "requesttimeoutseconds": 10
-    },
     "alloc": {
       "24defc2d149861d3d245749b81fe0e6b28e04f31": {
         "balance": "0x446c3b15f9926687d2c40534fdb564000000000000"

besu/src/test/resources/operator/config_generate_keys_ec_invalid.json

@@ -5,7 +5,9 @@
       "eip150Block": 0,
       "ecCurve": "abcd",
       "ibft2": {
-
+        "blockperiodseconds": 2,
+        "epochlength": 30000,
+        "requesttimeoutseconds": 10
       }
     },
     "nonce": "0x0",
@@ -17,11 +19,6 @@
     "difficulty": "0x1",
     "mixHash": "0x63746963616c2062797a616e74696e65206661756c7420746f6c6572616e6365",
     "coinbase": "0x0000000000000000000000000000000000000000",
-    "ibft2": {
-      "blockperiodseconds": 2,
-      "epochlength": 30000,
-      "requesttimeoutseconds": 10
-    },
     "alloc": {
       "24defc2d149861d3d245749b81fe0e6b28e04f31": {
         "balance": "0x446c3b15f9926687d2c40534fdb564000000000000"

besu/src/test/resources/operator/config_generate_keys_secp256r1.json

@@ -5,7 +5,9 @@
       "eip150Block": 0,
       "ecCurve": "secp256r1",
       "ibft2": {
-
+        "blockperiodseconds": 2,
+        "epochlength": 30000,
+        "requesttimeoutseconds": 10
       }
     },
     "nonce": "0x0",
@@ -17,11 +19,6 @@
     "difficulty": "0x1",
     "mixHash": "0x63746963616c2062797a616e74696e65206661756c7420746f6c6572616e6365",
     "coinbase": "0x0000000000000000000000000000000000000000",
-    "ibft2": {
-      "blockperiodseconds": 2,
-      "epochlength": 30000,
-      "requesttimeoutseconds": 10
-    },
     "alloc": {
       "24defc2d149861d3d245749b81fe0e6b28e04f31": {
         "balance": "0x446c3b15f9926687d2c40534fdb564000000000000"

besu/src/test/resources/operator/config_import_keys.json

@@ -4,7 +4,9 @@
       "chainId": 2017,
       "petersburgBlock": 0,
       "ibft2": {
-
+        "blockperiodseconds": 2,
+        "epochlength": 30000,
+        "requesttimeoutseconds": 10
       }
     },
     "nonce": "0x0",
@@ -16,11 +18,6 @@
     "difficulty": "0x1",
     "mixHash": "0x63746963616c2062797a616e74696e65206661756c7420746f6c6572616e6365",
     "coinbase": "0x0000000000000000000000000000000000000000",
-    "ibft2": {
-      "blockperiodseconds": 2,
-      "epochlength": 30000,
-      "requesttimeoutseconds": 10
-    },
     "alloc": {
       "24defc2d149861d3d245749b81fe0e6b28e04f31": {
         "balance": "0x446c3b15f9926687d2c40534fdb564000000000000"

besu/src/test/resources/operator/config_import_keys_invalid_keys.json

@@ -4,7 +4,9 @@
       "chainId": 2017,
       "petersburgBlock": 0,
       "ibft2": {
-
+        "blockperiodseconds": 2,
+        "epochlength": 30000,
+        "requesttimeoutseconds": 10
       }
     },
     "nonce": "0x0",
@@ -16,11 +18,6 @@
     "difficulty": "0x1",
     "mixHash": "0x63746963616c2062797a616e74696e65206661756c7420746f6c6572616e6365",
     "coinbase": "0x0000000000000000000000000000000000000000",
-    "ibft2": {
-      "blockperiodseconds": 2,
-      "epochlength": 30000,
-      "requesttimeoutseconds": 10
-    },
     "alloc": {
       "24defc2d149861d3d245749b81fe0e6b28e04f31": {
         "balance": "0x446c3b15f9926687d2c40534fdb564000000000000"

besu/src/test/resources/operator/config_import_keys_qbft.json

@@ -4,7 +4,9 @@
       "chainId": 2017,
       "petersburgBlock": 0,
       "qbft": {
-
+        "blockperiodseconds": 2,
+        "epochlength": 30000,
+        "requesttimeoutseconds": 10
       }
     },
     "nonce": "0x0",
@@ -16,11 +18,6 @@
     "difficulty": "0x1",
     "mixHash": "0x63746963616c2062797a616e74696e65206661756c7420746f6c6572616e6365",
     "coinbase": "0x0000000000000000000000000000000000000000",
-    "ibft2": {
-      "blockperiodseconds": 2,
-      "epochlength": 30000,
-      "requesttimeoutseconds": 10
-    },
     "alloc": {
       "24defc2d149861d3d245749b81fe0e6b28e04f31": {
         "balance": "0x446c3b15f9926687d2c40534fdb564000000000000"

besu/src/test/resources/operator/config_import_keys_secp256r1.json

@@ -5,7 +5,9 @@
       "petersburgBlock": 0,
       "ecCurve": "secp256r1",
       "ibft2": {
-
+        "blockperiodseconds": 2,
+        "epochlength": 30000,
+        "requesttimeoutseconds": 10
       }
     },
     "nonce": "0x0",
@@ -17,11 +19,6 @@
     "difficulty": "0x1",
     "mixHash": "0x63746963616c2062797a616e74696e65206661756c7420746f6c6572616e6365",
     "coinbase": "0x0000000000000000000000000000000000000000",
-    "ibft2": {
-      "blockperiodseconds": 2,
-      "epochlength": 30000,
-      "requesttimeoutseconds": 10
-    },
     "alloc": {
       "24defc2d149861d3d245749b81fe0e6b28e04f31": {
         "balance": "0x446c3b15f9926687d2c40534fdb564000000000000"

besu/src/test/resources/operator/config_import_keys_secp256r1_invalid_keys.json

@@ -5,7 +5,9 @@
       "petersburgBlock": 0,
       "ecCurve": "secp256r1",
       "ibft2": {
-
+        "blockperiodseconds": 2,
+        "epochlength": 30000,
+        "requesttimeoutseconds": 10
       }
     },
     "nonce": "0x0",
@@ -17,11 +19,6 @@
     "difficulty": "0x1",
     "mixHash": "0x63746963616c2062797a616e74696e65206661756c7420746f6c6572616e6365",
     "coinbase": "0x0000000000000000000000000000000000000000",
-    "ibft2": {
-      "blockperiodseconds": 2,
-      "epochlength": 30000,
-      "requesttimeoutseconds": 10
-    },
     "alloc": {
       "24defc2d149861d3d245749b81fe0e6b28e04f31": {
         "balance": "0x446c3b15f9926687d2c40534fdb564000000000000"

besu/src/test/resources/operator/config_no_config_section.json

@@ -9,11 +9,6 @@
     "difficulty": "0x1",
     "mixHash": "0x63746963616c2062797a616e74696e65206661756c7420746f6c6572616e6365",
     "coinbase": "0x0000000000000000000000000000000000000000",
-    "ibft2": {
-      "blockperiodseconds": 2,
-      "epochlength": 30000,
-      "requesttimeoutseconds": 10
-    },
     "alloc": {
       "24defc2d149861d3d245749b81fe0e6b28e04f31": {
         "balance": "0x446c3b15f9926687d2c40534fdb564000000000000"

config/src/main/java/org/hyperledger/besu/config/GenesisReader.java

@@ -53,12 +53,12 @@ interface GenesisReader {
     private final ObjectNode rootWithoutAllocations;
 
     public FromObjectNode(final ObjectNode root) {
-      final var removedAllocations = root.remove(ALLOCATION_FIELD);
       this.allocations =
-          removedAllocations != null
+          root.get(ALLOCATION_FIELD) != null
-              ? (ObjectNode) removedAllocations
+              ? (ObjectNode) root.get(ALLOCATION_FIELD)
               : JsonUtil.createEmptyObjectNode();
-      this.rootWithoutAllocations = normalizeKeys(root);
+      this.rootWithoutAllocations =
+          normalizeKeys(root, field -> !field.getKey().equals(ALLOCATION_FIELD));
     }
 
     @Override

config/src/main/java/org/hyperledger/besu/config/JsonUtil.java

@@ -25,6 +25,7 @@ import java.util.OptionalInt;
 import java.util.OptionalLong;
 import java.util.Set;
 import java.util.function.Function;
+import java.util.function.Predicate;
 
 import com.fasterxml.jackson.core.JsonFactory;
 import com.fasterxml.jackson.core.JsonParser;
@@ -37,6 +38,7 @@ import com.fasterxml.jackson.databind.ObjectMapper;
 import com.fasterxml.jackson.databind.node.ArrayNode;
 import com.fasterxml.jackson.databind.node.JsonNodeType;
 import com.fasterxml.jackson.databind.node.ObjectNode;
+import com.google.common.base.Predicates;
 import org.apache.tuweni.bytes.Bytes;
 
 /** The Json util class. */
@@ -59,11 +61,29 @@ public class JsonUtil {
    * @return a copy of the json object with all keys in lower case.
    */
   public static ObjectNode normalizeKeys(final ObjectNode objectNode) {
+    return normalizeKeys(objectNode, Predicates.alwaysTrue());
+  }
+
+  /**
+   * Converts all the object keys (but none of the string values) to lowercase for easier lookup.
+   * This is useful in cases such as the 'genesis.json' file where all keys are assumed to be case
+   * insensitive.
+   *
+   * @param objectNode The ObjectNode to be normalized
+   * @param fieldPredicate The predicate to filter the fields to normalize
+   * @return a copy of the json object with all keys in lower case.
+   */
+  public static ObjectNode normalizeKeys(
+      final ObjectNode objectNode, final Predicate<Map.Entry<String, JsonNode>> fieldPredicate) {
     final ObjectNode normalized = JsonUtil.createEmptyObjectNode();
     objectNode
         .fields()
         .forEachRemaining(
             entry -> {
+              if (!fieldPredicate.test(entry)) {
+                return;
+              }
+
               final String key = entry.getKey();
               final JsonNode value = entry.getValue();
               final String normalizedKey = normalizeKey(key);

config/src/test/java/org/hyperledger/besu/config/GenesisReaderTest.java

@@ -53,6 +53,22 @@ public class GenesisReaderTest {
         .containsExactly(new GenesisAccount(Address.BLS12_G2MUL, 0, Wei.ONE, null, Map.of(), null));
   }
 
+  @Test
+  public void readGenesisFromObjectDoesNotModifyObjectNodeArg() {
+    final var configNode = mapper.createObjectNode();
+    configNode.put("londonBlock", 1);
+    final var allocNode = mapper.createObjectNode();
+    allocNode.put(Address.BLS12_G2MUL.toUnprefixedHexString(), generateAllocation(Wei.ONE));
+    final var rootNode = mapper.createObjectNode();
+    rootNode.put("chainId", 12);
+    rootNode.put(CONFIG_FIELD, configNode);
+    rootNode.put(ALLOCATION_FIELD, allocNode);
+    var rootNodeCopy = rootNode.deepCopy();
+    new GenesisReader.FromObjectNode(rootNode);
+
+    assertThat(rootNode).isEqualTo(rootNodeCopy);
+  }
+
   @Test
   public void readGenesisFromURL(@TempDir final Path folder) throws IOException {
     final String jsonStr =

config/src/test/java/org/hyperledger/besu/config/JsonUtilTest.java

@@ -147,6 +147,36 @@ public class JsonUtilTest {
     assertThat(normalizedObj).isEqualTo(expectedObj);
   }
 
+  @Test
+  public void normalizeKeys_predicate() {
+    final ObjectNode originalObj =
+        mapper
+            .createObjectNode()
+            .put("Ant", "Tiny")
+            .put("Ape", "Smart")
+            .put("Armadillo", "Armored")
+            .put("Cat", "Meow")
+            .put("Bat", "Flying")
+            .put("Cow", "Moo")
+            .put("Crocodile", "Snap")
+            .put("Bear", "Strong")
+            .put("Cheetah", "Fast")
+            .put("Beaver", "Builder");
+
+    final ObjectNode expectedObj =
+        mapper
+            .createObjectNode()
+            .put("cat", "Meow")
+            .put("cow", "Moo")
+            .put("cheetah", "Fast")
+            .put("crocodile", "Snap");
+
+    final ObjectNode normalizedObj =
+        JsonUtil.normalizeKeys(originalObj, s -> s.getKey().startsWith("C"));
+
+    assertThat(normalizedObj).isEqualTo(expectedObj);
+  }
+
   @Test
   public void getLong_nonExistentKey() {
     final ObjectNode node = mapper.createObjectNode();