feat: Enhance --profile to load external profiles (#7292)

* feat: --profile can load external profiles
* fix external profile name method
* fix ProfilesCompletionCandidate
* test: Add unit tests
* changelog: Update changelog
* test: Fix TomlConfigurationDefaultProviderTest
* test: Fix BesuCommandTest

---------

Signed-off-by: Usman Saleem <usman@usmans.info>

CHANGELOG.md

@@ -5,6 +5,7 @@
 ### Breaking Changes
 
 ### Additions and Improvements
+- Add support to load external profiles using `--profile` [#7265](https://github.com/hyperledger/besu/issues/7265)
 
 ### Bug fixes
 

besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java

@@ -39,7 +39,7 @@ import org.hyperledger.besu.chainimport.JsonBlockImporter;
 import org.hyperledger.besu.chainimport.RlpBlockImporter;
 import org.hyperledger.besu.cli.config.EthNetworkConfig;
 import org.hyperledger.besu.cli.config.NetworkName;
-import org.hyperledger.besu.cli.config.ProfileName;
+import org.hyperledger.besu.cli.config.ProfilesCompletionCandidates;
 import org.hyperledger.besu.cli.converter.MetricCategoryConverter;
 import org.hyperledger.besu.cli.converter.PercentageConverter;
 import org.hyperledger.besu.cli.converter.SubnetInfoConverter;
@@ -565,9 +565,10 @@ public class BesuCommand implements DefaultCommandValues, Runnable {
   @Option(
       names = {PROFILE_OPTION_NAME},
       paramLabel = PROFILE_FORMAT_HELP,
+      completionCandidates = ProfilesCompletionCandidates.class,
       description =
           "Overwrite default settings. Possible values are ${COMPLETION-CANDIDATES}. (default: none)")
-  private final ProfileName profile = null;
+  private String profile = null; // don't set it as final due to picocli completion candidates
 
   @Option(
       names = {"--nat-method"},
@@ -2773,7 +2774,7 @@ public class BesuCommand implements DefaultCommandValues, Runnable {
     }
 
     if (profile != null) {
-      builder.setProfile(profile.toString());
+      builder.setProfile(profile);
     }
 
     builder.setHasCustomGenesis(genesisFile != null);

besu/src/main/java/org/hyperledger/besu/cli/config/InternalProfileName.java

@@ -14,10 +14,18 @@
  */
 package org.hyperledger.besu.cli.config;
 
+import java.util.Arrays;
+import java.util.Optional;
+import java.util.Set;
+import java.util.stream.Collectors;
+
 import org.apache.commons.lang3.StringUtils;
 
-/** Enum for profile names. Each profile corresponds to a configuration file. */
+/**
-public enum ProfileName {
+ * Enum for profile names which are bundled. Each profile corresponds to a bundled configuration
+ * file.
+ */
+public enum InternalProfileName {
   /** The 'STAKER' profile */
   STAKER("profiles/staker.toml"),
   /** The 'MINIMALIST_STAKER' profile */
@@ -31,12 +39,36 @@ public enum ProfileName {
 
   private final String configFile;
 
+  /**
+   * Returns the InternalProfileName that matches the given name, ignoring case.
+   *
+   * @param name The profile name
+   * @return Optional InternalProfileName if found, otherwise empty
+   */
+  public static Optional<InternalProfileName> valueOfIgnoreCase(final String name) {
+    return Arrays.stream(values())
+        .filter(profile -> profile.name().equalsIgnoreCase(name))
+        .findFirst();
+  }
+
+  /**
+   * Returns the set of internal profile names as lowercase.
+   *
+   * @return Set of internal profile names
+   */
+  public static Set<String> getInternalProfileNames() {
+    return Arrays.stream(InternalProfileName.values())
+        .map(InternalProfileName::name)
+        .map(String::toLowerCase)
+        .collect(Collectors.toSet());
+  }
+
   /**
    * Constructs a new ProfileName.
    *
    * @param configFile the configuration file corresponding to the profile
    */
-  ProfileName(final String configFile) {
+  InternalProfileName(final String configFile) {
     this.configFile = configFile;
   }
 

besu/src/main/java/org/hyperledger/besu/cli/config/ProfilesCompletionCandidates.java

@@ -0,0 +1,37 @@
+/*
+ * Copyright contributors to Hyperledger Besu.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+ * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
+package org.hyperledger.besu.cli.config;
+
+import org.hyperledger.besu.cli.util.ProfileFinder;
+
+import java.util.Iterator;
+import java.util.Set;
+import java.util.TreeSet;
+
+/** Provides a list of profile names that can be used for command line completion. */
+public class ProfilesCompletionCandidates implements Iterable<String> {
+  /**
+   * Create a new instance of ProfilesCompletionCandidates. This constructor is required for
+   * Picocli.
+   */
+  public ProfilesCompletionCandidates() {}
+
+  @Override
+  public Iterator<String> iterator() {
+    final Set<String> profileNames = new TreeSet<>(InternalProfileName.getInternalProfileNames());
+    profileNames.addAll(ProfileFinder.getExternalProfileNames());
+    return profileNames.iterator();
+  }
+}

besu/src/main/java/org/hyperledger/besu/cli/subcommands/ValidateConfigSubCommand.java

@@ -70,7 +70,7 @@ public class ValidateConfigSubCommand implements Runnable {
     checkNotNull(parentCommand);
     try {
       TomlConfigurationDefaultProvider.fromFile(commandLine, dataPath.toFile())
-          .loadConfigurationFromFile();
+          .loadConfigurationIfNotLoaded();
     } catch (Exception e) {
       this.out.println(e);
       return;

besu/src/main/java/org/hyperledger/besu/cli/util/ProfileFinder.java

@@ -16,11 +16,19 @@ package org.hyperledger.besu.cli.util;
 
 import static org.hyperledger.besu.cli.DefaultCommandValues.PROFILE_OPTION_NAME;
 
-import org.hyperledger.besu.cli.config.ProfileName;
+import org.hyperledger.besu.cli.config.InternalProfileName;
 
+import java.io.IOException;
 import java.io.InputStream;
+import java.io.UncheckedIOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
 import java.util.Map;
 import java.util.Optional;
+import java.util.Set;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
 
 import picocli.CommandLine;
 
@@ -50,30 +58,94 @@ public class ProfileFinder extends AbstractConfigurationFinder<InputStream> {
   @Override
   public Optional<InputStream> getFromOption(
       final CommandLine.ParseResult parseResult, final CommandLine commandLine) {
+    final String profileName;
     try {
-      return getProfile(parseResult.matchedOption(PROFILE_OPTION_NAME).getter().get(), commandLine);
+      profileName = parseResult.matchedOption(PROFILE_OPTION_NAME).getter().get();
-    } catch (Exception e) {
+    } catch (final Exception e) {
-      throw new RuntimeException(e);
+      throw new CommandLine.ParameterException(
+          commandLine, "Unexpected error in obtaining value of --profile", e);
     }
+    return getProfile(profileName, commandLine);
   }
 
   @Override
   public Optional<InputStream> getFromEnvironment(
       final Map<String, String> environment, final CommandLine commandLine) {
-    return getProfile(ProfileName.valueOf(environment.get(PROFILE_ENV_NAME)), commandLine);
+    return getProfile(environment.get(PROFILE_ENV_NAME), commandLine);
   }
 
   private static Optional<InputStream> getProfile(
-      final ProfileName profileName, final CommandLine commandLine) {
+      final String profileName, final CommandLine commandLine) {
-    return Optional.of(getTomlFile(commandLine, profileName.getConfigFile()));
+    final Optional<String> internalProfileConfigPath =
+        InternalProfileName.valueOfIgnoreCase(profileName).map(InternalProfileName::getConfigFile);
+    if (internalProfileConfigPath.isPresent()) {
+      return Optional.of(getTomlFileFromClasspath(internalProfileConfigPath.get()));
+    } else {
+      final Path externalProfileFile = defaultProfilesDir().resolve(profileName + ".toml");
+      if (Files.exists(externalProfileFile)) {
+        try {
+          return Optional.of(Files.newInputStream(externalProfileFile));
+        } catch (IOException e) {
+          throw new CommandLine.ParameterException(
+              commandLine, "Error reading external profile: " + profileName);
+        }
+      } else {
+        throw new CommandLine.ParameterException(
+            commandLine, "Unable to load external profile: " + profileName);
+      }
+    }
   }
 
-  private static InputStream getTomlFile(final CommandLine commandLine, final String file) {
+  private static InputStream getTomlFileFromClasspath(final String profileConfigFile) {
-    InputStream resourceUrl = ProfileFinder.class.getClassLoader().getResourceAsStream(file);
+    InputStream resourceUrl =
+        ProfileFinder.class.getClassLoader().getResourceAsStream(profileConfigFile);
+    // this is not meant to happen, because for each InternalProfileName there is a corresponding
+    // TOML file in resources
     if (resourceUrl == null) {
-      throw new CommandLine.ParameterException(
+      throw new IllegalStateException(
-          commandLine, String.format("TOML file %s not found", file));
+          String.format("Internal Profile TOML %s not found", profileConfigFile));
     }
     return resourceUrl;
   }
+
+  /**
+   * Returns the external profile names which are file names without extension in the default
+   * profiles directory.
+   *
+   * @return Set of external profile names
+   */
+  public static Set<String> getExternalProfileNames() {
+    final Path profilesDir = defaultProfilesDir();
+    if (!Files.exists(profilesDir)) {
+      return Set.of();
+    }
+
+    try (Stream<Path> pathStream = Files.list(profilesDir)) {
+      return pathStream
+          .filter(Files::isRegularFile)
+          .filter(path -> path.toString().endsWith(".toml"))
+          .map(
+              path ->
+                  path.getFileName()
+                      .toString()
+                      .substring(0, path.getFileName().toString().length() - 5))
+          .collect(Collectors.toSet());
+    } catch (IOException e) {
+      throw new UncheckedIOException(e);
+    }
+  }
+
+  /**
+   * Return default profiles directory location
+   *
+   * @return Path to default profiles directory
+   */
+  private static Path defaultProfilesDir() {
+    final String profilesDir = System.getProperty("besu.profiles.dir");
+    if (profilesDir == null) {
+      return Paths.get(System.getProperty("besu.home", "."), "profiles");
+    } else {
+      return Paths.get(profilesDir);
+    }
+  }
 }

besu/src/main/java/org/hyperledger/besu/cli/util/TomlConfigurationDefaultProvider.java

@@ -96,7 +96,7 @@ public class TomlConfigurationDefaultProvider implements IDefaultValueProvider {
 
   @Override
   public String defaultValue(final ArgSpec argSpec) {
-    loadConfigurationFromFile();
+    loadConfigurationIfNotLoaded();
 
     // only options can be used in config because a name is needed for the key
     // so we skip default for positional params
@@ -227,10 +227,10 @@ public class TomlConfigurationDefaultProvider implements IDefaultValueProvider {
   }
 
   private void checkConfigurationValidity() {
-    if (result == null || result.isEmpty())
+    if (result == null || result.isEmpty()) {
       throw new ParameterException(
-          commandLine,
+          commandLine, "Unable to read from empty TOML configuration file.");
-          String.format("Unable to read TOML configuration file %s", configurationInputStream));
+    }
 
     if (!isUnknownOptionsChecked && !commandLine.isUnmatchedArgumentsAllowed()) {
       checkUnknownOptions(result);
@@ -239,8 +239,7 @@ public class TomlConfigurationDefaultProvider implements IDefaultValueProvider {
   }
 
   /** Load configuration from file. */
-  public void loadConfigurationFromFile() {
+  public void loadConfigurationIfNotLoaded() {
-
     if (result == null) {
       try {
         final TomlParseResult result = Toml.parse(configurationInputStream);
@@ -289,12 +288,12 @@ public class TomlConfigurationDefaultProvider implements IDefaultValueProvider {
             .collect(Collectors.toSet());
 
     if (!unknownOptionsList.isEmpty()) {
-      final String options = unknownOptionsList.size() > 1 ? "options" : "option";
+      final String csvUnknownOptions = String.join(", ", unknownOptionsList);
-      final String csvUnknownOptions =
-          unknownOptionsList.stream().collect(Collectors.joining(", "));
       throw new ParameterException(
           commandLine,
-          String.format("Unknown %s in TOML configuration file: %s", options, csvUnknownOptions));
+          String.format(
+              "Unknown option%s in TOML configuration file: %s",
+              unknownOptionsList.size() > 1 ? "s" : "", csvUnknownOptions));
     }
   }
 }

besu/src/test/java/org/hyperledger/besu/cli/BesuCommandTest.java

@@ -259,7 +259,7 @@ public class BesuCommandTest extends CommandTestAbstract {
     final Path tempConfigFilePath = createTempFile("an-invalid-file-name-without-extension", "");
     parseCommand("--config-file", tempConfigFilePath.toString());
 
-    final String expectedOutputStart = "Unable to read TOML configuration file";
+    final String expectedOutputStart = "Unable to read from empty TOML configuration file.";
     assertThat(commandErrorOutput.toString(UTF_8)).startsWith(expectedOutputStart);
     assertThat(commandOutput.toString(UTF_8)).isEmpty();
   }

besu/src/test/java/org/hyperledger/besu/cli/ConfigurationOverviewBuilderTest.java

@@ -20,7 +20,7 @@ import static org.hyperledger.besu.ethereum.eth.transactions.TransactionPoolConf
 import static org.hyperledger.besu.ethereum.eth.transactions.TransactionPoolConfiguration.Implementation.SEQUENCED;
 import static org.mockito.Mockito.mock;
 
-import org.hyperledger.besu.cli.config.ProfileName;
+import org.hyperledger.besu.cli.config.InternalProfileName;
 import org.hyperledger.besu.evm.internal.EvmConfiguration;
 
 import java.math.BigInteger;
@@ -213,7 +213,7 @@ class ConfigurationOverviewBuilderTest {
 
   @Test
   void setProfile() {
-    builder.setProfile(ProfileName.DEV.name());
+    builder.setProfile(InternalProfileName.DEV.name());
     final String profileSelected = builder.build();
     assertThat(profileSelected).contains("Profile: DEV");
   }

besu/src/test/java/org/hyperledger/besu/cli/ProfilesTest.java

@@ -17,21 +17,104 @@ package org.hyperledger.besu.cli;
 import static java.nio.charset.StandardCharsets.UTF_8;
 import static org.assertj.core.api.Assertions.assertThat;
 
-import org.hyperledger.besu.cli.config.ProfileName;
+import org.hyperledger.besu.cli.config.InternalProfileName;
+import org.hyperledger.besu.cli.util.ProfileFinder;
 
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.Set;
+import java.util.TreeSet;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+import org.junit.jupiter.api.AfterAll;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.DisplayName;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
 import org.junit.jupiter.params.ParameterizedTest;
-import org.junit.jupiter.params.provider.EnumSource;
+import org.junit.jupiter.params.provider.Arguments;
+import org.junit.jupiter.params.provider.MethodSource;
 
 public class ProfilesTest extends CommandTestAbstract {
+  @TempDir private static Path tempProfilesDir;
+  private static String originalProfilesDirProperty;
 
-  /** Test if besu will validate the combination of options within the given profile. */
+  @BeforeAll
-  @ParameterizedTest
+  public static void copyExternalProfiles() throws IOException {
-  @EnumSource(ProfileName.class)
+    for (String internalProfileName : InternalProfileName.getInternalProfileNames()) {
-  public void testProfileWithNoOverrides_doesNotError(final ProfileName profileName) {
+      final Path profilePath = tempProfilesDir.resolve(internalProfileName + "_external.toml");
+
+      String profileConfigFile =
+          InternalProfileName.valueOfIgnoreCase(internalProfileName).get().getConfigFile();
+      try (InputStream resourceUrl =
+          ProfileFinder.class.getClassLoader().getResourceAsStream(profileConfigFile)) {
+        if (resourceUrl != null) {
+          Files.copy(resourceUrl, profilePath);
+        }
+      }
+    }
+
+    // add an empty external profile
+    Files.createFile(tempProfilesDir.resolve("empty_external.toml"));
+  }
+
+  @BeforeAll
+  public static void setupSystemProperty() {
+    originalProfilesDirProperty = System.getProperty("besu.profiles.dir");
+    // sets the system property for the test
+    System.setProperty("besu.profiles.dir", tempProfilesDir.toString());
+  }
+
+  static Stream<Arguments> profileNameProvider() {
+    final Set<String> profileNames = new TreeSet<>(InternalProfileName.getInternalProfileNames());
+    final Set<String> externalProfileNames =
+        InternalProfileName.getInternalProfileNames().stream()
+            .map(name -> name + "_external")
+            .collect(Collectors.toSet());
+    profileNames.addAll(externalProfileNames);
+    return profileNames.stream().map(Arguments::of);
+  }
 
-    parseCommand("--profile", profileName.name());
+  /** Test if besu will validate the combination of options within the given profile. */
+  @ParameterizedTest(name = "{index} - Profile Name override: {0}")
+  @DisplayName("Valid Profile with overrides does not error")
+  @MethodSource("profileNameProvider")
+  public void testProfileWithNoOverrides_doesNotError(final String profileName) {
+    parseCommand("--profile", profileName);
 
     assertThat(commandOutput.toString(UTF_8)).isEmpty();
     assertThat(commandErrorOutput.toString(UTF_8)).isEmpty();
   }
+
+  @Test
+  @DisplayName("Empty external profile file results in error")
+  public void emptyProfileFile_ShouldResultInError() {
+    parseCommand("--profile", "empty_external");
+
+    assertThat(commandOutput.toString(UTF_8)).isEmpty();
+    assertThat(commandErrorOutput.toString(UTF_8))
+        .contains("Unable to read from empty TOML configuration file.");
+  }
+
+  @Test
+  @DisplayName("Non Existing profile results in error")
+  public void nonExistentProfileFile_ShouldResultInError() {
+    parseCommand("--profile", "non_existent_profile");
+
+    assertThat(commandOutput.toString(UTF_8)).isEmpty();
+    assertThat(commandErrorOutput.toString(UTF_8))
+        .contains("Unable to load external profile: non_existent_profile");
+  }
+
+  @AfterAll
+  public static void clearSystemProperty() {
+    if (originalProfilesDirProperty != null) {
+      System.setProperty("besu.profiles.dir", originalProfilesDirProperty);
+    } else {
+      System.clearProperty("besu.profiles.dir");
+    }
+  }
 }

besu/src/test/java/org/hyperledger/besu/cli/TomlConfigurationDefaultProviderTest.java

@@ -241,7 +241,7 @@ public class TomlConfigurationDefaultProviderTest {
                 providerUnderTest.defaultValue(
                     OptionSpec.builder("an-option").type(String.class).build()))
         .isInstanceOf(ParameterException.class)
-        .hasMessageContaining("Unable to read TOML configuration file");
+        .hasMessageContaining("Unable to read from empty TOML configuration file.");
   }
 
   @Test

besu/src/test/java/org/hyperledger/besu/cli/config/ProfilesCompletionCandidatesTest.java

@@ -0,0 +1,91 @@
+/*
+ * Copyright contributors to Hyperledger Besu.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+ * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
+package org.hyperledger.besu.cli.config;
+
+import org.hyperledger.besu.cli.util.ProfileFinder;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Set;
+import java.util.TreeSet;
+import java.util.stream.Collectors;
+
+import org.assertj.core.api.Assertions;
+import org.junit.jupiter.api.AfterAll;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+
+class ProfilesCompletionCandidatesTest {
+  @TempDir private static Path tempProfilesDir;
+  private static String originalProfilesDirProperty;
+
+  @BeforeAll
+  public static void copyExternalProfiles() throws IOException {
+    for (String internalProfileName : InternalProfileName.getInternalProfileNames()) {
+      final Path profilePath = tempProfilesDir.resolve(internalProfileName + "_external.toml");
+
+      String profileConfigFile =
+          InternalProfileName.valueOfIgnoreCase(internalProfileName).get().getConfigFile();
+      try (InputStream resourceUrl =
+          ProfileFinder.class.getClassLoader().getResourceAsStream(profileConfigFile)) {
+        if (resourceUrl != null) {
+          Files.copy(resourceUrl, profilePath);
+        }
+      }
+    }
+  }
+
+  @BeforeAll
+  public static void setupSystemProperty() {
+    originalProfilesDirProperty = System.getProperty("besu.profiles.dir");
+    // sets the system property for the test
+    System.setProperty("besu.profiles.dir", tempProfilesDir.toString());
+  }
+
+  @Test
+  void profileCompletionCandidates_shouldIncludeInternalAndExternalProfiles() {
+    Iterator<String> candidates = new ProfilesCompletionCandidates().iterator();
+    // convert Iterator to List
+    List<String> candidatesList = new ArrayList<>();
+    candidates.forEachRemaining(candidatesList::add);
+
+    Assertions.assertThat(candidatesList).containsExactlyInAnyOrderElementsOf(allProfileNames());
+  }
+
+  static Set<String> allProfileNames() {
+    final Set<String> profileNames = new TreeSet<>(InternalProfileName.getInternalProfileNames());
+    final Set<String> externalProfileNames =
+        InternalProfileName.getInternalProfileNames().stream()
+            .map(name -> name + "_external")
+            .collect(Collectors.toSet());
+    profileNames.addAll(externalProfileNames);
+    return profileNames;
+  }
+
+  @AfterAll
+  public static void clearSystemProperty() {
+    if (originalProfilesDirProperty != null) {
+      System.setProperty("besu.profiles.dir", originalProfilesDirProperty);
+    } else {
+      System.clearProperty("besu.profiles.dir");
+    }
+  }
+}