From bbdee5dfc233a3517f52307cfe24d31885728e58 Mon Sep 17 00:00:00 2001 From: Jacky Wang Date: Fri, 17 Jul 2020 15:40:24 -0700 Subject: [PATCH] [cmd] changed config half way --- cmd/harmony/config.go | 103 +++++++++++++++++++ cmd/harmony/config_test.go | 116 +++++++++++++++++++++ cmd/harmony/main.go | 200 ++++++++++++++++++++++++++++--------- 3 files changed, 372 insertions(+), 47 deletions(-) create mode 100644 cmd/harmony/config.go create mode 100644 cmd/harmony/config_test.go diff --git a/cmd/harmony/config.go b/cmd/harmony/config.go new file mode 100644 index 000000000..a19d41921 --- /dev/null +++ b/cmd/harmony/config.go @@ -0,0 +1,103 @@ +package main + +import ( + "io/ioutil" + "time" + + "github.com/pelletier/go-toml" +) + +type hmyConfig struct { + Run runConfig + Network networkConfig + Consensus consensusConfig + BLSKey blsConfig + TxPool txPoolConfig + Storage storageConfig + Pprof pprofConfig + Log logConfig + Devnet *devnetConfig `toml:",omitempty"` +} + +type runConfig struct { + NodeType string + IsStaking bool + ShardID int +} + +type networkConfig struct { + Network string + IP string + Port int + MinPeers int + P2PKeyFile string + PublicRPC bool + + BootNodes []string + DNSZone string + DNSPort int +} + +type consensusConfig struct { + DelayCommit time.Duration + BlockTime time.Duration +} + +type blsConfig struct { + KeyDir string + KeyFiles []string + maxBLSKeys int + + PassSrcType string + PassFile string + SavePassphrase bool + KmsConfigSrcType string + KmsConfigFile string +} + +type txPoolConfig struct { + BlacklistFile string + BroadcastInvalidTx bool +} + +type storageConfig struct { + IsArchival bool + DatabaseDir string +} + +type pprofConfig struct { + Enabled bool + ListenAddr string +} + +type logConfig struct { + LogFolder string + LogMaxSize int +} + +type devnetConfig struct { + NumShards uint + ShardSize int + HmyNodeSize int +} + +func loadConfig(file string) (hmyConfig, error) { + b, err := ioutil.ReadFile(file) + if err != nil { + return hmyConfig{}, err + } + + var config hmyConfig + if err := toml.Unmarshal(b, &config); err != nil { + return hmyConfig{}, err + } + return config, nil +} + +func writeConfigToFile(config hmyConfig, file string) error { + b, err := toml.Marshal(config) + if err != nil { + return err + } + return ioutil.WriteFile(file, b, 0644) +} diff --git a/cmd/harmony/config_test.go b/cmd/harmony/config_test.go new file mode 100644 index 000000000..d9de0d4c7 --- /dev/null +++ b/cmd/harmony/config_test.go @@ -0,0 +1,116 @@ +package main + +import ( + "fmt" + "os" + "path/filepath" + "reflect" + "testing" + "time" +) + +var testHmyConfig = hmyConfig{ + Run: runConfig{ + NodeType: "validator", + IsStaking: true, + ShardID: 0, + }, + Network: networkConfig{ + Network: "mainnet", + IP: "127.0.0.1", + Port: 9000, + MinPeers: 32, + P2PKeyFile: "./.hmykey", + PublicRPC: false, + BootNodes: []string{ + "/ip4/100.26.90.187/tcp/9874/p2p/Qmdfjtk6hPoyrH1zVD9PEH4zfWLo38dP2mDvvKXfh3tnEv", + "/ip4/54.213.43.194/tcp/9874/p2p/QmZJJx6AdaoEkGLrYG4JeLCKeCKDjnFz2wfHNHxAqFSGA9", + "/ip4/13.113.101.219/tcp/12019/p2p/QmQayinFSgMMw5cSpDUiD9pQ2WeP6WNmGxpZ6ou3mdVFJX", + "/ip4/99.81.170.167/tcp/12019/p2p/QmRVbTpEYup8dSaURZfF6ByrMTSKa4UyUzJhSjahFzRqNj", + }, + DNSZone: "t.hmny.io", + DNSPort: 9000, + }, + Consensus: consensusConfig{ + DelayCommit: 0, + BlockTime: 8 * time.Second, + }, + BLSKey: blsConfig{ + KeyDir: "./.hmy/blskeys", + KeyFiles: []string{"./xxxx.key"}, + PassSrcType: "auto", + PassFile: "pass.file", + SavePassphrase: true, + KmsConfigSrcType: "shared", + KmsConfigFile: "config.json", + }, + TxPool: txPoolConfig{ + BlacklistFile: ".hmy/blacklist.txt", + BroadcastInvalidTx: false, + }, + Storage: storageConfig{ + IsArchival: false, + DatabaseDir: "./", + }, + Pprof: pprofConfig{ + Enabled: true, + ListenAddr: "localhost:6060", + }, + Log: logConfig{ + LogFolder: "latest", + LogMaxSize: 100, + }, + Devnet: nil, +} + +var devnetHmyConfig = hmyConfig{ + Network: networkConfig{ + Network: "devnet", + BootNodes: []string{}, + }, + Devnet: &devnetConfig{ + NumShards: 2, + }, + BLSKey: blsConfig{ + KeyFiles: []string{}, + }, +} + +var testBaseDir = filepath.Join(os.TempDir(), "harmony", "cmd", "harmony") + +func init() { + if _, err := os.Stat(testBaseDir); os.IsNotExist(err) { + os.MkdirAll(testBaseDir, 0777) + } +} + +func TestPersistConfig(t *testing.T) { + testDir := filepath.Join(testBaseDir, t.Name()) + os.RemoveAll(testDir) + os.MkdirAll(testDir, 0777) + + tests := []struct { + config hmyConfig + }{ + { + config: testHmyConfig, + }, + { + config: devnetHmyConfig, + }, + } + for i, test := range tests { + file := filepath.Join(testDir, fmt.Sprintf("%d.conf", i)) + + if err := writeConfigToFile(test.config, file); err != nil { + t.Fatal(err) + } + config, err := loadConfig(file) + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(config, test.config) { + t.Errorf("Test %v: unexpected config \n\t%+v \n\t%+v", i, config, test.config) + } + } +} diff --git a/cmd/harmony/main.go b/cmd/harmony/main.go index d9963d378..b816ae687 100644 --- a/cmd/harmony/main.go +++ b/cmd/harmony/main.go @@ -17,6 +17,8 @@ import ( "syscall" "time" + "github.com/spf13/cobra" + ethCommon "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/log" "github.com/harmony-one/bls/ffi/go/bls" @@ -60,55 +62,155 @@ func printVersion() { } var ( - ip = flag.String("ip", "127.0.0.1", "ip of the node") - port = flag.String("port", "9000", "port of the node.") - logFolder = flag.String("log_folder", "latest", "the folder collecting the logs of this execution") - logMaxSize = flag.Int("log_max_size", 100, "the max size in megabytes of the log file before it gets rotated") - freshDB = flag.Bool("fresh_db", false, "true means the existing disk based db will be removed") - pprof = flag.String("pprof", "", "what address and port the pprof profiling server should listen on") - versionFlag = flag.Bool("version", false, "Output version info") - dnsZone = flag.String("dns_zone", "", "if given and not empty, use peers from the zone (default: use libp2p peer discovery instead)") - dnsFlag = flag.Bool("dns", true, "[deprecated] equivalent to -dns_zone t.hmny.io") - dnsPort = flag.String("dns_port", "9000", "port of dns node") - //Leader needs to have a minimal number of peers to start consensus - minPeers = flag.Int("min_peers", 32, "Minimal number of Peers in shard") - // Key file to store the private key - keyFile = flag.String("key", "./.hmykey", "the p2p key file of the harmony node") - // isArchival indicates this node is an archival node that will save and archive current blockchain - isArchival = flag.Bool("is_archival", false, "false will enable cached state pruning") - // delayCommit is the commit-delay timer, used by Harmony nodes - delayCommit = flag.String("delay_commit", "0ms", "how long to delay sending commit messages in consensus, ex: 500ms, 1s") - // nodeType indicates the type of the node: validator, explorer - nodeType = flag.String("node_type", "validator", "node type: validator, explorer") - // networkType indicates the type of the network - networkType = flag.String("network_type", "mainnet", "type of the network: mainnet, testnet, pangaea, partner, stressnet, devnet, localnet") - // blockPeriod indicates the how long the leader waits to propose a new block. - blockPeriod = flag.Int("block_period", 5, "how long in second the leader waits to propose a new block.") - // staking indicates whether the node is operating in staking mode. - stakingFlag = flag.Bool("staking", false, "whether the node should operate in staking mode") - // shardID indicates the shard ID of this node - shardID = flag.Int("shard_id", -1, "the shard ID of this node") - // Sharding configuration parameters for devnet - devnetNumShards = flag.Uint("dn_num_shards", 2, "number of shards for -network_type=devnet (default: 2)") - devnetShardSize = flag.Int("dn_shard_size", 10, "number of nodes per shard for -network_type=devnet (default 10)") - devnetHarmonySize = flag.Int("dn_hmy_size", -1, "number of Harmony-operated nodes per shard for -network_type=devnet; negative (default) means equal to -dn_shard_size") - // logging verbosity - verbosity = flag.Int("verbosity", 5, "Logging verbosity: 0=silent, 1=error, 2=warn, 3=info, 4=debug, 5=detail (default: 5)") - // dbDir is the database directory. - dbDir = flag.String("db_dir", "", "blockchain database directory") - publicRPC = flag.Bool("public_rpc", false, "Enable Public RPC Access (default: false)") - // Bad block revert - doRevertBefore = flag.Int("do_revert_before", 0, "If the current block is less than do_revert_before, revert all blocks until (including) revert_to block") - revertTo = flag.Int("revert_to", 0, "The revert will rollback all blocks until and including block number revert_to") - revertBeacon = flag.Bool("revert_beacon", false, "Whether to revert beacon chain or the chain this node is assigned to") - // Blacklist of addresses - blacklistPath = flag.String("blacklist", "./.hmy/blacklist.txt", "Path to newline delimited file of blacklisted wallet addresses") - broadcastInvalidTx = flag.Bool("broadcast_invalid_tx", false, "Broadcast invalid transactions to sync pool state (default: false)") - webHookYamlPath = flag.String( - "webhook_yaml", "", "path for yaml config reporting double signing", - ) + // config file + configFile string + // run config + nodeType string + noStaking bool + // network config + networkType string + ip string + port int + p2pKeyFile string + publicRPC bool + bootNodes []string + dnsZone string + dnsPort int + // consensus config + delayCommit string + blockTime string + // bls config + keyDir string + keyFiles []string + maxBLSKeys int + passSrcType string + passFile string + savePassphrase bool + kmsConfigSrcType string + kmsConfigFile string + // transaction pool config + blacklistFile string + broadcastInvalidTx bool + // storage config + isArchival bool + databaseDir string + // pprof config + pprofListenAddr string + // log config + logFolder string + logMaxSize int + // devnet config + devnetNumShards uint + devnetShardSize int + hmyNodeSize int ) +// legacy fields +var ( + isStaking bool + legacyNodeType string + shardID int + + legacyNetworkType string + legacyMinPeers int +) + +var rootCmd = &cobra.Command{ + // TODO: elaborate the usage + Use: "harmony", + Short: "", + Long: "", + Run: runHarmonyNode, +} + +func init() { +} + +func setupRootFlags(cmd *cobra.Command) error { + flags := cmd.Flags() + flags.StringVarP(&configFile, "config", "c", "", "config toml file to load from") + flags.StringVarP(&nodeType, "node-type", "", "validator", "node type to run (validator, explorer)") + flags.BoolVarP(&noStaking, "no-staking", "", false, "run node in legacy mode") + + flags.StringVarP(&networkType, "network", "n", "mainnet", "network to join (mainnet, testnet, pangaea, localnet, partner, stressnet, devnet") + flags.StringVarP(&ip, "ip", "", "127.0.0.1", "ip address to listen") + flags.IntVarP(&port, "port", "", 9000, "port to listen") +} + +// TODO: When fully get rid of node.sh, change MarkHidden to MarkDeprecated +func setupRootLegacyFlags(cmd *cobra.Command) error { + flags := cmd.Flags() + flags.BoolVarP(&isStaking, "staking", "", false, "[deprecated] run node in staking mode") + if err := flags.MarkDeprecated("staking", "Staking mode is enabled by default. Use --no-staking to run in legacy mode"); err != nil { + return err + } + flags.StringVarP(&legacyNodeType, "node_type", "", "[deprecated] validator", "node type to run (validator, explorer)") + if err := flags.MarkHidden("node_type"); err != nil { + return err + } + flags.IntVarP(&shardID, "shard_id", "", -1, "[deprecated] the shard ID of this node") + if err := flags.MarkHidden("shard_id"); err != nil { + return err + } + flags.StringVarP(&legacyNetworkType, "network_type", "", "mainnet", "[deprecated] network to join") + if err := flags.MarkHidden("network_type"); err != nil { + return err + } + flags.IntVarP(&legacyMinPeers, "min_peers", "", 32, "minimal number of peers in shard") + if err := flags.MarkDeprecated() +} + +//ip = flag.String("ip", "127.0.0.1", "ip of the node") +//port = flag.String("port", "9000", "port of the node.") +//logFolder = flag.String("log_folder", "latest", "the folder collecting the logs of this execution") +//logMaxSize = flag.Int("log_max_size", 100, "the max size in megabytes of the log file before it gets rotated") +//// Remove this flag +//freshDB = flag.Bool("fresh_db", false, "true means the existing disk based db will be removed") +//pprof = flag.String("pprof", "", "what address and port the pprof profiling server should listen on") +//versionFlag = flag.Bool("version", false, "Output version info") +//dnsZone = flag.String("dns_zone", "", "if given and not empty, use peers from the zone (default: use libp2p peer discovery instead)") +//dnsFlag = flag.Bool("dns", true, "[deprecated] equivalent to -dns_zone t.hmny.io") +//dnsPort = flag.String("dns_port", "9000", "port of dns node") +////Leader needs to have a minimal number of peers to start consensus +//minPeers = flag.Int("min_peers", 32, "Minimal number of Peers in shard") +//// Key file to store the private key +//keyFile = flag.String("key", "./.hmykey", "the p2p key file of the harmony node") +//// isArchival indicates this node is an archival node that will save and archive current blockchain +//isArchival = flag.Bool("is_archival", false, "false will enable cached state pruning") +//// delayCommit is the commit-delay timer, used by Harmony nodes +//// TODO: check whether this field can be removed +//delayCommit = flag.String("delay_commit", "0ms", "how long to delay sending commit messages in consensus, ex: 500ms, 1s") +//// nodeType indicates the type of the node: validator, explorer +//nodeType = flag.String("node_type", "validator", "node type: validator, explorer") +//// networkType indicates the type of the network +//networkType = flag.String("network_type", "mainnet", "type of the network: mainnet, testnet, pangaea, partner, stressnet, devnet, localnet") +//// blockPeriod indicates the how long the leader waits to propose a new block. +//blockPeriod = flag.Int("block_period", 8, "how long in second the leader waits to propose a new block.") +//// staking indicates whether the node is operating in staking mode. +//stakingFlag = flag.Bool("staking", false, "whether the node should operate in staking mode") +//// shardID indicates the shard ID of this node +//shardID = flag.Int("shard_id", -1, "the shard ID of this node") +//// Sharding configuration parameters for devnet +//devnetNumShards = flag.Uint("dn_num_shards", 2, "number of shards for -network_type=devnet (default: 2)") +//devnetShardSize = flag.Int("dn_shard_size", 10, "number of nodes per shard for -network_type=devnet (default 10)") +//devnetHarmonySize = flag.Int("dn_hmy_size", -1, "number of Harmony-operated nodes per shard for -network_type=devnet; negative (default) means equal to -dn_shard_size") +//// logging verbosity +//verbosity = flag.Int("verbosity", 5, "Logging verbosity: 0=silent, 1=error, 2=warn, 3=info, 4=debug, 5=detail (default: 5)") +//// dbDir is the database directory. +//dbDir = flag.String("db_dir", "", "blockchain database directory") +//publicRPC = flag.Bool("public_rpc", false, "Enable Public RPC Access (default: false)") +//// Bad block revert +//// TODO: Seperate revert to a different command +//doRevertBefore = flag.Int("do_revert_before", 0, "If the current block is less than do_revert_before, revert all blocks until (including) revert_to block") +//revertTo = flag.Int("revert_to", 0, "The revert will rollback all blocks until and including block number revert_to") +//revertBeacon = flag.Bool("revert_beacon", false, "Whether to revert beacon chain or the chain this node is assigned to") +//// Blacklist of addresses +//blacklistPath = flag.String("blacklist", "./.hmy/blacklist.txt", "Path to newline delimited file of blacklisted wallet addresses") +//broadcastInvalidTx = flag.Bool("broadcast_invalid_tx", false, "Broadcast invalid transactions to sync pool state (default: false)") +//webHookYamlPath = flag.String( +// "webhook_yaml", "", "path for yaml config reporting double signing", +//) + func initSetup() { // Setup pprof @@ -144,6 +246,10 @@ func initSetup() { } } +func runHarmonyNode(cmd *cobra.Command, args []string) { + +} + func findAccountsByPubKeys(config shardingconfig.Instance, pubKeys multibls.PublicKeys) { for _, key := range pubKeys { keyStr := key.Bytes.Hex()