From 13b4e0367476826d1df44bde53905aebc445c8f2 Mon Sep 17 00:00:00 2001 From: Eugene Kim Date: Mon, 8 Jul 2019 22:14:27 -0700 Subject: [PATCH] Make sharding configuration flexible This is done by introducing two concepts: sharding configuration instance and sharding configuration schedule. A sharding configuration instance is a particular set of sharding parameters in effect, namely: - Number of shards; - Number of nodes/shard; and - Number of Harmony-operated nodes per shard. A sharding configuration schedule is a mapping from an epoch to the corresponding sharding configuration for that epoch. Two schedules are provided and to be maintained in the long term: Mainnet sharding schedule (4 shards, 150 nodes/shard, 112 Harmony-operated nodes/shard) and public testnet sharding schedule (2 shards, 150 nodes/shard, 150 Harmony-operated nodes/shard). Harmony node binary uses one of these for each -network_type=mainnet and -network_type=testnet respectively. In addition, for -network_type=devnet, a fixed sharding schedule can be specified with direct control over all three parameters (-dn_num_shards, -dn_shard_size, and -dn_hmy_size). The mainnet schedule code includes a commented-out example schedule. --- cmd/client/txgen/main.go | 4 +- cmd/harmony/main.go | 32 +++++++++- core/resharding.go | 30 +++++---- internal/configs/sharding/fixedschedule.go | 21 +++++++ internal/configs/sharding/instance.go | 70 +++++++++++++++++++++ internal/configs/sharding/mainnet.go | 24 +++++++ internal/configs/sharding/shardingconfig.go | 26 ++++++++ internal/configs/sharding/testnet.go | 18 ++++++ 8 files changed, 211 insertions(+), 14 deletions(-) create mode 100644 internal/configs/sharding/fixedschedule.go create mode 100644 internal/configs/sharding/instance.go create mode 100644 internal/configs/sharding/mainnet.go create mode 100644 internal/configs/sharding/shardingconfig.go create mode 100644 internal/configs/sharding/testnet.go diff --git a/cmd/client/txgen/main.go b/cmd/client/txgen/main.go index 81cdb345b..73e4807f4 100644 --- a/cmd/client/txgen/main.go +++ b/cmd/client/txgen/main.go @@ -106,8 +106,9 @@ func setUpTXGen() *node.Node { consensusObj.SetStakeInfoFinder(gsif) consensusObj.ChainReader = txGen.Blockchain() consensusObj.PublicKeys = nil + genesisShardingConfig := core.ShardingSchedule.InstanceForEpoch(big.NewInt(core.GenesisEpoch)) startIdx := 0 - endIdx := startIdx + core.GenesisShardSize + endIdx := startIdx + genesisShardingConfig.NumNodesPerShard() for _, acct := range genesis.HarmonyAccounts[startIdx:endIdx] { pub := &bls2.PublicKey{} if err := pub.DeserializeHexStr(acct.BlsPublicKey); err != nil { @@ -128,6 +129,7 @@ func setUpTXGen() *node.Node { return txGen } + func main() { flag.Var(&utils.BootNodes, "bootnodes", "a list of bootnode multiaddress") flag.Parse() diff --git a/cmd/harmony/main.go b/cmd/harmony/main.go index 4e8eb5415..2bdd8c8a9 100644 --- a/cmd/harmony/main.go +++ b/cmd/harmony/main.go @@ -4,6 +4,7 @@ import ( "encoding/hex" "flag" "fmt" + "math/big" "math/rand" "os" "path" @@ -20,6 +21,7 @@ import ( "github.com/harmony-one/harmony/internal/blsgen" "github.com/harmony-one/harmony/internal/common" nodeconfig "github.com/harmony-one/harmony/internal/configs/node" + shardingconfig "github.com/harmony-one/harmony/internal/configs/sharding" "github.com/harmony-one/harmony/internal/ctxerror" "github.com/harmony-one/harmony/internal/genesis" hmykey "github.com/harmony-one/harmony/internal/keystore" @@ -100,6 +102,11 @@ var ( blsPass = flag.String("blspass", "", "The file containing passphrase to decrypt the encrypted bls file.") blsPassphrase string + // Sharding configuration parameters for devnet + devnetNumShards = flag.Uint("dn_num_shards", 2, "number of shards for -network_type=devnet (default: 2)") + devnetShardSize = flag.Int("dn_shard_size", 10, "number of nodes per shard for -network_type=devnet (default 10)") + devnetHarmonySize = flag.Int("dn_hmy_size", -1, "number of Harmony-operated nodes per shard for -network_type=devnet; negative (default) means equal to -dn_shard_size") + // logConn logs incoming/outgoing connections logConn = flag.Bool("log_conn", false, "log incoming/outgoing connections") @@ -206,7 +213,8 @@ func setupECDSAKeys() { // TODO: lc try to enable multiple staking accounts per node accountIndex, genesisAccount = setUpConsensusKeyAndReturnIndex(nodeconfig.GetDefaultConfig()) - genesisAccount.ShardID = uint32(accountIndex % core.GenesisShardNum) + genesisShardingConfig := core.ShardingSchedule.InstanceForEpoch(big.NewInt(core.GenesisEpoch)) + genesisAccount.ShardID = uint32(accountIndex) % genesisShardingConfig.NumShards() fmt.Printf("My Genesis Account: %v\n", *genesisAccount) @@ -289,7 +297,9 @@ func createGlobalConfig() *nodeconfig.ConfigType { nodeConfig.SelfPeer = p2p.Peer{IP: *ip, Port: *port, ConsensusPubKey: nodeConfig.ConsensusPubKey} - if accountIndex < core.GenesisShardNum && !*isExplorer && !*leaderOverride { // The first node in a shard is the leader at genesis + genesisShardingConfig := core.ShardingSchedule.InstanceForEpoch(big.NewInt(core.GenesisEpoch)) + genesisShardNum := int(genesisShardingConfig.NumShards()) + if accountIndex < genesisShardNum && !*isExplorer && !*leaderOverride { // The first node in a shard is the leader at genesis nodeConfig.Leader = nodeConfig.SelfPeer nodeConfig.StringRole = "leader" } else { @@ -452,6 +462,24 @@ func setUpConsensusAndNode(nodeConfig *nodeconfig.ConfigType) *node.Node { func main() { initSetup() + switch *networkType { + case nodeconfig.Mainnet: + core.ShardingSchedule = shardingconfig.MainnetSchedule + case nodeconfig.Testnet: + core.ShardingSchedule = shardingconfig.TestnetSchedule + case nodeconfig.Devnet: + if *devnetHarmonySize < 0 { + *devnetHarmonySize = *devnetShardSize + } + devnetConfig, err := shardingconfig.NewInstance( + uint32(*devnetNumShards), *devnetShardSize, *devnetHarmonySize) + if err != nil { + _, _ = fmt.Fprintf(os.Stderr, "invalid devnet sharding config: %s", + err) + os.Exit(1) + } + core.ShardingSchedule = shardingconfig.NewFixedSchedule(devnetConfig) + } nodeConfig := createGlobalConfig() // Start Profiler for leader if profile argument is on diff --git a/core/resharding.go b/core/resharding.go index 9c0bcc481..73d2f3ddf 100644 --- a/core/resharding.go +++ b/core/resharding.go @@ -13,6 +13,7 @@ import ( "github.com/harmony-one/harmony/contracts/structs" "github.com/harmony-one/harmony/core/types" common2 "github.com/harmony-one/harmony/internal/common" + shardingconfig "github.com/harmony-one/harmony/internal/configs/sharding" "github.com/harmony-one/harmony/internal/ctxerror" "github.com/harmony-one/harmony/internal/genesis" "github.com/harmony-one/harmony/internal/utils" @@ -23,12 +24,6 @@ const ( GenesisEpoch = 0 // FirstEpoch is the number of the first epoch. FirstEpoch = 1 - // GenesisShardNum is the number of shard at genesis - GenesisShardNum = 4 - // GenesisShardSize is the size of each shard at genesis - GenesisShardSize = 150 - // GenesisShardHarmonyNodes is the number of harmony node at each shard - GenesisShardHarmonyNodes = 112 // CuckooRate is the percentage of nodes getting reshuffled in the second step of cuckoo resharding. CuckooRate = 0.1 ) @@ -223,14 +218,27 @@ func (ss *ShardingState) UpdateShardingState(stakeInfo *map[common.Address]*stru return newAddresses } +// TODO ek – shardingSchedule should really be part of a general-purpose network +// configuration. We are OK for the time being, +// until the day we should let one node process join multiple networks. + +// ShardingSchedule is the sharding configuration schedule. +// Depends on the type of the network. Defaults to the mainnet schedule. +var ShardingSchedule shardingconfig.Schedule = shardingconfig.MainnetSchedule + // GetInitShardState returns the initial shard state at genesis. func GetInitShardState() types.ShardState { utils.Logger().Info().Msg("Generating Genesis Shard State.") + initShardingConfig := ShardingSchedule.InstanceForEpoch( + big.NewInt(GenesisEpoch)) + genesisShardNum := int(initShardingConfig.NumShards()) + genesisShardHarmonyNodes := initShardingConfig.NumHarmonyOperatedNodesPerShard() + genesisShardSize := initShardingConfig.NumNodesPerShard() shardState := types.ShardState{} - for i := 0; i < GenesisShardNum; i++ { + for i := 0; i < genesisShardNum; i++ { com := types.Committee{ShardID: uint32(i)} - for j := 0; j < GenesisShardHarmonyNodes; j++ { - index := i + j*GenesisShardNum // The initial account to use for genesis nodes + for j := 0; j < genesisShardHarmonyNodes; j++ { + index := i + j*genesisShardNum // The initial account to use for genesis nodes pub := &bls.PublicKey{} pub.DeserializeHexStr(genesis.HarmonyAccounts[index].BlsPublicKey) @@ -242,8 +250,8 @@ func GetInitShardState() types.ShardState { } // add FN runner's key - for j := GenesisShardHarmonyNodes; j < GenesisShardSize; j++ { - index := i + (j-GenesisShardHarmonyNodes)*GenesisShardNum + for j := genesisShardHarmonyNodes; j < genesisShardSize; j++ { + index := i + (j-genesisShardHarmonyNodes)*genesisShardNum pub := &bls.PublicKey{} pub.DeserializeHexStr(genesis.FoundationalNodeAccounts[index].BlsPublicKey) diff --git a/internal/configs/sharding/fixedschedule.go b/internal/configs/sharding/fixedschedule.go new file mode 100644 index 000000000..4a246755b --- /dev/null +++ b/internal/configs/sharding/fixedschedule.go @@ -0,0 +1,21 @@ +package shardingconfig + +import ( + "math/big" +) + +type fixedSchedule struct { + instance Instance +} + +// InstanceForEpoch returns the fixed sharding configuration instance regardless +// the given epoch. +func (s fixedSchedule) InstanceForEpoch(epoch *big.Int) Instance { + return s.instance +} + +// NewFixedSchedule returns a sharding configuration schedule that uses the +// given config instance for all epochs. Useful for testing. +func NewFixedSchedule(instance Instance) Schedule { + return fixedSchedule{instance: instance} +} diff --git a/internal/configs/sharding/instance.go b/internal/configs/sharding/instance.go new file mode 100644 index 000000000..b157b780c --- /dev/null +++ b/internal/configs/sharding/instance.go @@ -0,0 +1,70 @@ +package shardingconfig + +import "github.com/harmony-one/harmony/internal/ctxerror" + +type instance struct { + numShards uint32 + numNodesPerShard int + numHarmonyOperatedNodesPerShard int +} + +// NewInstance creates and validates a new sharding configuration based +// upon given parameters. +func NewInstance( + numShards uint32, numNodesPerShard, numHarmonyOperatedNodesPerShard int, +) (Instance, error) { + if numShards < 1 { + return nil, ctxerror.New("sharding config must have at least one shard", + "numShards", numShards) + } + if numNodesPerShard < 1 { + return nil, ctxerror.New("each shard must have at least one node", + "numNodesPerShard", numNodesPerShard) + } + if numHarmonyOperatedNodesPerShard < 0 { + return nil, ctxerror.New("Harmony-operated nodes cannot be negative", + "numHarmonyOperatedNodesPerShard", numHarmonyOperatedNodesPerShard) + } + if numHarmonyOperatedNodesPerShard > numNodesPerShard { + return nil, ctxerror.New(""+ + "number of Harmony-operated nodes cannot exceed "+ + "overall number of nodes per shard", + "numHarmonyOperatedNodesPerShard", numHarmonyOperatedNodesPerShard, + "numNodesPerShard", numNodesPerShard) + } + return instance{ + numShards: numShards, + numNodesPerShard: numNodesPerShard, + numHarmonyOperatedNodesPerShard: numHarmonyOperatedNodesPerShard, + }, nil +} + +// MustNewInstance creates a new sharding configuration based upon +// given parameters. It panics if parameter validation fails. +// It is intended to be used for static initialization. +func MustNewInstance( + numShards uint32, numNodesPerShard, numHarmonyOperatedNodesPerShard int, +) Instance { + sc, err := NewInstance( + numShards, numNodesPerShard, numHarmonyOperatedNodesPerShard) + if err != nil { + panic(err) + } + return sc +} + +// NumShards returns the number of shards in the network. +func (sc instance) NumShards() uint32 { + return sc.numShards +} + +// NumNodesPerShard returns number of nodes in each shard. +func (sc instance) NumNodesPerShard() int { + return sc.numNodesPerShard +} + +// NumHarmonyOperatedNodesPerShard returns number of nodes in each shard +// that are operated by Harmony. +func (sc instance) NumHarmonyOperatedNodesPerShard() int { + return sc.numHarmonyOperatedNodesPerShard +} diff --git a/internal/configs/sharding/mainnet.go b/internal/configs/sharding/mainnet.go new file mode 100644 index 000000000..af873d855 --- /dev/null +++ b/internal/configs/sharding/mainnet.go @@ -0,0 +1,24 @@ +package shardingconfig + +import "math/big" + +// MainnetSchedule is the mainnet sharding configuration schedule. +var MainnetSchedule mainnetSchedule + +type mainnetSchedule struct{} + +func (mainnetSchedule) InstanceForEpoch(epoch *big.Int) Instance { + switch { + //case epoch.Cmp(big.NewInt(1000)) >= 0: + // return mainnet6400 + //case epoch.Cmp(big.NewInt(100)) >= 0: + // return mainnetV2 + default: // genesis + return mainnetV0 + } +} + +var mainnetV0 = MustNewInstance(4, 150, 112) + +//var mainnetV2 = MustNewInstance(8, 200, 100) +//var mainnet6400 = MustNewInstance(16, 400, 50) diff --git a/internal/configs/sharding/shardingconfig.go b/internal/configs/sharding/shardingconfig.go new file mode 100644 index 000000000..04d606eb2 --- /dev/null +++ b/internal/configs/sharding/shardingconfig.go @@ -0,0 +1,26 @@ +// Package shardingconfig defines types and utilities that deal with Harmony +// sharding configuration schedule. +package shardingconfig + +import ( + "math/big" +) + +// Schedule returns the sharding configuration instance for the given +// epoch. +type Schedule interface { + InstanceForEpoch(epoch *big.Int) Instance +} + +// Instance is one sharding configuration instance. +type Instance interface { + // NumShards returns the number of shards in the network. + NumShards() uint32 + + // NumNodesPerShard returns number of nodes in each shard. + NumNodesPerShard() int + + // NumHarmonyOperatedNodesPerShard returns number of nodes in each shard + // that are operated by Harmony. + NumHarmonyOperatedNodesPerShard() int +} diff --git a/internal/configs/sharding/testnet.go b/internal/configs/sharding/testnet.go new file mode 100644 index 000000000..8df67fe9b --- /dev/null +++ b/internal/configs/sharding/testnet.go @@ -0,0 +1,18 @@ +package shardingconfig + +import "math/big" + +// TestnetSchedule is the long-running public testnet sharding +// configuration schedule. +var TestnetSchedule testnetSchedule + +type testnetSchedule struct{} + +func (testnetSchedule) InstanceForEpoch(epoch *big.Int) Instance { + switch { + default: // genesis + return testnetV0 + } +} + +var testnetV0 = MustNewInstance(2, 150, 150)