Make sharding configuration flexible

This is done by introducing two concepts: sharding configuration
instance and sharding configuration schedule.

A sharding configuration instance is a particular set of sharding
parameters in effect, namely:

  - Number of shards;
  - Number of nodes/shard; and
  - Number of Harmony-operated nodes per shard.

A sharding configuration schedule is a mapping from an epoch to the
corresponding sharding configuration for that epoch.

Two schedules are provided and to be maintained in the long term:
Mainnet sharding schedule (4 shards, 150 nodes/shard, 112
Harmony-operated nodes/shard) and public testnet sharding schedule (2
shards, 150 nodes/shard, 150 Harmony-operated nodes/shard).

Harmony node binary uses one of these for each -network_type=mainnet and
-network_type=testnet respectively.

In addition, for -network_type=devnet, a fixed sharding schedule can be
specified with direct control over all three parameters (-dn_num_shards,
-dn_shard_size, and -dn_hmy_size).

The mainnet schedule code includes a commented-out example schedule.

cmd/client/txgen/main.go

@@ -106,8 +106,9 @@ func setUpTXGen() *node.Node {
 	consensusObj.SetStakeInfoFinder(gsif)
 	consensusObj.ChainReader = txGen.Blockchain()
 	consensusObj.PublicKeys = nil
+	genesisShardingConfig := core.ShardingSchedule.InstanceForEpoch(big.NewInt(core.GenesisEpoch))
 	startIdx := 0
-	endIdx := startIdx + core.GenesisShardSize
+	endIdx := startIdx + genesisShardingConfig.NumNodesPerShard()
 	for _, acct := range genesis.HarmonyAccounts[startIdx:endIdx] {
 		pub := &bls2.PublicKey{}
 		if err := pub.DeserializeHexStr(acct.BlsPublicKey); err != nil {
@@ -128,6 +129,7 @@ func setUpTXGen() *node.Node {
 
 	return txGen
 }
+
 func main() {
 	flag.Var(&utils.BootNodes, "bootnodes", "a list of bootnode multiaddress")
 	flag.Parse()

cmd/harmony/main.go

@@ -4,6 +4,7 @@ import (
 	"encoding/hex"
 	"flag"
 	"fmt"
+	"math/big"
 	"math/rand"
 	"os"
 	"path"
@@ -20,6 +21,7 @@ import (
 	"github.com/harmony-one/harmony/internal/blsgen"
 	"github.com/harmony-one/harmony/internal/common"
 	nodeconfig "github.com/harmony-one/harmony/internal/configs/node"
+	shardingconfig "github.com/harmony-one/harmony/internal/configs/sharding"
 	"github.com/harmony-one/harmony/internal/ctxerror"
 	"github.com/harmony-one/harmony/internal/genesis"
 	hmykey "github.com/harmony-one/harmony/internal/keystore"
@@ -100,6 +102,11 @@ var (
 	blsPass            = flag.String("blspass", "", "The file containing passphrase to decrypt the encrypted bls file.")
 	blsPassphrase      string
 
+	// Sharding configuration parameters for devnet
+	devnetNumShards   = flag.Uint("dn_num_shards", 2, "number of shards for -network_type=devnet (default: 2)")
+	devnetShardSize   = flag.Int("dn_shard_size", 10, "number of nodes per shard for -network_type=devnet (default 10)")
+	devnetHarmonySize = flag.Int("dn_hmy_size", -1, "number of Harmony-operated nodes per shard for -network_type=devnet; negative (default) means equal to -dn_shard_size")
+
 	// logConn logs incoming/outgoing connections
 	logConn = flag.Bool("log_conn", false, "log incoming/outgoing connections")
 
@@ -206,7 +213,8 @@ func setupECDSAKeys() {
 	// TODO: lc try to enable multiple staking accounts per node
 	accountIndex, genesisAccount = setUpConsensusKeyAndReturnIndex(nodeconfig.GetDefaultConfig())
 
-	genesisAccount.ShardID = uint32(accountIndex % core.GenesisShardNum)
+	genesisShardingConfig := core.ShardingSchedule.InstanceForEpoch(big.NewInt(core.GenesisEpoch))
+	genesisAccount.ShardID = uint32(accountIndex) % genesisShardingConfig.NumShards()
 
 	fmt.Printf("My Genesis Account: %v\n", *genesisAccount)
 
@@ -289,7 +297,9 @@ func createGlobalConfig() *nodeconfig.ConfigType {
 
 	nodeConfig.SelfPeer = p2p.Peer{IP: *ip, Port: *port, ConsensusPubKey: nodeConfig.ConsensusPubKey}
 
-	if accountIndex < core.GenesisShardNum && !*isExplorer && !*leaderOverride { // The first node in a shard is the leader at genesis
+	genesisShardingConfig := core.ShardingSchedule.InstanceForEpoch(big.NewInt(core.GenesisEpoch))
+	genesisShardNum := int(genesisShardingConfig.NumShards())
+	if accountIndex < genesisShardNum && !*isExplorer && !*leaderOverride { // The first node in a shard is the leader at genesis
 		nodeConfig.Leader = nodeConfig.SelfPeer
 		nodeConfig.StringRole = "leader"
 	} else {
@@ -452,6 +462,24 @@ func setUpConsensusAndNode(nodeConfig *nodeconfig.ConfigType) *node.Node {
 
 func main() {
 	initSetup()
+	switch *networkType {
+	case nodeconfig.Mainnet:
+		core.ShardingSchedule = shardingconfig.MainnetSchedule
+	case nodeconfig.Testnet:
+		core.ShardingSchedule = shardingconfig.TestnetSchedule
+	case nodeconfig.Devnet:
+		if *devnetHarmonySize < 0 {
+			*devnetHarmonySize = *devnetShardSize
+		}
+		devnetConfig, err := shardingconfig.NewInstance(
+			uint32(*devnetNumShards), *devnetShardSize, *devnetHarmonySize)
+		if err != nil {
+			_, _ = fmt.Fprintf(os.Stderr, "invalid devnet sharding config: %s",
+				err)
+			os.Exit(1)
+		}
+		core.ShardingSchedule = shardingconfig.NewFixedSchedule(devnetConfig)
+	}
 	nodeConfig := createGlobalConfig()
 
 	// Start Profiler for leader if profile argument is on

core/resharding.go

@@ -13,6 +13,7 @@ import (
 	"github.com/harmony-one/harmony/contracts/structs"
 	"github.com/harmony-one/harmony/core/types"
 	common2 "github.com/harmony-one/harmony/internal/common"
+	shardingconfig "github.com/harmony-one/harmony/internal/configs/sharding"
 	"github.com/harmony-one/harmony/internal/ctxerror"
 	"github.com/harmony-one/harmony/internal/genesis"
 	"github.com/harmony-one/harmony/internal/utils"
@@ -23,12 +24,6 @@ const (
 	GenesisEpoch = 0
 	// FirstEpoch is the number of the first epoch.
 	FirstEpoch = 1
-	// GenesisShardNum is the number of shard at genesis
-	GenesisShardNum = 4
-	// GenesisShardSize is the size of each shard at genesis
-	GenesisShardSize = 150
-	// GenesisShardHarmonyNodes is the number of harmony node at each shard
-	GenesisShardHarmonyNodes = 112
 	// CuckooRate is the percentage of nodes getting reshuffled in the second step of cuckoo resharding.
 	CuckooRate = 0.1
 )
@@ -223,14 +218,27 @@ func (ss *ShardingState) UpdateShardingState(stakeInfo *map[common.Address]*stru
 	return newAddresses
 }
 
+// TODO ek – shardingSchedule should really be part of a general-purpose network
+//  configuration.  We are OK for the time being,
+//  until the day we should let one node process join multiple networks.
+
+// ShardingSchedule is the sharding configuration schedule.
+// Depends on the type of the network.  Defaults to the mainnet schedule.
+var ShardingSchedule shardingconfig.Schedule = shardingconfig.MainnetSchedule
+
 // GetInitShardState returns the initial shard state at genesis.
 func GetInitShardState() types.ShardState {
 	utils.Logger().Info().Msg("Generating Genesis Shard State.")
+	initShardingConfig := ShardingSchedule.InstanceForEpoch(
+		big.NewInt(GenesisEpoch))
+	genesisShardNum := int(initShardingConfig.NumShards())
+	genesisShardHarmonyNodes := initShardingConfig.NumHarmonyOperatedNodesPerShard()
+	genesisShardSize := initShardingConfig.NumNodesPerShard()
 	shardState := types.ShardState{}
-	for i := 0; i < GenesisShardNum; i++ {
+	for i := 0; i < genesisShardNum; i++ {
 		com := types.Committee{ShardID: uint32(i)}
-		for j := 0; j < GenesisShardHarmonyNodes; j++ {
+		for j := 0; j < genesisShardHarmonyNodes; j++ {
-			index := i + j*GenesisShardNum // The initial account to use for genesis nodes
+			index := i + j*genesisShardNum // The initial account to use for genesis nodes
 
 			pub := &bls.PublicKey{}
 			pub.DeserializeHexStr(genesis.HarmonyAccounts[index].BlsPublicKey)
@@ -242,8 +250,8 @@ func GetInitShardState() types.ShardState {
 		}
 
 		// add FN runner's key
-		for j := GenesisShardHarmonyNodes; j < GenesisShardSize; j++ {
+		for j := genesisShardHarmonyNodes; j < genesisShardSize; j++ {
-			index := i + (j-GenesisShardHarmonyNodes)*GenesisShardNum
+			index := i + (j-genesisShardHarmonyNodes)*genesisShardNum
 
 			pub := &bls.PublicKey{}
 			pub.DeserializeHexStr(genesis.FoundationalNodeAccounts[index].BlsPublicKey)

internal/configs/sharding/fixedschedule.go

@@ -0,0 +1,21 @@
+package shardingconfig
+
+import (
+	"math/big"
+)
+
+type fixedSchedule struct {
+	instance Instance
+}
+
+// InstanceForEpoch returns the fixed sharding configuration instance regardless
+// the given epoch.
+func (s fixedSchedule) InstanceForEpoch(epoch *big.Int) Instance {
+	return s.instance
+}
+
+// NewFixedSchedule returns a sharding configuration schedule that uses the
+// given config instance for all epochs.  Useful for testing.
+func NewFixedSchedule(instance Instance) Schedule {
+	return fixedSchedule{instance: instance}
+}

internal/configs/sharding/instance.go

@@ -0,0 +1,70 @@
+package shardingconfig
+
+import "github.com/harmony-one/harmony/internal/ctxerror"
+
+type instance struct {
+	numShards                       uint32
+	numNodesPerShard                int
+	numHarmonyOperatedNodesPerShard int
+}
+
+// NewInstance creates and validates a new sharding configuration based
+// upon given parameters.
+func NewInstance(
+	numShards uint32, numNodesPerShard, numHarmonyOperatedNodesPerShard int,
+) (Instance, error) {
+	if numShards < 1 {
+		return nil, ctxerror.New("sharding config must have at least one shard",
+			"numShards", numShards)
+	}
+	if numNodesPerShard < 1 {
+		return nil, ctxerror.New("each shard must have at least one node",
+			"numNodesPerShard", numNodesPerShard)
+	}
+	if numHarmonyOperatedNodesPerShard < 0 {
+		return nil, ctxerror.New("Harmony-operated nodes cannot be negative",
+			"numHarmonyOperatedNodesPerShard", numHarmonyOperatedNodesPerShard)
+	}
+	if numHarmonyOperatedNodesPerShard > numNodesPerShard {
+		return nil, ctxerror.New(""+
+			"number of Harmony-operated nodes cannot exceed "+
+			"overall number of nodes per shard",
+			"numHarmonyOperatedNodesPerShard", numHarmonyOperatedNodesPerShard,
+			"numNodesPerShard", numNodesPerShard)
+	}
+	return instance{
+		numShards:                       numShards,
+		numNodesPerShard:                numNodesPerShard,
+		numHarmonyOperatedNodesPerShard: numHarmonyOperatedNodesPerShard,
+	}, nil
+}
+
+// MustNewInstance creates a new sharding configuration based upon
+// given parameters.  It panics if parameter validation fails.
+// It is intended to be used for static initialization.
+func MustNewInstance(
+	numShards uint32, numNodesPerShard, numHarmonyOperatedNodesPerShard int,
+) Instance {
+	sc, err := NewInstance(
+		numShards, numNodesPerShard, numHarmonyOperatedNodesPerShard)
+	if err != nil {
+		panic(err)
+	}
+	return sc
+}
+
+// NumShards returns the number of shards in the network.
+func (sc instance) NumShards() uint32 {
+	return sc.numShards
+}
+
+// NumNodesPerShard returns number of nodes in each shard.
+func (sc instance) NumNodesPerShard() int {
+	return sc.numNodesPerShard
+}
+
+// NumHarmonyOperatedNodesPerShard returns number of nodes in each shard
+// that are operated by Harmony.
+func (sc instance) NumHarmonyOperatedNodesPerShard() int {
+	return sc.numHarmonyOperatedNodesPerShard
+}

internal/configs/sharding/mainnet.go

@@ -0,0 +1,24 @@
+package shardingconfig
+
+import "math/big"
+
+// MainnetSchedule is the mainnet sharding configuration schedule.
+var MainnetSchedule mainnetSchedule
+
+type mainnetSchedule struct{}
+
+func (mainnetSchedule) InstanceForEpoch(epoch *big.Int) Instance {
+	switch {
+	//case epoch.Cmp(big.NewInt(1000)) >= 0:
+	//	return mainnet6400
+	//case epoch.Cmp(big.NewInt(100)) >= 0:
+	//	return mainnetV2
+	default: // genesis
+		return mainnetV0
+	}
+}
+
+var mainnetV0 = MustNewInstance(4, 150, 112)
+
+//var mainnetV2 = MustNewInstance(8, 200, 100)
+//var mainnet6400 = MustNewInstance(16, 400, 50)

internal/configs/sharding/shardingconfig.go

@@ -0,0 +1,26 @@
+// Package shardingconfig defines types and utilities that deal with Harmony
+// sharding configuration schedule.
+package shardingconfig
+
+import (
+	"math/big"
+)
+
+// Schedule returns the sharding configuration instance for the given
+// epoch.
+type Schedule interface {
+	InstanceForEpoch(epoch *big.Int) Instance
+}
+
+// Instance is one sharding configuration instance.
+type Instance interface {
+	// NumShards returns the number of shards in the network.
+	NumShards() uint32
+
+	// NumNodesPerShard returns number of nodes in each shard.
+	NumNodesPerShard() int
+
+	// NumHarmonyOperatedNodesPerShard returns number of nodes in each shard
+	// that are operated by Harmony.
+	NumHarmonyOperatedNodesPerShard() int
+}

internal/configs/sharding/testnet.go

@@ -0,0 +1,18 @@
+package shardingconfig
+
+import "math/big"
+
+// TestnetSchedule is the long-running public testnet sharding
+// configuration schedule.
+var TestnetSchedule testnetSchedule
+
+type testnetSchedule struct{}
+
+func (testnetSchedule) InstanceForEpoch(epoch *big.Int) Instance {
+	switch {
+	default: // genesis
+		return testnetV0
+	}
+}
+
+var testnetV0 = MustNewInstance(2, 150, 150)