add list of validators, propagate selection mode (#1530)

* add list of validators, propagate selection mode
* Check that genesis only has a single selection mode
* Override blockheader valdiators at a particular height
* remove invalid validation - needs to be done at startup
* change validators list of blockheader on specific transition

Co-authored-by: Antony Denyer <git@antonydenyer.co.uk>

Author: baptiste-b-pegasys

cmd/utils/flags.go

@@ -2513,23 +2513,6 @@ func MakeChain(ctx *cli.Context, stack *node.Node, useExist bool) (chain *core.B
 	}
 	if config.Clique != nil {
 		engine = clique.New(config.Clique, chainDb)
-	} else if config.QBFT != nil {
-		log.Debug("setBFTConfig", "proto", "qbft")
-		qbftConfig := setBFTConfig(config.QBFT.BFTConfig)
-		qbftConfig.TestQBFTBlock = big.NewInt(0)
-		qbftConfig.Transitions = config.Transitions
-		if config.QBFT.ValidatorContractAddress != (common.Address{}) {
-			qbftConfig.ValidatorContract = config.QBFT.ValidatorContractAddress
-		}
-		qbftConfig.Client = ethclient.NewClient(client)
-		engine = istanbulBackend.New(qbftConfig, stack.GetNodeKey(), chainDb)
-	} else if config.IBFT != nil {
-		log.Debug("setBFTConfig", "proto", "ibft")
-		ibftConfig := setBFTConfig(config.IBFT.BFTConfig)
-		ibftConfig.TestQBFTBlock = nil
-		ibftConfig.Transitions = config.Transitions
-		ibftConfig.Client = ethclient.NewClient(client)
-		engine = istanbulBackend.New(ibftConfig, stack.GetNodeKey(), chainDb)
 	} else if config.Istanbul != nil {
 		log.Warn("WARNING: The attribute config.istanbul is deprecated and will be removed in the future, please use config.ibft on genesis file")
 		// for IBFT
@@ -2546,9 +2529,7 @@ func MakeChain(ctx *cli.Context, stack *node.Node, useExist bool) (chain *core.B
 	} else if config.IBFT != nil {
 		ibftConfig := setBFTConfig(config.IBFT.BFTConfig)
 		ibftConfig.TestQBFTBlock = nil
-		if config.Transitions != nil && len(config.Transitions) != 0 {
-			ibftConfig.Transitions = config.Transitions
-		}
+		ibftConfig.Transitions = config.Transitions
 		ibftConfig.Client = ethclient.NewClient(client)
 		engine = istanbulBackend.New(ibftConfig, stack.GetNodeKey(), chainDb)
 	} else if config.QBFT != nil {
@@ -2558,12 +2539,10 @@ func MakeChain(ctx *cli.Context, stack *node.Node, useExist bool) (chain *core.B
 		qbftConfig.BeneficiaryList = config.QBFT.BeneficiaryList     // list mode
 		qbftConfig.MiningBeneficiary = config.QBFT.MiningBeneficiary // auto (undefined mode) and besu mode
 		qbftConfig.TestQBFTBlock = big.NewInt(0)
-		if config.Transitions != nil && len(config.Transitions) != 0 {
-			qbftConfig.Transitions = config.Transitions
-		}
-		if config.QBFT.ValidatorContractAddress != (common.Address{}) {
-			qbftConfig.ValidatorContract = config.QBFT.ValidatorContractAddress
-		}
+		qbftConfig.Transitions = config.Transitions
+		qbftConfig.ValidatorContract = config.QBFT.ValidatorContractAddress
+		qbftConfig.ValidatorSelectionMode = config.QBFT.ValidatorSelectionMode
+		qbftConfig.Validators = config.QBFT.Validators
 		qbftConfig.Client = ethclient.NewClient(client)
 		engine = istanbulBackend.New(qbftConfig, stack.GetNodeKey(), chainDb)
 	} else if config.IsQuorum {

consensus/istanbul/backend/engine.go

@@ -367,12 +367,10 @@ func (sb *Backend) snapshot(chain consensus.ChainHeaderReader, number uint64, ha
 			}
 
 			var validators []common.Address
-			targetBlockHeight := new(big.Int).SetUint64(number)
-			validatorContract := sb.config.GetValidatorContractAddress(targetBlockHeight)
-			if validatorContract != (common.Address{}) && sb.config.GetValidatorSelectionMode(targetBlockHeight) == params.ContractMode {
-				sb.logger.Info("Initialising snap with contract validators", "address", validatorContract, "client", sb.config.Client)
+			validatorContract := sb.config.GetValidatorContractAddress(big.NewInt(0))
+			if validatorContract != (common.Address{}) && sb.config.GetValidatorSelectionMode(big.NewInt(0)) == params.ContractMode {
 
-				validatorContractCaller, err := contract.NewValidatorContractInterfaceCaller(sb.config.GetValidatorContractAddress(targetBlockHeight), sb.config.Client)
+				validatorContractCaller, err := contract.NewValidatorContractInterfaceCaller(validatorContract, sb.config.Client)
 
 				if err != nil {
 					return nil, fmt.Errorf("invalid smart contract in genesis alloc: %w", err)
@@ -383,16 +381,24 @@ func (sb *Backend) snapshot(chain consensus.ChainHeaderReader, number uint64, ha
 					BlockNumber: big.NewInt(0),
 				}
 				validators, err = validatorContractCaller.GetValidators(&opts)
+				log.Trace("BFT: Initialising snap with contract validators", "address", validatorContract, "validators", validators)
 				if err != nil {
 					log.Error("BFT: invalid smart contract in genesis alloc", "err", err)
 					return nil, err
 				}
 			} else {
-				var err error
-				validators, err = sb.EngineForBlockNumber(big.NewInt(0)).ExtractGenesisValidators(genesis)
-				if err != nil {
-					sb.logger.Error("BFT: invalid genesis block", "err", err)
-					return nil, err
+				validatorsFromConfig := sb.config.GetValidatorsAt(big.NewInt(0))
+				if len(validatorsFromConfig) > 0 {
+					validators = validatorsFromConfig
+					log.Info("BFT: Initialising snap with config validators", "validators", validators)
+				} else {
+					var err error
+					validators, err = sb.EngineForBlockNumber(big.NewInt(0)).ExtractGenesisValidators(genesis)
+					log.Info("BFT: Initialising snap with extradata", "validators", validators)
+					if err != nil {
+						log.Error("BFT: invalid genesis block", "err", err)
+						return nil, err
+					}
 				}
 			}
 
@@ -459,6 +465,12 @@ func (sb *Backend) snapshot(chain consensus.ChainHeaderReader, number uint64, ha
 		sb.logger.Trace("Fetched validators from smart contract", "validators", validators)
 		valSet := validator.NewSet(validators, sb.config.ProposerPolicy)
 		snap.ValSet = valSet
+	} else if validatorsFromTransitions := sb.config.GetValidatorsAt(targetBlockHeight); len(validatorsFromTransitions) > 0 && sb.config.GetValidatorSelectionMode(targetBlockHeight) == params.BlockHeaderMode {
+		//Note! we only want to set this once at this block height. Subsequent blocks will be propagated with the same
+		// 		validator as they are copied into the block header on the next block. Then normal voting can take place
+		// 		again.
+		valSet := validator.NewSet(validatorsFromTransitions, sb.config.ProposerPolicy)
+		snap.ValSet = valSet
 	}
 
 	// If we've generated a new checkpoint snapshot, save to disk

consensus/istanbul/common/errors.go

@@ -92,4 +92,6 @@ var (
 	// ErrFailedDecodeMessageSet = errors.New("failed to decode message set")
 	// ErrInvalidSigner is returned when the message is signed by a validator different than message sender
 	ErrInvalidSigner = errors.New("message not signed by the sender")
+
+	ErrInvalidGenesis = errors.New("genesis must only specify single validator mode for block zero")
 )

consensus/istanbul/config.go

@@ -136,7 +136,9 @@ type Config struct {
 	BeneficiaryList        []common.Address      `toml:",omitempty"` // List of wallet addresses that have benefit at every new block (list mode)
 	MiningBeneficiary      *common.Address       `toml:",omitempty"` // Wallet address that benefits at every new block (besu mode)
 	Transitions            []params.Transition
-	ValidatorContract      common.Address
+	ValidatorContract      common.Address      `toml:",omitempty"`
+	Validators             []common.Address    `toml:",omitempty"`
+	ValidatorSelectionMode *string             `toml:",omitempty"`
 	Client                 bind.ContractCaller `toml:",omitempty"`
 }
 
@@ -211,6 +213,15 @@ func (c Config) GetConfig(blockNumber *big.Int) Config {
 		if transition.MiningBeneficiary != nil {
 			newConfig.MiningBeneficiary = transition.MiningBeneficiary
 		}
+		if transition.ValidatorSelectionMode != "" {
+			newConfig.ValidatorSelectionMode = &transition.ValidatorSelectionMode
+		}
+		if transition.ValidatorContractAddress != (common.Address{}) {
+			newConfig.ValidatorContract = transition.ValidatorContractAddress
+		}
+		if len(transition.Validators) > 0 {
+			newConfig.Validators = transition.Validators
+		}
 	})
 
 	return newConfig
@@ -236,6 +247,21 @@ func (c Config) GetValidatorSelectionMode(blockNumber *big.Int) string {
 	return mode
 }
 
+func (c Config) GetValidatorsAt(blockNumber *big.Int) []common.Address {
+	if blockNumber.Cmp(big.NewInt(0)) == 0 && len(c.Validators) > 0 {
+		return c.Validators
+	}
+
+	if blockNumber != nil && c.Transitions != nil {
+		for i := 0; i < len(c.Transitions) && c.Transitions[i].Block.Cmp(blockNumber) == 0; i++ {
+			return c.Transitions[i].Validators
+		}
+	}
+
+	//Note! empty means we will get the valset from previous block header which contains votes, validators etc
+	return []common.Address{}
+}
+
 func (c Config) Get2FPlus1Enabled(blockNumber *big.Int) bool {
 	twoFPlusOneEnabled := false
 	c.getTransitionValue(blockNumber, func(transition params.Transition) {

consensus/istanbul/qbft/engine/engine.go

@@ -336,17 +336,35 @@ func (e *Engine) Prepare(chain consensus.ChainHeaderReader, header *types.Header
 		header.Time = uint64(time.Now().Unix())
 	}
 
-	validatorContract := e.cfg.GetValidatorContractAddress(big.NewInt(0).SetUint64(number - 1))
-	if validatorContract != (common.Address{}) && e.cfg.GetValidatorSelectionMode(big.NewInt(0).SetUint64(number-1)) == params.ContractMode {
+	currentBlockNumber := big.NewInt(0).SetUint64(number - 1)
+	validatorContract := e.cfg.GetValidatorContractAddress(currentBlockNumber)
+	if validatorContract != (common.Address{}) && e.cfg.GetValidatorSelectionMode(currentBlockNumber) == params.ContractMode {
 		return ApplyHeaderQBFTExtra(
 			header,
 			WriteValidators([]common.Address{}),
 		)
 	} else {
+		for _, transition := range e.cfg.Transitions {
+			if transition.Block.Cmp(currentBlockNumber) == 0 && len(transition.Validators) > 0 {
+				toRemove := make([]istanbul.Validator, 0, validators.Size())
+				l := validators.List()
+				for i := range l {
+					toRemove = append(toRemove, l[i])
+				}
+				for i := range toRemove {
+					validators.RemoveValidator(toRemove[i].Address())
+				}
+				for i := range transition.Validators {
+					validators.AddValidator(transition.Validators[i])
+				}
+				break
+			}
+		}
+		validatorsList := validator.SortedAddresses(validators.List())
 		// add validators in snapshot to extraData's validators section
 		return ApplyHeaderQBFTExtra(
 			header,
-			WriteValidators(validator.SortedAddresses(validators.List())),
+			WriteValidators(validatorsList),
 		)
 	}
 }

eth/ethconfig/config.go

@@ -296,6 +296,9 @@ func CreateConsensusEngine(stack *node.Node, chainConfig *params.ChainConfig, co
 		if chainConfig.QBFT.ValidatorContractAddress != (common.Address{}) {
 			config.Istanbul.ValidatorContract = chainConfig.QBFT.ValidatorContractAddress
 		}
+		config.Istanbul.Validators = chainConfig.QBFT.Validators
+		config.Istanbul.ValidatorSelectionMode = chainConfig.QBFT.ValidatorSelectionMode
+
 		return istanbulBackend.New(&config.Istanbul, stack.GetNodeKey(), db)
 	}
 	// For Quorum, Raft run as a separate service, so

params/config.go

@@ -429,11 +429,12 @@ func (c IBFTConfig) String() string {
 
 type QBFTConfig struct {
 	*BFTConfig
-	BlockReward       *math.HexOrDecimal256 `json:"blockReward,omitempty"`       // Reward from start, works only on QBFT consensus protocol
-	BeneficiaryMode   *string               `json:"beneficiaryMode,omitempty"`   // Mode for setting the beneficiary, either: list, besu, validators (beneficiary list is the list of validators)
-	BeneficiaryList   []common.Address      `json:"beneficiaryList,omitempty"`   // List of wallet addresses that have benefit at every new block (list mode)
-	MiningBeneficiary *common.Address       `json:"miningBeneficiary,omitempty"` // Wallet address that benefits at every new block (besu mode)
-
+	BlockReward            *math.HexOrDecimal256 `json:"blockReward,omitempty"`            // Reward from start, works only on QBFT consensus protocol
+	BeneficiaryMode        *string               `json:"beneficiaryMode,omitempty"`        // Mode for setting the beneficiary, either: list, besu, validators (beneficiary list is the list of validators)
+	BeneficiaryList        []common.Address      `json:"beneficiaryList,omitempty"`        // List of wallet addresses that have benefit at every new block (list mode)
+	MiningBeneficiary      *common.Address       `json:"miningBeneficiary,omitempty"`      // Wallet address that benefits at every new block (besu mode)
+	ValidatorSelectionMode *string               `json:"validatorselectionmode,omitempty"` // Select model for validators
+	Validators             []common.Address      `json:"validators"`                       // Validators list
 }
 
 func (c QBFTConfig) String() string {
@@ -457,6 +458,7 @@ type Transition struct {
 	RequestTimeoutSeconds        uint64                `json:"requesttimeoutseconds,omitempty"`        // Minimum request timeout for each IBFT or QBFT round in milliseconds
 	ContractSizeLimit            uint64                `json:"contractsizelimit,omitempty"`            // Maximum smart contract code size
 	ValidatorContractAddress     common.Address        `json:"validatorcontractaddress"`               // Smart contract address for list of validators
+	Validators                   []common.Address      `json:"validators"`                             // List of validators
 	ValidatorSelectionMode       string                `json:"validatorselectionmode,omitempty"`       // Validator selection mode to switch to
 	EnhancedPermissioningEnabled *bool                 `json:"enhancedPermissioningEnabled,omitempty"` // aka QIP714Block
 	PrivacyEnhancementsEnabled   *bool                 `json:"privacyEnhancementsEnabled,omitempty"`   // privacy enhancements (mandatory party, private state validation)

params/config_test.go

@@ -332,10 +332,10 @@ func TestCheckTransitionsData(t *testing.T) {
 	var ibftTransitionsConfig, qbftTransitionsConfig, invalidTransition, invalidBlockOrder []Transition
 	var emptyBlockPeriodSeconds uint64 = 10
 
-	tranI0 := Transition{big.NewInt(0), IBFT, 30000, 5, nil, 10, 50, common.Address{}, "", nil, nil, nil, nil, 0, nil, 0, nil, nil, nil, nil}
-	tranQ5 := Transition{big.NewInt(5), QBFT, 30000, 5, &emptyBlockPeriodSeconds, 10, 50, common.Address{}, "", nil, nil, nil, nil, 0, nil, 0, nil, nil, nil, nil}
-	tranI10 := Transition{big.NewInt(10), IBFT, 30000, 5, nil, 10, 50, common.Address{}, "", nil, nil, nil, nil, 0, nil, 0, nil, nil, nil, nil}
-	tranQ8 := Transition{big.NewInt(8), QBFT, 30000, 5, &emptyBlockPeriodSeconds, 10, 50, common.Address{}, "", nil, nil, nil, nil, 0, nil, 0, nil, nil, nil, nil}
+	tranI0 := Transition{big.NewInt(0), IBFT, 30000, 5, nil, 10, 50, common.Address{}, nil, "", nil, nil, nil, nil, 0, nil, 0, nil, nil, nil, nil}
+	tranQ5 := Transition{big.NewInt(5), QBFT, 30000, 5, &emptyBlockPeriodSeconds, 10, 50, common.Address{}, nil, "", nil, nil, nil, nil, 0, nil, 0, nil, nil, nil, nil}
+	tranI10 := Transition{big.NewInt(10), IBFT, 30000, 5, nil, 10, 50, common.Address{}, nil, "", nil, nil, nil, nil, 0, nil, 0, nil, nil, nil, nil}
+	tranQ8 := Transition{big.NewInt(8), QBFT, 30000, 5, &emptyBlockPeriodSeconds, 10, 50, common.Address{}, nil, "", nil, nil, nil, nil, 0, nil, 0, nil, nil, nil, nil}
 
 	ibftTransitionsConfig = append(ibftTransitionsConfig, tranI0, tranI10)
 	qbftTransitionsConfig = append(qbftTransitionsConfig, tranQ5, tranQ8)
@@ -395,7 +395,7 @@ func TestCheckTransitionsData(t *testing.T) {
 			wantErr: ErrBlockOrder,
 		},
 		{
-			stored:  &ChainConfig{Transitions: []Transition{{nil, IBFT, 30000, 5, &emptyBlockPeriodSeconds, 10, 50, common.Address{}, "", nil, nil, nil, nil, 0, nil, 0, nil, nil, nil, nil}}},
+			stored:  &ChainConfig{Transitions: []Transition{{nil, IBFT, 30000, 5, &emptyBlockPeriodSeconds, 10, 50, common.Address{}, nil, "", nil, nil, nil, nil, 0, nil, 0, nil, nil, nil, nil}}},
 			wantErr: ErrBlockNumberMissing,
 		},
 		{