add local and remote mode for patroni attack

Author: Nikita Savchenko

cmd/attack/attack.go

@@ -42,6 +42,7 @@ func NewAttackCommand() *cobra.Command {
 		NewHTTPAttackCommand(&uid),
 		NewVMAttackCommand(&uid),
 		NewUserDefinedCommand(&uid),
+		NewPatroniAttackCommand(&uid),
 	)
 
 	return cmd

cmd/attack/patroni.go

@@ -48,37 +48,39 @@ func NewPatroniAttackCommand(uid *string) *cobra.Command {
 
 	cmd.PersistentFlags().StringVarP(&options.User, "user", "u", "patroni", "patroni cluster user")
 	cmd.PersistentFlags().StringVar(&options.Password, "password", "p", "patroni cluster password")
+	cmd.PersistentFlags().StringVarP(&options.Address, "address", "a", "", "patroni cluster address, any of available hosts")
+	cmd.PersistentFlags().BoolVarP(&options.LocalMode, "local-mode", "l", false, "execute patronictl on host with chaosd. User with privileges required.")
+	cmd.PersistentFlags().BoolVarP(&options.RemoteMode, "remote-mode", "r", false, "execute patroni command by REST API")
 
 	return cmd
 }
 
 func NewPatroniSwitchoverCommand(dep fx.Option, options *core.PatroniCommand) *cobra.Command {
 	cmd := &cobra.Command{
 		Use:   "switchover",
-		Short: "exec switchover, default without another attack. Warning! Command is not recover!",
+		Short: "exec switchover command. Warning! Command is not recover!",
 		Run: func(*cobra.Command, []string) {
 			options.Action = core.SwitchoverAction
 			utils.FxNewAppWithoutLog(dep, fx.Invoke(PatroniAttackF)).Run()
 		},
 	}
-	cmd.Flags().StringVarP(&options.Address, "address", "a", "", "patroni cluster address, any of available hosts")
 	cmd.Flags().StringVarP(&options.Candidate, "candidate", "c", "", "switchover candidate, default random unit for replicas")
-	cmd.Flags().StringVarP(&options.Scheduled_at, "scheduled_at", "d", fmt.Sprintln(time.Now().Add(time.Second*60).Format(time.RFC3339)), "scheduled switchover, default now()+1 minute")
+	cmd.Flags().StringVarP(&options.Scheduled_at, "scheduled_at", "d", fmt.Sprint(time.Now().Add(time.Second*60).Format(time.RFC3339)), `scheduled switchover, 
+	default now()+1 minute by remote mode`)
 
 	return cmd
 }
 
 func NewPatroniFailoverCommand(dep fx.Option, options *core.PatroniCommand) *cobra.Command {
 	cmd := &cobra.Command{
 		Use:   "failover",
-		Short: "exec failover, default without another attack",
+		Short: "Exec failover command. Warning! Command is not recover!",
 		Run: func(*cobra.Command, []string) {
 			options.Action = core.FailoverAction
 			utils.FxNewAppWithoutLog(dep, fx.Invoke(PatroniAttackF)).Run()
 		},
 	}
 
-	cmd.Flags().StringVarP(&options.Address, "address", "a", "", "patroni cluster address, any of available hosts")
 	cmd.Flags().StringVarP(&options.Candidate, "leader", "c", "", "failover new leader, default random unit for replicas")
 	return cmd
 }

pkg/core/patroni.go

@@ -35,6 +35,8 @@ type PatroniCommand struct {
 	User         string `json:"user,omitempty"`
 	Password     string `json:"password,omitempty"`
 	Scheduled_at string `json:"scheduled_at,omitempty"`
+	LocalMode    bool   `json:"local_mode,omitempty"`
+	RemoteMode   bool   `json:"remote_mode,omitempty"`
 	RecoverCmd   string `json:"recoverCmd,omitempty"`
 }
 
@@ -46,12 +48,18 @@ func (p *PatroniCommand) Validate() error {
 		return errors.New("address not provided")
 	}
 
-	if len(p.User) == 0 {
-		return errors.New("patroni user not provided")
+	if !p.RemoteMode && !p.LocalMode {
+		return errors.New("local or remote mode required")
 	}
 
-	if len(p.Password) == 0 {
-		return errors.New("patroni password not provided")
+	if p.RemoteMode {
+		if len(p.User) == 0 {
+			return errors.New("patroni user not provided")
+		}
+
+		if len(p.Password) == 0 {
+			return errors.New("patroni password not provided")
+		}
 	}
 
 	return nil

pkg/server/chaosd/patroni.go

@@ -14,12 +14,12 @@
 package chaosd
 
 import (
-	"bytes"
 	"encoding/json"
 	"fmt"
 	"io"
 	"math/rand"
-	"net/http"
+	"os/exec"
+	"strings"
 
 	"github.com/chaos-mesh/chaosd/pkg/core"
 	"github.com/chaos-mesh/chaosd/pkg/server/utils"
@@ -34,17 +34,19 @@ var PatroniAttack AttackType = patroniAttack{}
 func (patroniAttack) Attack(options core.AttackConfig, _ Environment) error {
 	attack := options.(*core.PatroniCommand)
 
-	candidate := attack.Candidate
+	var responce []byte
 
-	leader := attack.Leader
-
-	var scheduled_at string
-
-	var url string
+	var address string
 
 	values := make(map[string]string)
 
-	patroniInfo, err := utils.GetPatroniInfo(attack.Address)
+	if attack.RemoteMode {
+		address = attack.Address
+	} else if attack.LocalMode {
+		address = "localhost"
+	}
+
+	patroniInfo, err := utils.GetPatroniInfo(address)
 	if err != nil {
 		err = errors.Errorf("failed to get patroni info for %v: %v", options.String(), err)
 		return errors.WithStack(err)
@@ -55,75 +57,109 @@ func (patroniAttack) Attack(options core.AttackConfig, _ Environment) error {
 		return errors.WithStack(err)
 	}
 
-	if candidate == "" {
-		candidate = patroniInfo.Replicas[rand.Intn(len(patroniInfo.Replicas))]
+	if attack.Candidate == "" {
+		values["candidate"] = patroniInfo.Replicas[rand.Intn(len(patroniInfo.Replicas))]
 	}
 
-	if leader == "" {
-		leader = patroniInfo.Master
+	if attack.Leader == "" {
+		values["leader"] = patroniInfo.Master
 	}
 
-	switch options.String() {
-	case "switchover":
+	values["scheduled_at"] = attack.Scheduled_at
 
-		scheduled_at = attack.Scheduled_at
+	cmd := options.String()
 
-		values = map[string]string{"leader": leader, "scheduled_at": scheduled_at}
+	switch cmd {
+	case "switchover":
 
-		log.Info(fmt.Sprintf("Switchover will be done from %v to another available replica in %v", patroniInfo.Master, scheduled_at))
+		log.Info(fmt.Sprintf("Switchover will be done from %v to %v in %v", values["leader"], values["candidate"], values["scheduled_at"]))
 
 	case "failover":
 
-		values = map[string]string{"candidate": candidate}
+		log.Info(fmt.Sprintf("Failover will be done from %v to %v", values["leader"], values["candidate"]))
 
-		log.Info(fmt.Sprintf("Failover will be done from %v to %v", patroniInfo.Master, candidate))
+	}
 
+	if attack.RemoteMode {
+		responce, err = execPatroniAttackByRemoteMode(attack, cmd, values)
+		if err != nil {
+			return err
+		}
+	} else if attack.LocalMode {
+		responce, err = execPatroniAttackByLocalMode(attack, cmd, values)
+		if err != nil {
+			return err
+		}
 	}
 
-	patroniAddr := attack.Address
+	if attack.RemoteMode {
+		log.S().Infof("Execute %v successfully: %v", cmd, string(responce))
+	}
 
-	cmd := options.String()
+	if attack.LocalMode {
+		log.S().Infof("Execute %v successfully", cmd)
+		fmt.Println(string(responce))
+	}
+
+	return nil
+}
+
+func execPatroniAttackByRemoteMode(attack *core.PatroniCommand, cmd string, values map[string]string) ([]byte, error) {
+	patroniAddr := attack.Address
 
 	data, err := json.Marshal(values)
 	if err != nil {
 		err = errors.Errorf("failed to marshal data: %v", values)
-		return errors.WithStack(err)
+		return nil, errors.WithStack(err)
 	}
 
-	url = fmt.Sprintf("http://%v:8008/%v", patroniAddr, cmd)
-
-	request, err := http.NewRequest("POST", url, bytes.NewBuffer(data))
+	resp, err := utils.MakePostHTTPRequest(patroniAddr, 8008, cmd, data, attack.User, attack.Password)
 	if err != nil {
-		err = errors.Errorf("failed to %v: %v", cmd, err)
-		return errors.WithStack(err)
+		return nil, errors.WithStack(err)
 	}
 
-	request.Header.Set("Content-Type", "application/json")
-	request.SetBasicAuth(attack.User, attack.Password)
-
-	client := &http.Client{}
-	resp, error := client.Do(request)
-	if error != nil {
-		err = errors.Errorf("failed to %v: %v", cmd, err)
-		return errors.WithStack(err)
+	if resp.StatusCode != 200 && resp.StatusCode != 202 {
+		//to simplify diagnostics
+		buf, err := io.ReadAll(resp.Body)
+		if err != nil {
+			err = errors.Errorf("failed to read from %s responce: status code %v, responce %v, error %v", cmd, resp.StatusCode, resp.Body, err)
+			return nil, err
+		}
+		err = errors.Errorf("failed to exec %v request: status code %v, responce %v", cmd, resp.StatusCode, buf)
+		return nil, errors.WithStack(err)
 	}
 
-	defer resp.Body.Close()
-
 	buf, err := io.ReadAll(resp.Body)
 	if err != nil {
-		err = errors.Errorf("failed to read %v responce: %v", cmd, err)
-		return errors.WithStack(err)
+		err = errors.Errorf("failed to read %v from %s responce: %v", resp.Body, cmd, err)
+		return nil, errors.WithStack(err)
 	}
 
-	if resp.StatusCode != 200 && resp.StatusCode != 202 {
-		err = errors.Errorf("failed to %v: status code %v, responce %v", cmd, resp.StatusCode, string(buf))
-		return errors.WithStack(err)
+	return buf, nil
+}
+
+func execPatroniAttackByLocalMode(attack *core.PatroniCommand, cmd string, values map[string]string) ([]byte, error) {
+	var cmdTemplate string
+
+	if cmd == "failover" {
+		cmdTemplate = fmt.Sprintf("patronictl %v --master %v --candidate %v --force", cmd, values["leader"], values["candidate"])
+	} else if cmd == "switchover" {
+		cmdTemplate = fmt.Sprintf("patronictl %v --master %v --candidate %v --scheduled %v --force", cmd, values["leader"], values["candidate"], values["scheduled_at"])
+	}
+
+	execCmd := exec.Command("bash", "-c", cmdTemplate)
+	output, err := execCmd.CombinedOutput()
+	if err != nil {
+		log.S().Errorf(fmt.Sprintf("failed to %v: %v", cmdTemplate, string(output)))
+		return nil, err
 	}
 
-	log.S().Infof("Execute %v successfully: %v", cmd, string(buf))
+	if strings.Contains(string(output), "failed") {
+		err = errors.New(string(output))
+		return nil, err
+	}
 
-	return nil
+	return output, nil
 }
 
 func (patroniAttack) Recover(exp core.Experiment, _ Environment) error {

pkg/server/utils/status.go renamed to pkg/server/utils/http_requests.go

@@ -10,9 +10,11 @@
 // distributed under the License is distributed on an "AS IS" BASIS,
 // See the License for the specific language governing permissions and
 // limitations under the License.
+
 package utils
 
 import (
+	"bytes"
 	"fmt"
 	"io"
 	"net/http"
@@ -23,9 +25,10 @@ import (
 )
 
 type PatroniInfo struct {
-	Master   string
-	Replicas []string
-	Status   []string
+	Master      string
+	Replicas    []string
+	SyncStandby []string
+	Status      []string
 }
 
 func GetPatroniInfo(address string) (PatroniInfo, error) {
@@ -50,17 +53,47 @@ func GetPatroniInfo(address string) (PatroniInfo, error) {
 	members := gjson.Get(data, "members")
 
 	for _, member := range members.Array() {
-		if member.Get("role").Str == "leader" {
+		switch member.Get("role").Str {
+		case "leader":
 			patroniInfo.Master = member.Get("name").Str
 			patroniInfo.Status = append(patroniInfo.Status, member.Get("state").Str)
-		} else if member.Get("role").Str == "replica" || member.Get("role").Str == "sync_standby" {
+		case "replica":
 			patroniInfo.Replicas = append(patroniInfo.Replicas, member.Get("name").Str)
 			patroniInfo.Status = append(patroniInfo.Status, member.Get("state").Str)
+		case "sync_standby":
+			patroniInfo.SyncStandby = append(patroniInfo.SyncStandby, member.Get("name").Str)
+			patroniInfo.Status = append(patroniInfo.Status, member.Get("state").Str)
+
 		}
 	}
 
-	log.Info(fmt.Sprintf("patroni info: master %v, replicas %v, statuses %v\n", patroniInfo.Master, patroniInfo.Replicas, patroniInfo.Status))
+	log.Info(fmt.Sprintf("patroni info: master %v, replicas %v, sync_standy %s, statuses %v\n", patroniInfo.Master, patroniInfo.Replicas,
+		patroniInfo.SyncStandby, patroniInfo.Status))
 
 	return patroniInfo, nil
 
 }
+
+func MakePostHTTPRequest(address string, port int64, path string, body []byte, user string, password string) (*http.Response, error) {
+	url := fmt.Sprintf("http://%v:%v/%v", address, port, path)
+
+	request, err := http.NewRequest("POST", url, bytes.NewBuffer(body))
+	if err != nil {
+		err = errors.Errorf("failed to request %v: %v", url, err)
+		return nil, errors.WithStack(err)
+	}
+
+	request.Header.Set("Content-Type", "application/json")
+	request.SetBasicAuth(user, password)
+
+	client := &http.Client{}
+	resp, error := client.Do(request)
+	if error != nil {
+		err = errors.Errorf("failed to exec request %v: %v", url, err)
+		return nil, errors.WithStack(err)
+	}
+
+	defer resp.Body.Close()
+
+	return resp, nil
+}