playground: wait PD ready before starting TiDB to avoid cluster ID mismatch

Author: amyangfei

components/playground/instance/pd.go

@@ -15,9 +15,12 @@ package instance
 
 import (
 	"context"
+	"encoding/json"
 	"fmt"
+	"net/http"
 	"path/filepath"
 	"strings"
+	"time"
 
 	"github.com/pingcap/errors"
 	"github.com/pingcap/tiup/pkg/tidbver"
@@ -236,3 +239,36 @@ func (inst *PDInstance) LogFile() string {
 func (inst *PDInstance) Addr() string {
 	return utils.JoinHostPort(AdvertiseHost(inst.Host), inst.StatusPort)
 }
+
+// Ready return nil when PD is ready to serve.
+func (inst *PDInstance) Ready(ctx context.Context) error {
+	url := fmt.Sprintf("http://%s/pd/api/v1/members", inst.Addr())
+
+	var r struct {
+		Header struct {
+			ClusterID uint64 `json:"cluster_id"`
+		} `json:"header"`
+	}
+
+	for {
+		resp, err := http.Get(url)
+		if err == nil {
+			if resp.StatusCode == 200 {
+				err = json.NewDecoder(resp.Body).Decode(&r)
+				_ = resp.Body.Close()
+				if err == nil && r.Header.ClusterID != 0 {
+					return nil
+				}
+			} else {
+				_ = resp.Body.Close()
+			}
+		}
+
+		select {
+		case <-ctx.Done():
+			return ctx.Err()
+		case <-time.After(time.Second):
+			// retry
+		}
+	}
+}

components/playground/playground.go

@@ -1285,6 +1285,7 @@ func (p *Playground) bootCluster(ctx context.Context, env *environment.Environme
 
 	anyPumpReady := false
 	allDMMasterReady := false
+	allPDReady := false
 	// Start all instance except tiflash.
 	err := p.WalkInstances(func(cid string, ins instance.Instance) error {
 		if cid == spec.ComponentTiFlash {
@@ -1296,6 +1297,19 @@ func (p *Playground) bootCluster(ctx context.Context, env *environment.Environme
 			allDMMasterReady = true
 		}
 
+		// wait all PD ready before starting TiDB to avoid cluster ID mismatch
+		if cid == spec.ComponentTiDB && !allPDReady {
+			for _, pd := range p.pds {
+				pdCtx, cancel := context.WithTimeout(ctx, time.Second*120)
+				err := pd.Ready(pdCtx)
+				cancel()
+				if err != nil {
+					return errors.Annotatef(err, "failed to wait PD %s to be ready", pd.Addr())
+				}
+			}
+			allPDReady = true
+		}
+
 		err := p.startInstance(ctx, ins)
 		if err != nil {
 			return err