Skip to content

Commit b000e13

Browse files
Lloyd-PottigerLloyd-Pottiger
committed
cluster: support tikv-worker
1 parent 7d025cb commit b000e13

File tree

12 files changed

+418
-12
lines changed

12 files changed

+418
-12
lines changed

‎components/cluster/command/template.go‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,7 @@ type LocalTemplate struct {
4343
PDServers []string // pd_servers in yaml template
4444
TiDBServers []string // tidb_servers in yaml template
4545
TiKVServers []string // tikv_servers in yaml template
46+
TiKVWorkerServers []string // tikv_worker_servers in yaml template
4647
TiFlashServers []string // tiflash_servers in yaml template
4748
MonitoringServers []string // monitoring_servers in yaml template
4849
GrafanaServers []string // grafana_servers in yaml template
@@ -131,6 +132,7 @@ func newTemplateCmd() *cobra.Command {
131132
cmd.Flags().StringSliceVar(&localOpt.PDServers, "pd-servers", []string{"127.0.0.1"}, "List of PD servers")
132133
cmd.Flags().StringSliceVar(&localOpt.TiDBServers, "tidb-servers", []string{"127.0.0.1"}, "List of TiDB servers")
133134
cmd.Flags().StringSliceVar(&localOpt.TiKVServers, "tikv-servers", []string{"127.0.0.1"}, "List of TiKV servers")
135+
cmd.Flags().StringSliceVar(&localOpt.TiKVWorkerServers, "tikv-worker-servers", nil, "List of TiKV worker servers")
134136
cmd.Flags().StringSliceVar(&localOpt.TiFlashServers, "tiflash-servers", nil, "List of TiFlash servers")
135137
cmd.Flags().StringSliceVar(&localOpt.MonitoringServers, "monitoring-servers", []string{"127.0.0.1"}, "List of monitor servers")
136138
cmd.Flags().StringSliceVar(&localOpt.GrafanaServers, "grafana-servers", []string{"127.0.0.1"}, "List of grafana servers")

‎embed/examples/cluster/local.tpl‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,12 @@ tikv_servers:
4343
- host: {{ . }}
4444
{{- end }}
4545
{{ end }}
46+
{{ if .TiKVWorkerServers -}}
47+
tikv_worker_servers:
48+
{{- range .TiKVWorkerServers }}
49+
- host: {{ . }}
50+
{{- end }}
51+
{{ end }}
4652
{{- if .TiFlashServers }}
4753
tiflash_servers:
4854
{{- range .TiFlashServers }}

‎embed/templates/config/prometheus.yml.tpl‎

Lines changed: 26 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -145,6 +145,23 @@ scrape_configs:
145145
- targets:
146146
{{- range .TiKVStatusAddrs}}
147147
- '{{.}}'
148+
{{- end}}
149+
{{- if .TiKVWorkerAddrs}}
150+
- job_name: "tikv-worker"
151+
honor_labels: true # don't overwrite job & instance labels
152+
{{- if .TLSEnabled}}
153+
scheme: https
154+
tls_config:
155+
insecure_skip_verify: false
156+
ca_file: ../tls/ca.crt
157+
cert_file: ../tls/prometheus.crt
158+
key_file: ../tls/prometheus.pem
159+
{{- end}}
160+
static_configs:
161+
- targets:
162+
{{- range .TiKVWorkerAddrs}}
163+
- '{{.}}'
164+
{{- end}}
148165
{{- end}}
149166
- job_name: "pd"
150167
honor_labels: true # don't overwrite job & instance labels
@@ -377,6 +394,14 @@ scrape_configs:
377394
{{- end}}
378395
labels:
379396
group: 'tikv'
397+
{{- if .TiKVWorkerAddrs}}
398+
- targets:
399+
{{- range .TiKVWorkerAddrs}}
400+
- '{{.}}'
401+
{{- end}}
402+
labels:
403+
group: 'tikv-worker'
404+
{{- end}}
380405
- targets:
381406
{{- range .PDAddrs}}
382407
- '{{.}}'
@@ -511,4 +536,4 @@ scrape_configs:
511536

512537
{{- if .RemoteConfig}}
513538
{{.RemoteConfig}}
514-
{{- end}}
539+
{{- end}}

‎embed/templates/scripts/run_tikv-worker.sh.tpl‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
#!/bin/bash
2+
set -e
3+
4+
# WARNING: This file was auto-generated. Do not edit!
5+
# All your edit might be overwritten!
6+
cd "{{.DeployDir}}" || exit 1
7+
8+
{{- if and .NumaNode .NumaCores}}
9+
exec numactl --cpunodebind={{.NumaNode}} --membind={{.NumaNode}} -C {{.NumaCores}} bin/tikv-worker \
10+
{{- else if .NumaNode}}
11+
exec numactl --cpunodebind={{.NumaNode}} --membind={{.NumaNode}} bin/tikv-worker \
12+
{{- else}}
13+
exec bin/tikv-worker \
14+
{{- end}}
15+
--addr "{{.Addr}}" \
16+
--pd-endpoints "{{.PD}}" \
17+
--config conf/tikv-worker.toml \
18+
--log-file "{{.LogDir}}/tikv_worker.log" 2>> "{{.LogDir}}/tikv_worker_stderr.log"
19+

‎pkg/cluster/spec/monitoring.go‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -360,6 +360,13 @@ func (i *MonitorInstance) InitConfig(
360360
cfig.AddTiKV(kv.Host, uint64(kv.StatusPort))
361361
}
362362
}
363+
if servers, found := topoHasField("TiKVWorkerServers"); found {
364+
for i := 0; i < servers.Len(); i++ {
365+
worker := servers.Index(i).Interface().(*TiKVWorkerSpec)
366+
uniqueHosts.Insert(worker.Host)
367+
cfig.AddTiKVWorker(worker.Host, uint64(worker.Port))
368+
}
369+
}
363370
if servers, found := topoHasField("TiDBServers"); found {
364371
for i := 0; i < servers.Len(); i++ {
365372
db := servers.Index(i).Interface().(*TiDBSpec)

‎pkg/cluster/spec/spec.go‎

Lines changed: 19 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -120,6 +120,7 @@ type (
120120
ServerConfigs struct {
121121
TiDB map[string]any `yaml:"tidb"`
122122
TiKV map[string]any `yaml:"tikv"`
123+
TiKVWorker map[string]any `yaml:"tikv_worker"`
123124
PD map[string]any `yaml:"pd"`
124125
TSO map[string]any `yaml:"tso"`
125126
Scheduling map[string]any `yaml:"scheduling"`
@@ -140,6 +141,7 @@ type (
140141
ComponentVersions struct {
141142
TiDB string `yaml:"tidb,omitempty"`
142143
TiKV string `yaml:"tikv,omitempty"`
144+
TiKVWorker string `yaml:"tikv_worker,omitempty"`
143145
TiFlash string `yaml:"tiflash,omitempty"`
144146
PD string `yaml:"pd,omitempty"`
145147
TSO string `yaml:"tso,omitempty"`
@@ -162,15 +164,16 @@ type (
162164

163165
// ComponentSources represents the source of components
164166
ComponentSources struct {
165-
TiDB string `yaml:"tidb,omitempty" validate:"tidb:editable"`
166-
TiKV string `yaml:"tikv,omitempty" validate:"tikv:editable"`
167-
TiFlash string `yaml:"tiflash,omitempty" validate:"tiflash:editable"`
168-
PD string `yaml:"pd,omitempty" validate:"pd:editable"`
169-
Dashboard string `yaml:"tidb_dashboard,omitempty" validate:"tidb_dashboard:editable"`
170-
Pump string `yaml:"pump,omitempty" validate:"pump:editable"`
171-
Drainer string `yaml:"drainer,omitempty" validate:"drainer:editable"`
172-
CDC string `yaml:"cdc,omitempty" validate:"cdc:editable"`
173-
TiKVCDC string `yaml:"kvcdc,omitempty" validate:"kvcdc:editable"`
167+
TiDB string `yaml:"tidb,omitempty" validate:"tidb:editable"`
168+
TiKV string `yaml:"tikv,omitempty" validate:"tikv:editable"`
169+
TiKVWorker string `yaml:"tikv_worker,omitempty" validate:"tikv_worker:editable"`
170+
TiFlash string `yaml:"tiflash,omitempty" validate:"tiflash:editable"`
171+
PD string `yaml:"pd,omitempty" validate:"pd:editable"`
172+
Dashboard string `yaml:"tidb_dashboard,omitempty" validate:"tidb_dashboard:editable"`
173+
Pump string `yaml:"pump,omitempty" validate:"pump:editable"`
174+
Drainer string `yaml:"drainer,omitempty" validate:"drainer:editable"`
175+
CDC string `yaml:"cdc,omitempty" validate:"cdc:editable"`
176+
TiKVCDC string `yaml:"kvcdc,omitempty" validate:"kvcdc:editable"`
174177
}
175178

176179
// Specification represents the specification of topology.yaml
@@ -182,6 +185,7 @@ type (
182185
ServerConfigs ServerConfigs `yaml:"server_configs,omitempty" validate:"server_configs:ignore"`
183186
TiDBServers []*TiDBSpec `yaml:"tidb_servers"`
184187
TiKVServers []*TiKVSpec `yaml:"tikv_servers"`
188+
TiKVWorkerServers []*TiKVWorkerSpec `yaml:"tikv_worker_servers,omitempty"`
185189
TiFlashServers []*TiFlashSpec `yaml:"tiflash_servers"`
186190
TiProxyServers []*TiProxySpec `yaml:"tiproxy_servers"`
187191
PDServers []*PDSpec `yaml:"pd_servers"`
@@ -570,6 +574,7 @@ func (s *Specification) Merge(that Topology) Topology {
570574
ComponentVersions: s.ComponentVersions.Merge(spec.ComponentVersions),
571575
TiDBServers: append(s.TiDBServers, spec.TiDBServers...),
572576
TiKVServers: append(s.TiKVServers, spec.TiKVServers...),
577+
TiKVWorkerServers: append(s.TiKVWorkerServers, spec.TiKVWorkerServers...),
573578
PDServers: append(s.PDServers, spec.PDServers...),
574579
DashboardServers: append(s.DashboardServers, spec.DashboardServers...),
575580
TiFlashServers: append(s.TiFlashServers, spec.TiFlashServers...),
@@ -595,6 +600,7 @@ func (v *ComponentVersions) Merge(that ComponentVersions) ComponentVersions {
595600
return ComponentVersions{
596601
TiDB: utils.Ternary(that.TiDB != "", that.TiDB, v.TiDB).(string),
597602
TiKV: utils.Ternary(that.TiKV != "", that.TiKV, v.TiKV).(string),
603+
TiKVWorker: utils.Ternary(that.TiKVWorker != "", that.TiKVWorker, v.TiKVWorker).(string),
598604
PD: utils.Ternary(that.PD != "", that.PD, v.PD).(string),
599605
TSO: utils.Ternary(that.TSO != "", that.TSO, v.TSO).(string),
600606
Scheduling: utils.Ternary(that.Scheduling != "", that.Scheduling, v.Scheduling).(string),
@@ -812,7 +818,7 @@ func (s *Specification) ComponentsByStopOrder() (comps []Component) {
812818

813819
// ComponentsByStartOrder return component in the order need to start.
814820
func (s *Specification) ComponentsByStartOrder() (comps []Component) {
815-
// "pd", "tso", "scheduling", "resource-manager", "dashboard", "tiproxy", "tikv", "pump", "tidb", "tiflash", "drainer", "cdc", "tikv-cdc", "prometheus", "grafana", "alertmanager"
821+
// "pd", "tso", "scheduling", "resource-manager", "dashboard", "tiproxy", "tikv", "tikv-worker", "pump", "tidb", "tiflash", "drainer", "cdc", "tikv-cdc", "prometheus", "grafana", "alertmanager"
816822
comps = append(comps, &PDComponent{s})
817823
comps = append(comps, &TSOComponent{s})
818824
comps = append(comps, &SchedulingComponent{s})
@@ -821,6 +827,7 @@ func (s *Specification) ComponentsByStartOrder() (comps []Component) {
821827
comps = append(comps, &DashboardComponent{s})
822828
comps = append(comps, &TiProxyComponent{s})
823829
comps = append(comps, &TiKVComponent{s})
830+
comps = append(comps, &TiKVWorkerComponent{s})
824831
comps = append(comps, &PumpComponent{s})
825832
comps = append(comps, &TiDBComponent{s})
826833
comps = append(comps, &TiFlashComponent{s})
@@ -840,7 +847,7 @@ func (s *Specification) ComponentsByUpdateOrder(curVer string) (comps []Componen
840847
// Ref: https://github.com/pingcap/tiup/issues/2166
841848
cdcUpgradeBeforePDTiKVTiDB := tidbver.TiCDCUpgradeBeforePDTiKVTiDB(curVer)
842849

843-
// "tiflash", <"cdc">, "pd", "tso", "scheduling", "resource-manager","router", "dashboard", "tiproxy", "tikv", "pump", "tidb", "drainer", <"cdc>", "prometheus", "grafana", "alertmanager"
850+
// "tiflash", <"cdc">, "pd", "tso", "scheduling", "resource-manager","router", "dashboard", "tiproxy", "tikv", "tikv-worker", "pump", "tidb", "drainer", <"cdc>", "prometheus", "grafana", "alertmanager"
844851
comps = append(comps, &TiFlashComponent{s})
845852
if cdcUpgradeBeforePDTiKVTiDB {
846853
comps = append(comps, &CDCComponent{s})
@@ -853,6 +860,7 @@ func (s *Specification) ComponentsByUpdateOrder(curVer string) (comps []Componen
853860
comps = append(comps, &DashboardComponent{s})
854861
comps = append(comps, &TiProxyComponent{s})
855862
comps = append(comps, &TiKVComponent{s})
863+
comps = append(comps, &TiKVWorkerComponent{s})
856864
comps = append(comps, &PumpComponent{s})
857865
comps = append(comps, &TiDBComponent{s})
858866
comps = append(comps, &DrainerComponent{s})

‎pkg/cluster/spec/spec_test.go‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@ func TestDefaultDataDir(t *testing.T) {
3232
// Test with without global DataDir.
3333
topo := new(Specification)
3434
topo.TiKVServers = append(topo.TiKVServers, &TiKVSpec{Host: "1.1.1.1", Port: 22})
35+
topo.TiKVWorkerServers = append(topo.TiKVWorkerServers, &TiKVWorkerSpec{Host: "1.1.1.2", Port: 22})
3536
topo.CDCServers = append(topo.CDCServers, &CDCSpec{Host: "2.3.3.3", Port: 22})
3637
topo.TiKVCDCServers = append(topo.TiKVCDCServers, &TiKVCDCSpec{Host: "3.3.3.3", Port: 22})
3738
data, err := yaml.Marshal(topo)
@@ -43,6 +44,7 @@ func TestDefaultDataDir(t *testing.T) {
4344
require.NoError(t, err)
4445
require.Equal(t, "data", topo.GlobalOptions.DataDir)
4546
require.Equal(t, "data", topo.TiKVServers[0].DataDir)
47+
require.Equal(t, "data", topo.TiKVWorkerServers[0].DataDir)
4648
require.Equal(t, "data", topo.CDCServers[0].DataDir)
4749
require.Equal(t, "data", topo.TiKVCDCServers[0].DataDir)
4850

@@ -54,6 +56,7 @@ func TestDefaultDataDir(t *testing.T) {
5456
require.NoError(t, err)
5557
require.Equal(t, "data", topo.GlobalOptions.DataDir)
5658
require.Equal(t, "data", topo.TiKVServers[0].DataDir)
59+
require.Equal(t, "data", topo.TiKVWorkerServers[0].DataDir)
5760
require.Equal(t, "data", topo.CDCServers[0].DataDir)
5861
require.Equal(t, "data", topo.TiKVCDCServers[0].DataDir)
5962

@@ -62,6 +65,8 @@ func TestDefaultDataDir(t *testing.T) {
6265
topo.GlobalOptions.DataDir = "/global_data"
6366
topo.TiKVServers = append(topo.TiKVServers, &TiKVSpec{Host: "1.1.1.1", Port: 22})
6467
topo.TiKVServers = append(topo.TiKVServers, &TiKVSpec{Host: "1.1.1.2", Port: 33, DataDir: "/my_data"})
68+
topo.TiKVWorkerServers = append(topo.TiKVWorkerServers, &TiKVWorkerSpec{Host: "1.1.1.3", Port: 22})
69+
topo.TiKVWorkerServers = append(topo.TiKVWorkerServers, &TiKVWorkerSpec{Host: "1.1.1.4", Port: 33, DataDir: "/my_worker_data"})
6570
topo.CDCServers = append(topo.CDCServers, &CDCSpec{Host: "2.3.3.3", Port: 22})
6671
topo.CDCServers = append(topo.CDCServers, &CDCSpec{Host: "2.3.3.4", Port: 22, DataDir: "/cdc_data"})
6772
topo.TiKVCDCServers = append(topo.TiKVCDCServers, &TiKVCDCSpec{Host: "3.3.3.3", Port: 22})
@@ -76,6 +81,9 @@ func TestDefaultDataDir(t *testing.T) {
7681
require.Equal(t, "/global_data/tikv-22", topo.TiKVServers[0].DataDir)
7782
require.Equal(t, "/my_data", topo.TiKVServers[1].DataDir)
7883

84+
require.Equal(t, "/global_data/tikv-worker-22", topo.TiKVWorkerServers[0].DataDir)
85+
require.Equal(t, "/my_worker_data", topo.TiKVWorkerServers[1].DataDir)
86+
7987
require.Equal(t, "/global_data/cdc-22", topo.CDCServers[0].DataDir)
8088
require.Equal(t, "/cdc_data", topo.CDCServers[1].DataDir)
8189

0 commit comments

Comments
 (0)