Skip to content

Commit 0fc6c28

Browse files
author
Junlong Gao
committed
Working on a solution
Also, added a basic raw kv interface demo and remove the scheduler part.
1 parent e55ab1a commit 0fc6c28

File tree

150 files changed

+3356
-57825
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

150 files changed

+3356
-57825
lines changed

.dockerignore

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
proto/_tools
2+
bin/*
3+
doc/*
4+
Dockerfile

.gitignore

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,3 +5,4 @@ unikv/unikv
55
bin/*
66
_tools
77
/vendor
8+
proto/pkg

Dockerfile

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
FROM golang:1.13.8
2+
3+
RUN apt-get update && apt-get install -y wget zip \
4+
&& wget https://github.com/protocolbuffers/protobuf/releases/download/v3.12.3/protoc-3.12.3-linux-x86_64.zip \
5+
&& unzip protoc-3.12.3-linux-x86_64.zip \
6+
&& cp bin/protoc /usr/local/bin
7+
8+
RUN mkdir -p /src
9+
COPY kv /src/kv
10+
COPY log /src/log
11+
COPY proto /src/proto
12+
COPY raft /src/raft
13+
COPY go.mod /src/go.mod
14+
COPY go.sum /src/go.sum
15+
COPY Makefile /src/Makefile
16+
COPY scheduler /src/scheduler
17+
18+
WORKDIR /src/
19+
ENTRYPOINT [ "make", "test" ]

Makefile

Lines changed: 16 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -9,24 +9,15 @@ endif
99

1010
GO := GO111MODULE=on go
1111
GOBUILD := $(GO) build $(BUILD_FLAG) -tags codes
12-
GOTEST := $(GO) test -v --count=1 --parallel=1 -p=1
12+
GOTEST := $(GO) test -v --count=1 --parallel=1 -p=1 -failfast -timeout 600m
1313

1414
TEST_LDFLAGS := ""
1515

1616
PACKAGE_LIST := go list ./...| grep -vE "cmd"
1717
PACKAGES := $$($(PACKAGE_LIST))
1818

1919
# Targets
20-
.PHONY: clean test proto kv scheduler dev
21-
22-
default: kv scheduler
23-
24-
dev: default test
25-
26-
test:
27-
@echo "Running tests in native mode."
28-
@export TZ='Asia/Shanghai'; \
29-
LOG_LEVEL=fatal $(GOTEST) -cover $(PACKAGES)
20+
.PHONY: clean test proto kv dev
3021

3122
CURDIR := $(shell pwd)
3223
export PATH := $(CURDIR)/bin/:$(PATH)
@@ -35,11 +26,20 @@ proto:
3526
(cd proto && ./generate_go.sh)
3627
GO111MODULE=on go build ./proto/pkg/...
3728

38-
kv:
39-
$(GOBUILD) -o bin/tinykv-server kv/main.go
29+
default: kv example
4030

41-
scheduler:
42-
$(GOBUILD) -o bin/tinyscheduler-server scheduler/main.go
31+
dev: default test
32+
33+
example: proto
34+
$(GOBUILD) -o bin/tinykv-client client/client.go
35+
36+
test: proto
37+
@echo "Running tests in native mode."
38+
@export TZ='Asia/Shanghai'; \
39+
LOG_LEVEL=fatal $(GOTEST) -cover $(PACKAGES)
40+
41+
kv: proto
42+
$(GOBUILD) -o bin/tinykv-server kv/main.go
4343

4444
ci: default
4545
@echo "Checking formatting"
@@ -73,17 +73,14 @@ project2b:
7373
project2c:
7474
$(GOTEST) ./raft ./kv/test_raftstore -run 2C
7575

76-
project3: project3a project3b project3c
76+
project3: project3a project3b
7777

7878
project3a:
7979
$(GOTEST) ./raft -run 3A
8080

8181
project3b:
8282
$(GOTEST) ./kv/test_raftstore -run 3B
8383

84-
project3c:
85-
$(GOTEST) ./scheduler/server ./scheduler/server/schedulers -check.f="3C"
86-
8784
project4: project4a project4b project4c
8885

8986
project4a:

README.md

Lines changed: 26 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,17 @@
1+
# Progress
2+
|Module|Status|
3+
|-|-|
4+
| Project 1 | Done |
5+
| Project 2a| Done |
6+
| Project 2b| Done |
7+
| Project 2c| Done |
8+
| Project 3a| Done |
9+
| Project 3b| Done |
10+
| Project 3c| De-scoped, using official placement driver v3.0.13 : )|
11+
| Project 4a| Done |
12+
| Project 4b| Done |
13+
| Project 4c| Done (No error handling) |
14+
115
# The TinyKV Course
216

317
This is a series of projects on a key-value storage system built with the Raft consensus algorithm. These projects are inspired by the famous [MIT 6.824](http://nil.csail.mit.edu/6.824/2018/index.html) course, but aim to be closer to industry implementations. The whole course is pruned from [TiKV](https://github.com/tikv/tikv) and re-written in Go. After completing this course, you will have the knowledge to implement a horizontal scalable, high available, key-value storage service with distributed transaction support and a better understanding of TiKV implementation.
@@ -65,35 +79,27 @@ After you finished the whole implementation, it's runnable now. You can try Tiny
6579
### Build
6680

6781
```
68-
make
82+
make kv
6983
```
7084

71-
It builds the binary of `tinykv-server` and `tinyscheduler-server` to `bin` dir.
85+
It builds the binary of `tinykv-server` to `bin` dir.
7286

7387
### Run
74-
75-
Put the binary of `tinyscheduler-server`, `tinykv-server` and `tinysql-server` into a single dir.
76-
77-
Under the binary dir, run the following commands:
78-
88+
On your laptop with local docker:
7989
```
80-
mkdir -p data
90+
$ ./start-pd.sh
8191
```
92+
To start the scheduler.
8293

94+
Start 3 store servers (since by default replication factor is 3):
8395
```
84-
./tinyscheduler-server
85-
```
86-
96+
$ ./start-server.sh 20601
97+
$ ./start-server.sh 20602
98+
$ ./start-server.sh 20603
8799
```
88-
./tinykv-server -path=data
89-
```
90-
91-
```
92-
./tinysql-server --store=tikv --path="127.0.0.1:2379"
93-
```
94-
95-
### Play
96100

101+
After the placement driver does the magic, hack the client dir:
97102
```
98-
mysql -u root -h 127.0.0.1 -P 4000
103+
$ make client
104+
$ ./bin/tinykv-client
99105
```

client/client.go

Lines changed: 135 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,135 @@
1+
package main
2+
3+
import (
4+
"context"
5+
"fmt"
6+
"github.com/pingcap-incubator/tinykv/log"
7+
"github.com/pingcap-incubator/tinykv/proto/pkg/kvrpcpb"
8+
"github.com/pingcap-incubator/tinykv/proto/pkg/metapb"
9+
10+
"google.golang.org/grpc"
11+
12+
sched "github.com/pingcap-incubator/tinykv/proto/pkg/pdpb"
13+
kv "github.com/pingcap-incubator/tinykv/proto/pkg/tinykvpb"
14+
)
15+
16+
type ctx struct {
17+
conn kv.TinyKvClient
18+
h *kvrpcpb.Context
19+
}
20+
21+
var pdconn *grpc.ClientConn
22+
var tikvconn *grpc.ClientConn
23+
24+
func locate(key []byte) (*grpc.ClientConn, ctx) {
25+
if pdconn == nil {
26+
pd, err := grpc.Dial("127.0.0.1:32379", grpc.WithInsecure(), grpc.WithBlock())
27+
if err != nil {
28+
panic(err)
29+
}
30+
pdconn = pd
31+
}
32+
33+
client := sched.NewPDClient(pdconn)
34+
var clusterID uint64
35+
{
36+
resp, err := client.GetMembers(context.TODO(), &sched.GetMembersRequest{})
37+
38+
if err != nil {
39+
panic(err)
40+
}
41+
42+
clusterID = resp.Header.ClusterId
43+
}
44+
45+
var region *metapb.Region
46+
var leader *metapb.Peer
47+
{
48+
resp, err := client.GetRegion(context.TODO(), &sched.GetRegionRequest{
49+
Header: &sched.RequestHeader{ClusterId: clusterID},
50+
RegionKey: key,
51+
})
52+
53+
if err != nil {
54+
panic(err)
55+
}
56+
57+
region = resp.Region
58+
leader = resp.Leader
59+
}
60+
61+
var store *metapb.Store
62+
{
63+
resp, err := client.GetStore(context.TODO(), &sched.GetStoreRequest{
64+
Header: &sched.RequestHeader{ClusterId: clusterID},
65+
StoreId: leader.StoreId,
66+
})
67+
if err != nil {
68+
panic(err)
69+
}
70+
store = resp.Store
71+
}
72+
73+
// XXX cache region and close connection in case leader changes
74+
if tikvconn == nil {
75+
conn, err := grpc.Dial(store.Address, grpc.WithInsecure(), grpc.WithBlock())
76+
if err != nil {
77+
panic(err)
78+
}
79+
tikvconn = conn
80+
}
81+
82+
return tikvconn, ctx{
83+
conn: kv.NewTinyKvClient(tikvconn),
84+
h: &kvrpcpb.Context{
85+
RegionId: region.Id,
86+
RegionEpoch: region.RegionEpoch,
87+
Peer: leader,
88+
},
89+
}
90+
}
91+
92+
func Put(key, val []byte) {
93+
_, ctx := locate(key)
94+
_, err := ctx.conn.RawPut(context.TODO(), &kvrpcpb.RawPutRequest{
95+
Context: ctx.h,
96+
Key: key,
97+
Value: val,
98+
Cf: "default",
99+
})
100+
101+
if err != nil {
102+
panic(err)
103+
}
104+
105+
if err != nil {
106+
panic(err)
107+
}
108+
}
109+
110+
func Get(key []byte) []byte {
111+
_, ctx := locate(key)
112+
resp, err := ctx.conn.RawGet(context.TODO(), &kvrpcpb.RawGetRequest{
113+
Context: ctx.h,
114+
Key: key,
115+
Cf: "default",
116+
})
117+
118+
if err != nil {
119+
panic(err)
120+
}
121+
122+
if err != nil {
123+
panic(err)
124+
}
125+
return resp.Value
126+
}
127+
128+
func main() {
129+
for i := 0; i < 128; i++ {
130+
key := []byte("hello " + fmt.Sprintf("%4d", i))
131+
Put(key, []byte("world "+fmt.Sprintf("%4d", i)))
132+
133+
log.Infof("Get val %v", Get(key))
134+
}
135+
}

doc/project2-RaftKV.md

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -122,7 +122,7 @@ The format is as below and some helper functions are provided in `kv/raftstore/m
122122

123123
> You may wonder why TinyKV needs two badger instances. Actually, it can use only one badger to store both raft log and state machine data. Separating into two instances is just to be consistent with TiKV design.
124124
125-
These metadatas should be created and updated in `PeerStorage`. When creating PeerStorage, see `kv/raftstore/peer_storager.go`. It initializes RaftLocalState, RaftApplyState of this Peer or gets the previous value from the underlying engine in the case of restart. Note that the value of both RAFT_INIT_LOG_TERM and RAFT_INIT_LOG_INDEX is 5 (as long as it's larger than 1) but not 0. The reason why not set it to 0 is to distinguish with the case that peer created passively after conf change. You may not quite understand it now, so just keep it in mind and the detail will be described in project3b when you are implementing conf change.
125+
The metadata should be created and updated in `PeerStorage`. When creating PeerStorage, see `kv/raftstore/peer_storage.go`. It initializes RaftLocalState, RaftApplyState of this Peer or gets the previous value from the underlying engine in the case of restart. Note that the value of both RAFT_INIT_LOG_TERM and RAFT_INIT_LOG_INDEX is 5 (as long as it's larger than 1) but not 0. The reason why not set it to 0 is to distinguish with the case that peer created passively after conf change. You may not quite understand it now, so just keep it in mind and the detail will be described in project3b when you are implementing conf change.
126126

127127
The code you need to implement in this part is only one function: `PeerStorage.SaveReadyState`, what this function does is to save the data in `raft.Ready` to badger, including append log entries and save the Raft hard state.
128128

kv/config/config.go

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -52,6 +52,11 @@ func (c *Config) Validate() error {
5252
return fmt.Errorf("election tick must be greater than heartbeat tick.")
5353
}
5454

55+
if c.SchedulerHeartbeatTickInterval <= c.RaftBaseTickInterval {
56+
return fmt.Errorf("schduler heartbeat tick must be larger than raft base tick")
57+
}
58+
//... and others too
59+
5560
return nil
5661
}
5762

@@ -73,7 +78,7 @@ func NewDefaultConfig() *Config {
7378
// Assume the average size of entries is 1k.
7479
RaftLogGcCountLimit: 128000,
7580
SplitRegionCheckTickInterval: 10 * time.Second,
76-
SchedulerHeartbeatTickInterval: 100 * time.Millisecond,
81+
SchedulerHeartbeatTickInterval: 2 * time.Second,
7782
SchedulerStoreHeartbeatTickInterval: 10 * time.Second,
7883
RegionMaxSize: 144 * MB,
7984
RegionSplitSize: 96 * MB,
@@ -83,8 +88,8 @@ func NewDefaultConfig() *Config {
8388

8489
func NewTestConfig() *Config {
8590
return &Config{
86-
LogLevel: "info",
8791
Raft: true,
92+
LogLevel: "info",
8893
RaftBaseTickInterval: 50 * time.Millisecond,
8994
RaftHeartbeatTicks: 2,
9095
RaftElectionTimeoutTicks: 10,

kv/raftstore/node.go

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -13,8 +13,8 @@ import (
1313
"github.com/pingcap-incubator/tinykv/kv/util/engine_util"
1414
"github.com/pingcap-incubator/tinykv/log"
1515
"github.com/pingcap-incubator/tinykv/proto/pkg/metapb"
16+
schedulerpb "github.com/pingcap-incubator/tinykv/proto/pkg/pdpb"
1617
"github.com/pingcap-incubator/tinykv/proto/pkg/raft_serverpb"
17-
"github.com/pingcap-incubator/tinykv/proto/pkg/schedulerpb"
1818
"github.com/pingcap/errors"
1919
)
2020

@@ -162,7 +162,7 @@ func (n *Node) BootstrapCluster(ctx context.Context, engines *engine_util.Engine
162162
time.Sleep(time.Second)
163163
}
164164

165-
res, err := n.schedulerClient.Bootstrap(ctx, n.store)
165+
res, err := n.schedulerClient.Bootstrap(ctx, n.store, firstRegion)
166166
if err != nil {
167167
log.Errorf("bootstrap cluster failed, clusterID: %d, err: %v", n.clusterID, err)
168168
continue

kv/raftstore/peer.go

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -83,7 +83,7 @@ type peer struct {
8383

8484
// Record the callback of the proposals
8585
// (Used in 2B)
86-
proposals []*proposal
86+
proposals map[uint64]proposal
8787

8888
// Index of last scheduled compacted raft log.
8989
// (Used in 2C)
@@ -133,6 +133,10 @@ func NewPeer(storeId uint64, cfg *config.Config, engines *engine_util.Engines, r
133133
Storage: ps,
134134
}
135135

136+
for _, p := range region.GetPeers() {
137+
raftCfg.Peers = append(raftCfg.Peers, p.Id)
138+
}
139+
136140
raftGroup, err := raft.NewRawNode(raftCfg)
137141
if err != nil {
138142
return nil, err
@@ -146,6 +150,7 @@ func NewPeer(storeId uint64, cfg *config.Config, engines *engine_util.Engines, r
146150
PeersStartPendingTime: make(map[uint64]time.Time),
147151
Tag: tag,
148152
ticker: newTicker(region.GetId(), cfg),
153+
proposals: map[uint64]proposal{},
149154
}
150155

151156
// If this region has only one peer and I am the one, campaign directly.
@@ -353,6 +358,7 @@ func (p *peer) HeartbeatScheduler(ch chan<- worker.Task) {
353358
Peer: p.Meta,
354359
PendingPeers: p.CollectPendingPeers(),
355360
ApproximateSize: p.ApproximateSize,
361+
Term: p.RaftGroup.Raft.Term,
356362
}
357363
}
358364

0 commit comments

Comments
 (0)