Skip to content

Commit 1d8b8bc

Browse files
author
Junlong Gao
committed
Working on a solution
Also, added a basic raw kv interface demo
1 parent 0a3cb4d commit 1d8b8bc

File tree

85 files changed

+3372
-41222
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

85 files changed

+3372
-41222
lines changed

.dockerignore

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
proto/_tools
2+
bin/*
3+
doc/*
4+
out

.gitignore

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,4 +3,5 @@
33
node/node
44
unikv/unikv
55
bin/*
6-
_tools
6+
_tools
7+
proto/pkg

Dockerfile

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
FROM golang:1.13.8 AS builder
2+
3+
RUN apt-get update && apt-get install -y wget zip \
4+
&& wget https://github.com/protocolbuffers/protobuf/releases/download/v3.12.3/protoc-3.12.3-linux-x86_64.zip \
5+
&& unzip protoc-3.12.3-linux-x86_64.zip \
6+
&& cp bin/protoc /usr/local/bin
7+
8+
RUN mkdir -p /src
9+
COPY kv /src/kv
10+
COPY log /src/log
11+
COPY proto /src/proto
12+
COPY raft /src/raft
13+
COPY go.mod /src/go.mod
14+
COPY go.sum /src/go.sum
15+
COPY Makefile /src/Makefile
16+
COPY scheduler /src/scheduler
17+
18+
WORKDIR /src
19+
RUN ls && make proto && go mod download && go test -c ./kv/test_raftstore && go test -c ./raft && go test -c ./kv/transaction
20+
21+
FROM ubuntu:latest
22+
WORKDIR /root/
23+
COPY --from=builder /src/ .

Makefile

Lines changed: 7 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -9,20 +9,23 @@ endif
99

1010
GO := GO111MODULE=on go
1111
GOBUILD := $(GO) build $(BUILD_FLAG) -tags codes
12-
GOTEST := $(GO) test -v --count=1 --parallel=1 -p=1
12+
GOTEST := $(GO) test -v --count=1 --parallel=1 -p=1 -failfast -timeout 600m
1313

1414
TEST_LDFLAGS := ""
1515

1616
PACKAGE_LIST := go list ./...| grep -vE "cmd"
1717
PACKAGES := $$($(PACKAGE_LIST))
1818

1919
# Targets
20-
.PHONY: clean test proto kv scheduler dev
20+
.PHONY: clean test proto kv dev
2121

22-
default: kv scheduler
22+
default: kv example
2323

2424
dev: default test
2525

26+
example:
27+
$(GOBUILD) -o bin/tinykv-client client/client.go
28+
2629
test:
2730
@echo "Running tests in native mode."
2831
@export TZ='Asia/Shanghai'; \
@@ -38,9 +41,6 @@ proto:
3841
kv:
3942
$(GOBUILD) -o bin/tinykv-server kv/main.go
4043

41-
scheduler:
42-
$(GOBUILD) -o bin/tinyscheduler-server scheduler/main.go
43-
4444
ci: default
4545
@echo "Checking formatting"
4646
@test -z "$$(gofmt -s -l $$(find . -name '*.go' -type f -print) | tee /dev/stderr)"
@@ -73,17 +73,14 @@ project2b:
7373
project2c:
7474
$(GOTEST) ./raft ./kv/test_raftstore -run 2C
7575

76-
project3: project3a project3b project3c
76+
project3: project3a project3b
7777

7878
project3a:
7979
$(GOTEST) ./raft -run 3A
8080

8181
project3b:
8282
$(GOTEST) ./kv/test_raftstore -run 3B
8383

84-
project3c:
85-
$(GOTEST) ./scheduler/server ./scheduler/server/schedulers -check.f="3C"
86-
8784
project4: project4a project4b project4c
8885

8986
project4a:

README.md

Lines changed: 26 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,17 @@
1+
# Progress
2+
|Module|Status|
3+
|-|-|
4+
| Project 1 | Done |
5+
| Project 2a| Done |
6+
| Project 2b| Done |
7+
| Project 2c| Done |
8+
| Project 3a| Done |
9+
| Project 3b| Done |
10+
| Project 3c| De-scoped, using official placement driver v3.0.13 : )|
11+
| Project 4a| Done |
12+
| Project 4b| Done |
13+
| Project 4c| Done (No error handling) |
14+
115
# The TinyKV Course
216

317
This is a series of projects on a key-value storage system built with the Raft consensus algorithm. These projects are inspired by the famous [MIT 6.824](http://nil.csail.mit.edu/6.824/2018/index.html) course, but aim to be closer to industry implementations. The whole course is pruned from [TiKV](https://github.com/tikv/tikv) and re-written in Go. After completing this course, you will have the knowledge to implement a horizontal scalable, high available, key-value storage service with distributed transaction support and a better understanding of TiKV implementation.
@@ -65,35 +79,27 @@ After you finished the whole implementation, it's runnable now. You can try Tiny
6579
### Build
6680

6781
```
68-
make
82+
make kv
6983
```
7084

71-
It builds the binary of `tinykv-server` and `tinyscheduler-server` to `bin` dir.
85+
It builds the binary of `tinykv-server` to `bin` dir.
7286

7387
### Run
74-
75-
Put the binary of `tinyscheduler-server`, `tinykv-server` and `tinysql-server` into a single dir.
76-
77-
Under the binary dir, run the following commands:
78-
88+
On your laptop with local docker:
7989
```
80-
mkdir -p data
90+
$ ./start-pd.sh
8191
```
92+
To start the scheduler.
8293

94+
Start 3 store servers (since by default replication factor is 3):
8395
```
84-
./tinyscheduler-server
85-
```
86-
96+
$ ./start-server.sh 20601
97+
$ ./start-server.sh 20602
98+
$ ./start-server.sh 20603
8799
```
88-
./tinykv-server -path=data
89-
```
90-
91-
```
92-
./tinysql-server --store=tikv --path="127.0.0.1:2379"
93-
```
94-
95-
### Play
96100

101+
After the placement driver does the magic, hack the client dir:
97102
```
98-
mysql -u root -h 127.0.0.1 -P 4000
103+
$ make client
104+
$ ./bin/tinykv-client
99105
```

client/client.go

Lines changed: 135 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,135 @@
1+
package main
2+
3+
import (
4+
"context"
5+
"fmt"
6+
"github.com/pingcap-incubator/tinykv/log"
7+
"github.com/pingcap-incubator/tinykv/proto/pkg/kvrpcpb"
8+
"github.com/pingcap-incubator/tinykv/proto/pkg/metapb"
9+
10+
"google.golang.org/grpc"
11+
12+
sched "github.com/pingcap-incubator/tinykv/proto/pkg/pdpb"
13+
kv "github.com/pingcap-incubator/tinykv/proto/pkg/tinykvpb"
14+
)
15+
16+
type ctx struct {
17+
conn kv.TinyKvClient
18+
h *kvrpcpb.Context
19+
}
20+
21+
var pdconn *grpc.ClientConn
22+
var tikvconn *grpc.ClientConn
23+
24+
func locate(key []byte) (*grpc.ClientConn, ctx) {
25+
if pdconn == nil {
26+
pd, err := grpc.Dial("127.0.0.1:32379", grpc.WithInsecure(), grpc.WithBlock())
27+
if err != nil {
28+
panic(err)
29+
}
30+
pdconn = pd
31+
}
32+
33+
client := sched.NewPDClient(pdconn)
34+
var clusterID uint64
35+
{
36+
resp, err := client.GetMembers(context.TODO(), &sched.GetMembersRequest{})
37+
38+
if err != nil {
39+
panic(err)
40+
}
41+
42+
clusterID = resp.Header.ClusterId
43+
}
44+
45+
var region *metapb.Region
46+
var leader *metapb.Peer
47+
{
48+
resp, err := client.GetRegion(context.TODO(), &sched.GetRegionRequest{
49+
Header: &sched.RequestHeader{ClusterId: clusterID},
50+
RegionKey: key,
51+
})
52+
53+
if err != nil {
54+
panic(err)
55+
}
56+
57+
region = resp.Region
58+
leader = resp.Leader
59+
}
60+
61+
var store *metapb.Store
62+
{
63+
resp, err := client.GetStore(context.TODO(), &sched.GetStoreRequest{
64+
Header: &sched.RequestHeader{ClusterId: clusterID},
65+
StoreId: leader.StoreId,
66+
})
67+
if err != nil {
68+
panic(err)
69+
}
70+
store = resp.Store
71+
}
72+
73+
// XXX cache region and close connection in case leader changes
74+
if tikvconn == nil {
75+
conn, err := grpc.Dial(store.Address, grpc.WithInsecure(), grpc.WithBlock())
76+
if err != nil {
77+
panic(err)
78+
}
79+
tikvconn = conn
80+
}
81+
82+
return tikvconn, ctx{
83+
conn: kv.NewTinyKvClient(tikvconn),
84+
h: &kvrpcpb.Context{
85+
RegionId: region.Id,
86+
RegionEpoch: region.RegionEpoch,
87+
Peer: leader,
88+
},
89+
}
90+
}
91+
92+
func Put(key, val []byte) {
93+
_, ctx := locate(key)
94+
_, err := ctx.conn.RawPut(context.TODO(), &kvrpcpb.RawPutRequest{
95+
Context: ctx.h,
96+
Key: key,
97+
Value: val,
98+
Cf: "default",
99+
})
100+
101+
if err != nil {
102+
panic(err)
103+
}
104+
105+
if err != nil {
106+
panic(err)
107+
}
108+
}
109+
110+
func Get(key []byte) []byte {
111+
_, ctx := locate(key)
112+
resp, err := ctx.conn.RawGet(context.TODO(), &kvrpcpb.RawGetRequest{
113+
Context: ctx.h,
114+
Key: key,
115+
Cf: "default",
116+
})
117+
118+
if err != nil {
119+
panic(err)
120+
}
121+
122+
if err != nil {
123+
panic(err)
124+
}
125+
return resp.Value
126+
}
127+
128+
func main() {
129+
for i := 0; i < 128; i++ {
130+
key := []byte("hello " + fmt.Sprintf("%4d", i))
131+
Put(key, []byte("world "+fmt.Sprintf("%4d", i)))
132+
133+
log.Infof("Get val %v", Get(key))
134+
}
135+
}

doc/project2-RaftKV.md

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -122,7 +122,7 @@ The format is as below and some helper functions are provided in `kv/raftstore/m
122122

123123
> You may wonder why TinyKV needs two badger instances. Actually, it can use only one badger to store both raft log and state machine data. Separating into two instances is just to be consistent with TiKV design.
124124
125-
These metadatas should be created and updated in `PeerStorage`. When creating PeerStorage, see `kv/raftstore/peer_storager.go`. It initializes RaftLocalState, RaftApplyState of this Peer or gets the previous value from the underlying engine in the case of restart. Note that the value of both RAFT_INIT_LOG_TERM and RAFT_INIT_LOG_INDEX is 5 (as long as it's larger than 1) but not 0. The reason why not set it to 0 is to distinguish with the case that peer created passively after conf change. You may not quite understand it now, so just keep it in mind and the detail will be described in project3b when you are implementing conf change.
125+
The metadata should be created and updated in `PeerStorage`. When creating PeerStorage, see `kv/raftstore/peer_storage.go`. It initializes RaftLocalState, RaftApplyState of this Peer or gets the previous value from the underlying engine in the case of restart. Note that the value of both RAFT_INIT_LOG_TERM and RAFT_INIT_LOG_INDEX is 5 (as long as it's larger than 1) but not 0. The reason why not set it to 0 is to distinguish with the case that peer created passively after conf change. You may not quite understand it now, so just keep it in mind and the detail will be described in project3b when you are implementing conf change.
126126

127127
The code you need to implement in this part is only one function: `PeerStorage.SaveReadyState`, what this function does is to save the data in `raft.Ready` to badger, including append log entries and save the Raft hard state.
128128

kv/config/config.go

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -52,6 +52,11 @@ func (c *Config) Validate() error {
5252
return fmt.Errorf("election tick must be greater than heartbeat tick.")
5353
}
5454

55+
if c.SchedulerHeartbeatTickInterval <= c.RaftBaseTickInterval {
56+
return fmt.Errorf("schduler heartbeat tick must be larger than raft base tick")
57+
}
58+
//... and others too
59+
5560
return nil
5661
}
5762

@@ -73,7 +78,7 @@ func NewDefaultConfig() *Config {
7378
// Assume the average size of entries is 1k.
7479
RaftLogGcCountLimit: 128000,
7580
SplitRegionCheckTickInterval: 10 * time.Second,
76-
SchedulerHeartbeatTickInterval: 100 * time.Millisecond,
81+
SchedulerHeartbeatTickInterval: 2 * time.Second,
7782
SchedulerStoreHeartbeatTickInterval: 10 * time.Second,
7883
RegionMaxSize: 144 * MB,
7984
RegionSplitSize: 96 * MB,
@@ -84,6 +89,7 @@ func NewDefaultConfig() *Config {
8489
func NewTestConfig() *Config {
8590
return &Config{
8691
Raft: true,
92+
LogLevel: "info",
8793
RaftBaseTickInterval: 50 * time.Millisecond,
8894
RaftHeartbeatTicks: 2,
8995
RaftElectionTimeoutTicks: 10,

kv/raftstore/node.go

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -13,8 +13,8 @@ import (
1313
"github.com/pingcap-incubator/tinykv/kv/util/engine_util"
1414
"github.com/pingcap-incubator/tinykv/log"
1515
"github.com/pingcap-incubator/tinykv/proto/pkg/metapb"
16+
schedulerpb "github.com/pingcap-incubator/tinykv/proto/pkg/pdpb"
1617
"github.com/pingcap-incubator/tinykv/proto/pkg/raft_serverpb"
17-
"github.com/pingcap-incubator/tinykv/proto/pkg/schedulerpb"
1818
"github.com/pingcap/errors"
1919
)
2020

@@ -162,7 +162,7 @@ func (n *Node) BootstrapCluster(ctx context.Context, engines *engine_util.Engine
162162
time.Sleep(time.Second)
163163
}
164164

165-
res, err := n.schedulerClient.Bootstrap(ctx, n.store)
165+
res, err := n.schedulerClient.Bootstrap(ctx, n.store, firstRegion)
166166
if err != nil {
167167
log.Errorf("bootstrap cluster failed, clusterID: %d, err: %v", n.clusterID, err)
168168
continue

kv/raftstore/peer.go

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -83,7 +83,7 @@ type peer struct {
8383

8484
// Record the callback of the proposals
8585
// (Used in 2B)
86-
proposals []*proposal
86+
proposals map[uint64]proposal
8787

8888
// Index of last scheduled compacted raft log.
8989
// (Used in 2C)
@@ -133,6 +133,10 @@ func NewPeer(storeId uint64, cfg *config.Config, engines *engine_util.Engines, r
133133
Storage: ps,
134134
}
135135

136+
for _, p := range region.GetPeers() {
137+
raftCfg.Peers = append(raftCfg.Peers, p.Id)
138+
}
139+
136140
raftGroup, err := raft.NewRawNode(raftCfg)
137141
if err != nil {
138142
return nil, err
@@ -146,6 +150,7 @@ func NewPeer(storeId uint64, cfg *config.Config, engines *engine_util.Engines, r
146150
PeersStartPendingTime: make(map[uint64]time.Time),
147151
Tag: tag,
148152
ticker: newTicker(region.GetId(), cfg),
153+
proposals: map[uint64]proposal{},
149154
}
150155

151156
// If this region has only one peer and I am the one, campaign directly.
@@ -353,6 +358,7 @@ func (p *peer) HeartbeatScheduler(ch chan<- worker.Task) {
353358
Peer: p.Meta,
354359
PendingPeers: p.CollectPendingPeers(),
355360
ApproximateSize: p.ApproximateSize,
361+
Term: p.RaftGroup.Raft.Term,
356362
}
357363
}
358364

0 commit comments

Comments
 (0)