Skip to content

Commit dc4a121

Browse files
lance6716lance6716
committed
checkout more files
Signed-off-by: lance6716 <lance6716@gmail.com>
1 parent addc44a commit dc4a121

File tree

5 files changed

+277
-10
lines changed

5 files changed

+277
-10
lines changed

‎.editorconfig‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
root = true
2+
3+
[*.proto]
4+
indent_style = space
5+
indent_size = 2
6+
tab_width = 2
7+

‎README.md‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,16 @@ Protocol buffer files for TiKV
1717
* Go
1818
* Protoc 3.8.0
1919

20+
# Proto formatting
21+
22+
To avoid IDE-induced diffs, we use `buf format` to keep all `.proto` files consistently formatted.
23+
24+
* Check formatting (also runs as part of `make check`): `make proto-fmt-check`
25+
* Format in-place: `make proto-fmt`
26+
27+
The formatter is pinned and will be downloaded automatically into `./bin` (gitignored) when needed.
28+
If `make check` fails on formatting, run `make proto-fmt` and then rerun `make check`.
29+
2030
# Docker image
2131

2232
The easiest way to compile the protobufs is to use docker.

‎scripts/check.sh‎

Lines changed: 49 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,11 @@
11
#!/usr/bin/env bash
22

3+
PROGRAM=$(basename "$0")
4+
SCRIPTS_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
5+
KVPROTO_ROOT=$(cd "${SCRIPTS_DIR}/.." && pwd)
6+
7+
cd "${KVPROTO_ROOT}"
8+
39
check_protoc_version() {
410
version=$(protoc --version | awk '{print $NF}')
511
major=$(echo ${version} | cut -d '.' -f 1)
@@ -16,6 +22,10 @@ check_protoc_version() {
1622
return 1
1723
}
1824

25+
check-protos-format() {
26+
"${SCRIPTS_DIR}/proto_format.sh" --check
27+
}
28+
1929
check-protos-compatible() {
2030
GOPATH=$(go env GOPATH)
2131
if [ -z $GOPATH ]; then
@@ -36,7 +46,7 @@ check-protos-compatible() {
3646
protolock commit --force -lockdir=scripts -protoroot=proto
3747
fi
3848
# git report error like "fatal: detected dubious ownership in repository at" when reading the host's git folder
39-
git config --global --add safe.directory $(pwd)
49+
git config --global --add safe.directory "$(pwd)"
4050
# If the output message is encountered, please add proto.lock to git as well.
4151
git diff scripts/proto.lock | cat
4252
git diff --quiet scripts/proto.lock
@@ -47,6 +57,43 @@ check-protos-compatible() {
4757
return 0
4858
}
4959

50-
if ! check_protoc_version || ! check-protos-compatible; then
60+
check-protos-options() {
61+
local options=(
62+
'import "gogoproto/gogo.proto";'
63+
'import "rustproto.proto";'
64+
65+
'option (gogoproto.sizer_all) = true;'
66+
'option (gogoproto.marshaler_all) = true;'
67+
'option (gogoproto.unmarshaler_all) = true;'
68+
# Remove unnecessary fields from pb structs.
69+
# XXX_NoUnkeyedLiteral struct{} `json:"-"`
70+
# XXX_unrecognized []byte `json:"-"`
71+
# XXX_sizecache int32 `json:"-"`
72+
'option (gogoproto.goproto_unkeyed_all) = false;'
73+
'option (gogoproto.goproto_unrecognized_all) = false;'
74+
'option (gogoproto.goproto_sizecache_all) = false;'
75+
# TiKV does not need reflection and descriptor data.
76+
'option (rustproto.lite_runtime_all) = true;'
77+
)
78+
79+
local folder="./proto"
80+
for pb in "$folder"/*; do
81+
if [[ "$pb" == *.proto ]]; then
82+
# Iterate through the array
83+
for option in "${options[@]}"; do
84+
if ! grep -q "$option" "$pb"; then
85+
echo "Please add option \"$option\" to $pb"
86+
return 1
87+
fi
88+
done
89+
fi
90+
done
91+
92+
return 0
93+
}
94+
95+
96+
97+
if ! check_protoc_version || ! check-protos-format || ! check-protos-compatible || ! check-protos-options; then
5198
exit 1
5299
fi

‎scripts/common.sh‎

Lines changed: 207 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -21,8 +21,212 @@ function sed_inplace()
2121
function clean_gogo_proto()
2222
{
2323
local file=$1
24-
sed_inplace '/gogo.proto/d' ${file}
25-
sed_inplace '/option\ *(gogoproto/d' ${file}
26-
sed_inplace -e 's/\[.*gogoproto.*\]//g' ${file}
24+
# C++ codegen doesn't need gogoproto options, and we intentionally do not
25+
# compile/include gogoproto/gogo.proto in the C++ build.
26+
#
27+
# buf format may break field options into multi-line blocks like:
28+
# bytes data = 1 [
29+
# (gogoproto.customtype) = "...",
30+
# (gogoproto.nullable) = false
31+
# ];
32+
# so the old one-line sed approach is not sufficient.
33+
awk '
34+
function ltrim(s) {
35+
sub(/^[[:space:]]+/, "", s)
36+
return s
37+
}
38+
39+
function rtrim(s) {
40+
sub(/[[:space:]]+$/, "", s)
41+
return s
42+
}
43+
44+
function trim(s) {
45+
return rtrim(ltrim(s))
46+
}
47+
48+
function is_comment_line(s) {
49+
return (s ~ /^[[:space:]]*\/\//) || (s ~ /^[[:space:]]*\/\*/) || (s ~ /^[[:space:]]*\*/) || (s ~ /^[[:space:]]*\*\//)
50+
}
51+
52+
function is_option_line(s) {
53+
return !is_comment_line(s) && s !~ /^[[:space:]]*$/
54+
}
55+
56+
function is_gogo_option_line(s, t) {
57+
t = trim(s)
58+
return t ~ /^\(gogoproto\./
59+
}
60+
61+
function set_trailing_comma(s, want_comma, body, comment) {
62+
body = s
63+
comment = ""
64+
if (match(body, /\/\/.*/)) {
65+
comment = substr(body, RSTART)
66+
body = substr(body, 1, RSTART - 1)
67+
}
68+
body = rtrim(body)
69+
sub(/,$/, "", body)
70+
if (want_comma) {
71+
body = body ","
72+
}
73+
if (comment != "") {
74+
return body " " comment
75+
}
76+
return body
77+
}
78+
79+
function reset_block() {
80+
in_block = 0
81+
block_open = ""
82+
block_close = ""
83+
block_len = 0
84+
delete block_lines
85+
}
86+
87+
function flush_block( i, kept_len, option_count, last_option_idx, out_line) {
88+
if (!in_block) return
89+
90+
# Malformed proto: keep the original block.
91+
if (block_close == "") {
92+
print block_open
93+
for (i = 1; i <= block_len; i++) {
94+
print block_lines[i]
95+
}
96+
reset_block()
97+
return
98+
}
99+
100+
delete kept_lines
101+
delete option_indexes
102+
kept_len = 0
103+
option_count = 0
104+
for (i = 1; i <= block_len; i++) {
105+
if (is_gogo_option_line(block_lines[i])) {
106+
continue
107+
}
108+
kept_lines[++kept_len] = block_lines[i]
109+
if (is_option_line(block_lines[i])) {
110+
option_indexes[++option_count] = kept_len
111+
}
112+
}
113+
114+
if (option_count == 0) {
115+
out_line = block_open
116+
sub(/[[:space:]]*\[[[:space:]]*$/, ";", out_line)
117+
print out_line
118+
} else {
119+
last_option_idx = option_indexes[option_count]
120+
print block_open
121+
for (i = 1; i <= kept_len; i++) {
122+
out_line = kept_lines[i]
123+
if (is_option_line(out_line)) {
124+
out_line = set_trailing_comma(out_line, i != last_option_idx)
125+
}
126+
print out_line
127+
}
128+
print block_close
129+
}
130+
131+
delete kept_lines
132+
delete option_indexes
133+
reset_block()
134+
}
135+
136+
BEGIN {
137+
reset_block()
138+
}
139+
140+
{
141+
line = $0
142+
143+
# Ignore comment-only lines when looking for field option blocks.
144+
# Some protos include examples like:
145+
# // [
146+
# // ...
147+
# // ]
148+
# which must not be treated as an options block.
149+
is_comment = is_comment_line(line)
150+
151+
# Drop import + file-level options for gogoproto.
152+
if (line ~ /^[[:space:]]*import[[:space:]]+"gogoproto\/gogo\.proto";[[:space:]]*$/) next
153+
if (line ~ /^[[:space:]]*option[[:space:]]*\(gogoproto\./) next
154+
155+
# Inside a multi-line field options block.
156+
if (in_block) {
157+
if (line ~ /\][[:space:]]*;/) {
158+
block_close = line
159+
flush_block()
160+
} else {
161+
block_lines[++block_len] = line
162+
}
163+
next
164+
}
165+
166+
# Single-line field options that contain gogoproto.
167+
# Example: repeated Foo bars = 1 [(gogoproto.nullable) = false];
168+
if (!is_comment && line ~ /\[[^\]]*gogoproto\.[^\]]*\]/) {
169+
open_idx = index(line, "[")
170+
rest = substr(line, open_idx + 1)
171+
close_rel = index(rest, "]")
172+
if (close_rel == 0) {
173+
print line
174+
next
175+
}
176+
177+
prefix = substr(line, 1, open_idx - 1)
178+
inside = substr(rest, 1, close_rel - 1)
179+
suffix = substr(rest, close_rel + 1)
180+
181+
delete tokens
182+
delete kept_tokens
183+
token_len = split(inside, tokens, ",")
184+
kept_token_len = 0
185+
for (i = 1; i <= token_len; i++) {
186+
token = trim(tokens[i])
187+
if (token == "") {
188+
continue
189+
}
190+
if (token ~ /^\(gogoproto\./) {
191+
continue
192+
}
193+
kept_tokens[++kept_token_len] = token
194+
}
195+
196+
if (kept_token_len == 0) {
197+
prefix = rtrim(prefix)
198+
print prefix suffix
199+
} else {
200+
out_line = prefix "["
201+
for (i = 1; i <= kept_token_len; i++) {
202+
if (i > 1) {
203+
out_line = out_line ", "
204+
}
205+
out_line = out_line kept_tokens[i]
206+
}
207+
out_line = out_line "]" suffix
208+
print out_line
209+
}
210+
211+
next
212+
}
213+
214+
# Start of a multi-line field options block.
215+
if (!is_comment && line ~ /\[[[:space:]]*$/) {
216+
in_block = 1
217+
block_open = line
218+
block_close = ""
219+
block_len = 0
220+
delete block_lines
221+
next
222+
}
223+
224+
print line
225+
}
226+
227+
END {
228+
flush_block()
229+
}
230+
' "${file}" > "${file}.tmp" && mv "${file}.tmp" "${file}"
27231
}
28232
export -f clean_gogo_proto

‎scripts/generate_cpp.sh‎

Lines changed: 4 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -14,11 +14,10 @@ GRPC_INCLUDE=.:../include
1414
rm -rf proto-cpp && mkdir -p proto-cpp
1515
rm -rf cpp/kvproto && mkdir cpp/kvproto
1616

17-
cp proto/* proto-cpp/
18-
19-
sed_inplace '/gogo.proto/d' proto-cpp/*
20-
sed_inplace '/option\ *(gogoproto/d' proto-cpp/*
21-
sed_inplace -e 's/\[.*gogoproto.*\]//g' proto-cpp/*
17+
cp proto/*.proto proto-cpp/
18+
for pb in proto-cpp/*.proto; do
19+
clean_gogo_proto "${pb}"
20+
done
2221

2322
pushd proto-cpp >/dev/null
2423
protoc -I${GRPC_INCLUDE} --cpp_out ../cpp/kvproto *.proto

0 commit comments

Comments
 (0)