Skip to content

Commit 205014d

Browse files
yyyshiyyyshi
committed
test
test
1 parent efb9fde commit 205014d

File tree

19 files changed

+119
-9
lines changed

19 files changed

+119
-9
lines changed

‎src/bio/SConscript‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,8 @@ def scons():
2323
denv.AppendUnique(OBJPREFIX='b_')
2424

2525
# SPDK related libs
26+
# libbio.so 依赖spdk 的库
27+
# 所在目录:/opt/daos/prereq/release/spdk/lib# ll *.so
2628
libs = ['spdk_log', 'spdk_env_dpdk', 'spdk_thread', 'spdk_bdev', 'rte_mempool']
2729
libs += ['rte_mempool_ring', 'rte_bus_pci', 'rte_pci', 'rte_ring']
2830
libs += ['rte_mbuf', 'rte_eal', 'rte_kvargs', 'spdk_bdev_aio']

‎src/bio/bio_xstream.c‎

Lines changed: 22 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -100,6 +100,7 @@ static struct bio_nvme_data nvme_glb;
100100
static int
101101
bio_spdk_env_init(void)
102102
{
103+
// 构建spdk opt
103104
struct spdk_env_opts opts;
104105
bool enable_rpc_srv = false;
105106
int rc;
@@ -123,6 +124,7 @@ bio_spdk_env_init(void)
123124
// == true
124125
if (bio_nvme_configured(SMD_DEV_TYPE_MAX)) {
125126
// 传递allowed bdev pci 地址给spdk，即将nvme addr 添加到opts 的allowlist 中
127+
// nvme 类型的bdev，创建命令是conf 中的 attach controller
126128
rc = bio_add_allowed_alloc(nvme_glb.bd_nvme_conf, &opts, &roles);
127129
if (rc != 0) {
128130
D_ERROR("Failed to add allowed devices to SPDK env, "DF_RC"\n",
@@ -483,6 +485,7 @@ struct common_cp_arg {
483485
static void
484486
common_prep_arg(struct common_cp_arg *arg)
485487
{
488+
// 设置一个 inflight
486489
arg->cca_inflights = 1;
487490
arg->cca_rc = 0;
488491
arg->cca_bs = NULL;
@@ -491,6 +494,7 @@ common_prep_arg(struct common_cp_arg *arg)
491494
static void
492495
common_init_cb(void *arg, int rc)
493496
{
497+
// rpc 执行成功或者超时后，inflight 自减
494498
struct common_cp_arg *cp_arg = arg;
495499

496500
D_ASSERT(cp_arg->cca_inflights == 1);
@@ -1732,28 +1736,39 @@ bio_xsctxt_alloc(struct bio_xs_context **pctxt, int tgt_id, bool self_polling)
17321736
common_prep_arg(&cp_arg);
17331737
// 这里用的是全局的nvme conf 文件，每个engine 有自己单独的
17341738
// spdk 中子系统其实就是模块的意思，比如子系统名字为bdev，表示bdev 模块
1739+
// 这里面会调用 spdk_rpc_initialize，会启动rpc 服务监听
17351740
spdk_subsystem_init_from_json_config(nvme_glb.bd_nvme_conf,
17361741
SPDK_DEFAULT_RPC_ADDR,
17371742
subsys_init_cb, &cp_arg,
17381743
true);
1744+
// init from json 会注册 rpc_subsystem_poll，rpc_client_connect_poller，用于接收rpc 请求
1745+
// 这里是为了等待rpc req 被处理完，或者req 超时
17391746
rc = xs_poll_completion(ctxt, &cp_arg.cca_inflights, 0);
17401747
D_ASSERT(rc == 0);
17411748

1749+
// 初始化失败了
17421750
if (cp_arg.cca_rc != 0) {
17431751
rc = cp_arg.cca_rc;
17441752
D_ERROR("failed to init bdevs, rc:%d\n", rc);
17451753
goto out;
17461754
}
17471755

17481756
/* Continue poll until no more events */
1749-
// todo: 这里是为啥
1757+
// 这个时间点:
1758+
// 1. 不会有其他xs，所有没人给这个 init_thread 通过 spdk_thread_send_msg 发meg
1759+
// 2. 所以只能是处理已经注册过的poller：rpc_subsystem_poll，rpc_client_connect_poller
1760+
// 1-1: spdk_subsystem_init_from_json_config --> spdk_rpc_initialize --> rpc_subsystem_poll --> spdk_rpc_accept ---> spdk_jsonrpc_server_poll: 检查监听的socket，处理连接
1761+
// 2-2: spdk_subsystem_init_from_json_config --> rpc_client_connect_poller --> spdk_jsonrpc_client_poll --> jsonrpc_client_poll / jsonrpc_client_poll_connecting
1762+
// return 1 if work was done. 0 if no work was done.
1763+
// 返回0表示轮询成功，但是没有处理任何事件；返回>0 表示轮询成功，且处理了n 个事件；返回 <0 表示轮询失败
1764+
// 这里是loop 直到所有的事件都被处理完
17501765
while (spdk_thread_poll(ctxt->bxc_thread, 0, 0) > 0)
17511766
;
17521767
// 这个日志会打印，tgt_id 为 0
1753-
// todo: 什么样才算初始化完成，此时可以通过 spdk_bdev_first 拿到spdk bdev 了吗？
1768+
// 初始化完成，此时可以通过 spdk_bdev_first 拿到spdk bdev 了
17541769
D_DEBUG(DB_MGMT, "SPDK bdev initialized, tgt_id:%d", tgt_id);
17551770

1756-
// 将spdk thread 设置到全局数据中，表示在众多（每个xs 有各自的spdk thread）的thread 中，当前是扫描bdev那个
1771+
// 将spdk thread 设置到全局数据中，表示在众多（每个xs 有各自的spdk thread）的thread 中，当前是初始化的那个
17571772
nvme_glb.bd_init_thread = ctxt->bxc_thread;
17581773

17591774
// 内部会load_blobstore 最终创建spdk bs，只有target 0 会执行
@@ -1764,7 +1779,10 @@ bio_xsctxt_alloc(struct bio_xs_context **pctxt, int tgt_id, bool self_polling)
17641779
goto out;
17651780
}
17661781

1767-
// bio bdev 初始化完成后，重启spdk rpc server。rpc 默认是禁用的
1782+
// 上面 spdk_subsystem_init_from_json_config 中会通过调用 spdk_rpc_initialize 自动开启rpc 监听
1783+
// 用于处理json 文件中配置的method，其中 nvme_attach_controller 会1. 创建nvme ctrlr 2. 添加ns 到nvme ctrlr 3. 注册spdk nvme bdev
1784+
// 在 init_bio_bdevs 函数执行之前，spdk nvme bdev 注册已经完成（通过nvme_attach_controller 对应的mapping 函数，走的rpc 流程）
1785+
// 当 bio bdev 初始化完成后，重启spdk rpc server，这里是false，所以不用重启
17681786
/* After bio_bdevs are initialized, restart SPDK JSON-RPC server if required. */
17691787
// 默认是false
17701788
if (nvme_glb.bd_enable_rpc_srv) {

‎src/control/cmd/daos_server_helper/handler.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -230,7 +230,6 @@ type bdevHandler struct {
230230
bdevProvider *bdev.Provider
231231
}
232232

233-
// todo: provider 怎么理解
234233
func (h *bdevHandler) setupProvider(log logging.Logger) {
235234
if h.bdevProvider == nil {
236235
// 调用了bdev 包下的 DefaultProvider。对应的后端是 spdkbackend
@@ -257,6 +256,7 @@ func (h *bdevScanHandler) Handle(log logging.Logger, req *pbin.Request) *pbin.Re
257256
// bdev scan 外层函数，内部通过 bdev 包下的 setupProvider 最终使用spdkbackend
258257
h.setupProvider(log)
259258

259+
// helper app 收到请求，构建provider 调用scan
260260
sRes, err := h.bdevProvider.Scan(sReq)
261261
if err != nil {
262262
return pbin.NewResponseWithError(err)

‎src/control/cmd/daos_server_helper/main.go‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@ import (
1313
)
1414

1515
func main() {
16+
// daos_server 可以调用daos_server_helper
1617
app := pbin.NewApp().
1718
WithAllowedCallers("daos_server")
1819

‎src/control/pbin/app.go‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -86,6 +86,7 @@ func (a *App) configureLogging(logPath string) {
8686

8787
// WithAllowedCallers adds a list of process names allowed to call this
8888
// application.
89+
// 添加一个列表，允许调用当前app
8990
func (a *App) WithAllowedCallers(callers ...string) *App {
9091
a.allowedCallers = callers
9192
return a
@@ -123,7 +124,7 @@ func (a *App) logError(err error) error {
123124
}
124125

125126
// Run executes the helper application process.
126-
127+
// 执行helper app
127128
func (a *App) Run() error {
128129
parentName, err := a.process.ParentProcessName()
129130
if err != nil {
@@ -153,6 +154,7 @@ func (a *App) Run() error {
153154
return a.logError(err)
154155
}
155156

157+
// 处理请求
156158
resp := a.handleRequest(req)
157159

158160
err = a.writeResponse(resp, conn)
@@ -206,6 +208,7 @@ func (a *App) readRequest(rdr io.Reader) (*Request, error) {
206208
}
207209

208210
func (a *App) handleRequest(req *Request) *Response {
211+
// 按照名字获取hdl
209212
reqHandler, ok := a.handlers[req.Method]
210213
if !ok {
211214
err := a.logError(errors.Errorf("unhandled method %q", req.Method))

‎src/control/pbin/exec.go‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -122,6 +122,8 @@ func ReadMessage(conn io.Reader) ([]byte, error) {
122122

123123
// ExecReq executes the supplied Request by starting a child process
124124
// to service the request. Returns a Response if successful.
125+
// 开启一个子协程执行转发命令
126+
// binPath 为 "daos_server_helper"
125127
func ExecReq(parent context.Context, log logging.Logger, binPath string, req *Request) (res *Response, err error) {
126128
if req == nil {
127129
return nil, errors.New("nil request")
@@ -130,6 +132,7 @@ func ExecReq(parent context.Context, log logging.Logger, binPath string, req *Re
130132
ctx, killChild := context.WithCancel(parent)
131133
defer killChild()
132134

135+
// 传入bin，调用go api
133136
child := exec.CommandContext(ctx, binPath)
134137
child.Stderr = &cmdLogger{
135138
logFn: log.Error,
@@ -160,12 +163,14 @@ func ExecReq(parent context.Context, log logging.Logger, binPath string, req *Re
160163
defer func() {
161164
// If there was an error, kill the child so that it can't
162165
// hang around waiting for input.
166+
// 出错时，主动kill child
163167
if err != nil {
164168
killChild()
165169
return
166170
}
167171

168172
// Otherwise, the child should exit normally.
173+
// 成功时，child 会自动退出
169174
err = child.Wait()
170175
}()
171176

‎src/control/pbin/forwarding.go‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@ import (
2020

2121
type (
2222
// Forwarder provides a common implementation of a request forwarder.
23+
// 提供通用的请求转发功能
2324
Forwarder struct {
2425
Disabled bool
2526

@@ -57,6 +58,7 @@ func NewForwarder(log logging.Logger, pbinName string) *Forwarder {
5758
}
5859

5960
// GetBinaryName returns the name of the binary requests will be forwarded to.
61+
// 获取请求要转发到的二进制程序名字，这里是转给daos_server_helper
6062
func (f *Forwarder) GetBinaryName() string {
6163
return f.pbinName
6264
}
@@ -74,6 +76,7 @@ func (f *Forwarder) CanForward() bool {
7476
// SendReq is responsible for marshaling the forwarded request into a message
7577
// that is sent to the privileged binary, then unmarshaling the response for
7678
// the caller.
79+
// 负责转发给又特权的 daos_server_helper
7780
func (f *Forwarder) SendReq(method string, fwdReq interface{}, fwdRes interface{}) error {
7881
if fwdReq == nil {
7982
return errors.New("nil request")
@@ -82,6 +85,7 @@ func (f *Forwarder) SendReq(method string, fwdReq interface{}, fwdRes interface{
8285
return errors.New("nil response")
8386
}
8487

88+
// 找daos_server_helper
8589
pbinPath, err := common.FindBinary(f.pbinName)
8690
if err != nil {
8791
return err
@@ -92,12 +96,15 @@ func (f *Forwarder) SendReq(method string, fwdReq interface{}, fwdRes interface{
9296
return errors.Wrap(err, "failed to marshal forwarded request as payload")
9397
}
9498

99+
// 构建req
95100
req := &Request{
96101
Method: method,
97102
Payload: payload,
98103
}
99104

100105
ctx := context.TODO()
106+
// 抓发执行cmd
107+
// pbinPath 为daos_server_helper
101108
res, err := ExecReq(ctx, f.log, pbinPath, req)
102109
if err != nil {
103110
if fault.IsFault(err) {

‎src/control/server/config/server.go‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -399,6 +399,7 @@ func (cfg *Server) SetPath(inPath string) error {
399399
return err
400400
}
401401

402+
// active 配置文件：/var/run/daos_server/.daos_server.active.yml
402403
// SaveActiveConfig saves read-only active config, tries config dir then /tmp/.
403404
func (cfg *Server) SaveActiveConfig(log logging.Logger) {
404405
activeConfig := filepath.Join(cfg.SocketDir, ConfigOut)
@@ -650,10 +651,12 @@ func (cfg *Server) Validate(log logging.Logger) (err error) {
650651
}
651652

652653
// Set DisableVMD reference if unset in config file.
654+
// 如果配置中没有默认为 vmd 开启
653655
if cfg.DisableVMD == nil {
654656
cfg.WithDisableVMD(false)
655657
}
656658

659+
// 日志中：vfio=true hotplug=false vmd=true requested in config
657660
log.Debugf("vfio=%v hotplug=%v vmd=%v requested in config", !cfg.DisableVFIO,
658661
cfg.EnableHotplug, !(*cfg.DisableVMD))
659662

‎src/control/server/ctl_storage.go‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@ import (
2525
// StorageControlService encapsulates the storage part of the control service
2626
type StorageControlService struct {
2727
log logging.Logger
28+
// provider 包含多种类型的抽象类
2829
storage *storage.Provider
2930
instanceStorage map[uint32]*storage.Config
3031
getMemInfo common.GetMemInfoFn
@@ -58,6 +59,7 @@ func (scs *StorageControlService) WithVMDEnabled() *StorageControlService {
5859
}
5960

6061
// NewStorageControlService returns an initialized *StorageControlService
62+
// 构建最外层的service
6163
func NewStorageControlService(log logging.Logger, ecs []*engine.Config) *StorageControlService {
6264
topCfg := &storage.Config{
6365
Tiers: nil,
@@ -70,9 +72,11 @@ func NewStorageControlService(log logging.Logger, ecs []*engine.Config) *Storage
7072
instanceStorage[uint32(i)] = &c.Storage
7173
}
7274

75+
// 构建scs
7376
return &StorageControlService{
7477
log: log,
7578
instanceStorage: instanceStorage,
79+
// 使用默认的provider
7680
storage: storage.DefaultProvider(log, 0, topCfg),
7781
getMemInfo: common.GetMemInfo,
7882
}

‎src/control/server/ctl_svc.go‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@ import (
1616

1717
// ControlService implements the control plane control service, satisfying
1818
// ctlpb.CtlSvcServer, and is the data container for the service.
19+
// 控制层服务
1920
type ControlService struct {
2021
ctlpb.UnimplementedCtlSvcServer
2122
StorageControlService

0 commit comments

Comments
 (0)