关于训练的问题

[tj1116]

使用Z1_StackBox数据集进行训练的时候，报如下错误：
Epoch 0: 0%| | 0/316 [00:00<?, ?it/s][rank0]: Traceback (most recent call last):
[rank0]: File "/home/tianjian/vla/unifolm-world-model-action/./scripts/trainer.py", line 212, in
[rank0]: trainer.fit(model, data)
[rank0]: File "/home/tianjian/.conda/envs/unifolm-wma/lib/python3.10/site-packages/pytorch_lightning/trainer/trainer.py", line 608, in fit
[rank0]: call._call_and_handle_interrupt(
[rank0]: File "/home/tianjian/.conda/envs/unifolm-wma/lib/python3.10/site-packages/pytorch_lightning/trainer/call.py", line 38, in _call_and_handle_interrupt
[rank0]: return trainer_fn(*args, **kwargs)
[rank0]: File "/home/tianjian/.conda/envs/unifolm-wma/lib/python3.10/site-packages/pytorch_lightning/trainer/trainer.py", line 650, in _fit_impl
[rank0]: self._run(model, ckpt_path=self.ckpt_path)
[rank0]: File "/home/tianjian/.conda/envs/unifolm-wma/lib/python3.10/site-packages/pytorch_lightning/trainer/trainer.py", line 1112, in _run
[rank0]: results = self._run_stage()
[rank0]: File "/home/tianjian/.conda/envs/unifolm-wma/lib/python3.10/site-packages/pytorch_lightning/trainer/trainer.py", line 1191, in _run_stage
[rank0]: self._run_train()
[rank0]: File "/home/tianjian/.conda/envs/unifolm-wma/lib/python3.10/site-packages/pytorch_lightning/trainer/trainer.py", line 1214, in _run_train
[rank0]: self.fit_loop.run()
[rank0]: File "/home/tianjian/.conda/envs/unifolm-wma/lib/python3.10/site-packages/pytorch_lightning/loops/loop.py", line 199, in run
[rank0]: self.advance(*args, **kwargs)
[rank0]: File "/home/tianjian/.conda/envs/unifolm-wma/lib/python3.10/site-packages/pytorch_lightning/loops/fit_loop.py", line 267, in advance
[rank0]: self._outputs = self.epoch_loop.run(self._data_fetcher)
[rank0]: File "/home/tianjian/.conda/envs/unifolm-wma/lib/python3.10/site-packages/pytorch_lightning/loops/loop.py", line 199, in run
[rank0]: self.advance(*args, **kwargs)
[rank0]: File "/home/tianjian/.conda/envs/unifolm-wma/lib/python3.10/site-packages/pytorch_lightning/loops/epoch/training_epoch_loop.py", line 213, in advance
[rank0]: batch_output = self.batch_loop.run(kwargs)
[rank0]: File "/home/tianjian/.conda/envs/unifolm-wma/lib/python3.10/site-packages/pytorch_lightning/loops/loop.py", line 199, in run
[rank0]: self.advance(*args, **kwargs)
[rank0]: File "/home/tianjian/.conda/envs/unifolm-wma/lib/python3.10/site-packages/pytorch_lightning/loops/batch/training_batch_loop.py", line 88, in advance
[rank0]: outputs = self.optimizer_loop.run(optimizers, kwargs)
[rank0]: File "/home/tianjian/.conda/envs/unifolm-wma/lib/python3.10/site-packages/pytorch_lightning/loops/loop.py", line 199, in run
[rank0]: self.advance(*args, **kwargs)
[rank0]: File "/home/tianjian/.conda/envs/unifolm-wma/lib/python3.10/site-packages/pytorch_lightning/loops/optimization/optimizer_loop.py", line 202, in advance
[rank0]: result = self._run_optimization(kwargs, self._optimizers[self.optim_progress.optimizer_position])
[rank0]: File "/home/tianjian/.conda/envs/unifolm-wma/lib/python3.10/site-packages/pytorch_lightning/loops/optimization/optimizer_loop.py", line 241, in _run_optimization
[rank0]: closure()
[rank0]: File "/home/tianjian/.conda/envs/unifolm-wma/lib/python3.10/site-packages/pytorch_lightning/loops/optimization/optimizer_loop.py", line 149, in call
[rank0]: self._result = self.closure(*args, **kwargs)
[rank0]: File "/home/tianjian/.conda/envs/unifolm-wma/lib/python3.10/site-packages/pytorch_lightning/loops/optimization/optimizer_loop.py", line 135, in closure
[rank0]: step_output = self._step_fn()
[rank0]: File "/home/tianjian/.conda/envs/unifolm-wma/lib/python3.10/site-packages/pytorch_lightning/loops/optimization/optimizer_loop.py", line 419, in _training_step
[rank0]: training_step_output = self.trainer._call_strategy_hook("training_step", *kwargs.values())
[rank0]: File "/home/tianjian/.conda/envs/unifolm-wma/lib/python3.10/site-packages/pytorch_lightning/trainer/trainer.py", line 1494, in _call_strategy_hook
[rank0]: output = fn(*args, **kwargs)
[rank0]: File "/home/tianjian/.conda/envs/unifolm-wma/lib/python3.10/site-packages/pytorch_lightning/strategies/ddp.py", line 351, in training_step
[rank0]: return self.model(*args, **kwargs)
[rank0]: File "/home/tianjian/.conda/envs/unifolm-wma/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1532, in _wrapped_call_impl
[rank0]: return self._call_impl(*args, **kwargs)
[rank0]: File "/home/tianjian/.conda/envs/unifolm-wma/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1541, in _call_impl
[rank0]: return forward_call(*args, **kwargs)
[rank0]: File "/home/tianjian/.conda/envs/unifolm-wma/lib/python3.10/site-packages/fairscale/nn/data_parallel/sharded_ddp.py", line 230, in forward
[rank0]: return self.module(*inputs, **kwargs)
[rank0]: File "/home/tianjian/.conda/envs/unifolm-wma/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1532, in _wrapped_call_impl
[rank0]: return self._call_impl(*args, **kwargs)
[rank0]: File "/home/tianjian/.conda/envs/unifolm-wma/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1541, in _call_impl
[rank0]: return forward_call(*args, **kwargs)
[rank0]: File "/home/tianjian/.conda/envs/unifolm-wma/lib/python3.10/site-packages/pytorch_lightning/overrides/base.py", line 98, in forward
[rank0]: output = self._forward_module.training_step(*inputs, **kwargs)
[rank0]: File "/home/tianjian/vla/unifolm-world-model-action/src/unifolm_wma/models/ddpms.py", line 1413, in training_step
[rank0]: loss, loss_dict = self.shared_step(
[rank0]: File "/home/tianjian/vla/unifolm-world-model-action/src/unifolm_wma/models/ddpms.py", line 2105, in shared_step
[rank0]: loss, loss_dict = self(x, x_action, x_state, c, **kwargs)
[rank0]: File "/home/tianjian/.conda/envs/unifolm-wma/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1532, in _wrapped_call_impl
[rank0]: return self._call_impl(*args, **kwargs)
[rank0]: File "/home/tianjian/.conda/envs/unifolm-wma/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1541, in _call_impl
[rank0]: return forward_call(*args, **kwargs)
[rank0]: File "/home/tianjian/vla/unifolm-world-model-action/src/unifolm_wma/models/ddpms.py", line 1222, in forward
[rank0]: return self.p_losses(x, x_action, x_state, c, t, **kwargs)
[rank0]: File "/home/tianjian/vla/unifolm-world-model-action/src/unifolm_wma/models/ddpms.py", line 1327, in p_losses
[rank0]: model_output, model_action_output, model_state_output = self.apply_model(
[rank0]: File "/home/tianjian/vla/unifolm-world-model-action/src/unifolm_wma/models/ddpms.py", line 1267, in apply_model
[rank0]: x_recon, x_action_recon, x_state_recon = self.model(
[rank0]: File "/home/tianjian/.conda/envs/unifolm-wma/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1532, in _wrapped_call_impl
[rank0]: return self._call_impl(*args, **kwargs)
[rank0]: File "/home/tianjian/.conda/envs/unifolm-wma/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1541, in _call_impl
[rank0]: return forward_call(*args, **kwargs)
[rank0]: File "/home/tianjian/vla/unifolm-world-model-action/src/unifolm_wma/models/ddpms.py", line 2473, in forward
[rank0]: out = self.diffusion_model(xc,
[rank0]: File "/home/tianjian/.conda/envs/unifolm-wma/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1532, in _wrapped_call_impl
[rank0]: return self._call_impl(*args, **kwargs)
[rank0]: File "/home/tianjian/.conda/envs/unifolm-wma/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1541, in _call_impl
[rank0]: return forward_call(*args, **kwargs)
[rank0]: File "/home/tianjian/vla/unifolm-world-model-action/src/unifolm_wma/modules/networks/wma_model.py", line 821, in forward
[rank0]: h = torch.cat([h, hs.pop()], dim=1)
[rank0]: RuntimeError: Sizes of tensors must match except in dimension 1. Expected size 16 but got size 15 for tensor number 1 in the list.
E1010 07:22:08.326000 140564910450496 torch/distributed/elastic/multiprocessing/api.py:826] failed (exitcode: 1) local_rank: 0 (pid: 3529872) of binary: /home/tianjian/.conda/envs/unifolm-wma/bin/python3
Traceback (most recent call last):
File "/home/tianjian/.conda/envs/unifolm-wma/lib/python3.10/runpy.py", line 196, in _run_module_as_main
return _run_code(code, main_globals, None,
File "/home/tianjian/.conda/envs/unifolm-wma/lib/python3.10/runpy.py", line 86, in _run_code
exec(code, run_globals)
File "/home/tianjian/.conda/envs/unifolm-wma/lib/python3.10/site-packages/torch/distributed/launch.py", line 198, in
main()
File "/home/tianjian/.conda/envs/unifolm-wma/lib/python3.10/site-packages/torch/distributed/launch.py", line 194, in main
launch(args)
File "/home/tianjian/.conda/envs/unifolm-wma/lib/python3.10/site-packages/torch/distributed/launch.py", line 179, in launch
run(args)
File "/home/tianjian/.conda/envs/unifolm-wma/lib/python3.10/site-packages/torch/distributed/run.py", line 870, in run
elastic_launch(
File "/home/tianjian/.conda/envs/unifolm-wma/lib/python3.10/site-packages/torch/distributed/launcher/api.py", line 132, in call
return launch_agent(self._config, self._entrypoint, list(args))
File "/home/tianjian/.conda/envs/unifolm-wma/lib/python3.10/site-packages/torch/distributed/launcher/api.py", line 263, in launch_agent
raise ChildFailedError(
torch.distributed.elastic.multiprocessing.errors.ChildFailedError
主要错误是：[rank0]: RuntimeError: Sizes of tensors must match except in dimension 1. Expected size 16 but got size 15 for tensor number 1 in the list.
请问怎么解决

[Behavior-Cloning]

感觉是动作维度你在config or yaml里面没改好

[Mooncakebro]

问问解决了吗？我也遇到这个问题。