RuntimeError: CUDA error: out of memory #31
Description
Hi,zhuoyanZhao
I met a problem when I run the demo,
here is the error:
RuntimeError Traceback (most recent call last)
in
----> 1 run_tasks('./config.yml')
~/.local/lib/python3.6/site-packages/nest/parser.py in run_tasks(config_file, param_file, verbose)
135 logger.info('Finished (%s).' % (U.format_elapse(seconds=(end_time - param_start_time).total_seconds())))
136 else:
--> 137 resolved_config = parse_config(config, env_vars=env_vars)
138 check_all_resolved(resolved_config)
139
~/.local/lib/python3.6/site-packages/nest/parser.py in parse_config(config, env_vars, global_vars)
68 return nest_module(**config)
69 else:
---> 70 return nest_module(**config, delay_resolve=True)
71
72 return config
~/.local/lib/python3.6/site-packages/nest/modules.py in call(self, *args, **kwargs)
160 try:
161 self._check_params(resolved_params)
--> 162 returns = self.func(**resolved_params)
163 except KeyError as exc_info:
164 if 'Nest module' in str(exc_info):
~/Nest-pytorch/install/trainer.py in network_trainer(data_loaders, model, criterion, optimizer, parameter, meters, hooks, max_epoch, test_interval, resume, log_path, device, use_data_parallel, use_cudnn_benchmark, random_seed)
81
82 # setup model
---> 83 model = model.to(device)
84
85 # multi-gpu support
~/.local/lib/python3.6/site-packages/torch/nn/modules/module.py in to(self, *args, **kwargs)
423 return t.to(device, dtype if t.is_floating_point() else None, non_blocking)
424
--> 425 return self._apply(convert)
426
427 def register_backward_hook(self, hook):
~/.local/lib/python3.6/site-packages/torch/nn/modules/module.py in _apply(self, fn)
199 def _apply(self, fn):
200 for module in self.children():
--> 201 module._apply(fn)
202
203 def compute_should_use_set_data(tensor, tensor_applied):
~/.local/lib/python3.6/site-packages/torch/nn/modules/module.py in _apply(self, fn)
199 def _apply(self, fn):
200 for module in self.children():
--> 201 module._apply(fn)
202
203 def compute_should_use_set_data(tensor, tensor_applied):
~/.local/lib/python3.6/site-packages/torch/nn/modules/module.py in _apply(self, fn)
199 def _apply(self, fn):
200 for module in self.children():
--> 201 module._apply(fn)
202
203 def compute_should_use_set_data(tensor, tensor_applied):
~/.local/lib/python3.6/site-packages/torch/nn/modules/module.py in _apply(self, fn)
221 # with torch.no_grad():
222 with torch.no_grad():
--> 223 param_applied = fn(param)
224 should_use_set_data = compute_should_use_set_data(param, param_applied)
225 if should_use_set_data:
~/.local/lib/python3.6/site-packages/torch/nn/modules/module.py in convert(t)
421
422 def convert(t):
--> 423 return t.to(device, dtype if t.is_floating_point() else None, non_blocking)
424
425 return self._apply(convert)
RuntimeError: CUDA error: out of memory
I have one GPU: GeForce RTX 2060
python :3.6.9
It is really weird it has out of memory before training. could you please give me some suggestions?