在windows环境下训练时报错
zzhbb2002 opened this issue · comments
Traceback (most recent call last):
File "train.py", line 336, in
main()
File "train.py", line 55, in main
mp.spawn(run, nprocs=n_gpus, args=(n_gpus, hps,))
File "C:\Users\zzh\AppData\Local\Programs\Python\Python38\lib\site-packages\torch\multiprocessing\spawn.py", line 230, in spawn
return start_processes(fn, args, nprocs, join, daemon, start_method='spawn')
File "C:\Users\zzh\AppData\Local\Programs\Python\Python38\lib\site-packages\torch\multiprocessing\spawn.py", line 188, in start_processes
while not context.join():
File "C:\Users\zzh\AppData\Local\Programs\Python\Python38\lib\site-packages\torch\multiprocessing\spawn.py", line 150, in join
raise ProcessRaisedException(msg, error_index, failed_process.pid)
torch.multiprocessing.spawn.ProcessRaisedException:
-- Process 0 terminated with the following error:
Traceback (most recent call last):
File "C:\Users\zzh\AppData\Local\Programs\Python\Python38\lib\site-packages\torch\multiprocessing\spawn.py", line 59, in _wrap
fn(i, *args)
File "C:\Users\zzh\Desktop\ai\Sovits-main\train.py", line 135, in run
train_and_evaluate(rank, epoch, hps, [net_g, net_d, F0_model], use_F0_model, [optim_g, optim_d], [scheduler_g, scheduler_d], scaler,
UnboundLocalError: local variable 'F0_model' referenced before assignment
谢谢大佬的回复，在修改后出现了如下问题
Traceback (most recent call last):
File "C:\Users\zzh\AppData\Local\Programs\Python\Python38\lib\site-packages\torch\utils\data\dataloader.py", line 1328, in del
self._shutdown_workers()
File "C:\Users\zzh\AppData\Local\Programs\Python\Python38\lib\site-packages\torch\utils\data\dataloader.py", line 1268, in _shutdown_workers
if not self._shutdown:
AttributeError: '_MultiProcessingDataLoaderIter' object has no attribute '_shutdown'
Traceback (most recent call last):
File "train.py", line 340, in
main()
File "train.py", line 56, in main
mp.spawn(run, nprocs=n_gpus, args=(n_gpus, hps,))
File "C:\Users\zzh\AppData\Local\Programs\Python\Python38\lib\site-packages\torch\multiprocessing\spawn.py", line 230, in spawn
return start_processes(fn, args, nprocs, join, daemon, start_method='spawn')
File "C:\Users\zzh\AppData\Local\Programs\Python\Python38\lib\site-packages\torch\multiprocessing\spawn.py", line 188, in start_processes
while not context.join():
File "C:\Users\zzh\AppData\Local\Programs\Python\Python38\lib\site-packages\torch\multiprocessing\spawn.py", line 150, in join
raise ProcessRaisedException(msg, error_index, failed_process.pid)
torch.multiprocessing.spawn.ProcessRaisedException:
-- Process 0 terminated with the following error:
Traceback (most recent call last):
File "C:\Users\zzh\AppData\Local\Programs\Python\Python38\lib\site-packages\torch\multiprocessing\spawn.py", line 59, in _wrap
fn(i, *args)
File "C:\Users\zzh\Desktop\ai\Sovits-main\train.py", line 137, in run
train_and_evaluate(rank, epoch, hps, [net_g, net_d, F0_model], use_F0_model, [optim_g, optim_d],
File "C:\Users\zzh\Desktop\ai\Sovits-main\train.py", line 161, in train_and_evaluate
for batch_idx, (x, x_lengths, spec, spec_lengths, y, y_lengths) in enumerate(train_loader):
File "C:\Users\zzh\AppData\Local\Programs\Python\Python38\lib\site-packages\torch\utils\data\dataloader.py", line 359, in iter
return self._get_iterator()
File "C:\Users\zzh\AppData\Local\Programs\Python\Python38\lib\site-packages\torch\utils\data\dataloader.py", line 305, in _get_iterator
return _MultiProcessingDataLoaderIter(self)
File "C:\Users\zzh\AppData\Local\Programs\Python\Python38\lib\site-packages\torch\utils\data\dataloader.py", line 878, in init
super(_MultiProcessingDataLoaderIter, self).init(loader)
File "C:\Users\zzh\AppData\Local\Programs\Python\Python38\lib\site-packages\torch\utils\data\dataloader.py", line 497, in init
self._sampler_iter = iter(self._index_sampler)
File "C:\Users\zzh\Desktop\ai\Sovits-main\data_utils.py", line 296, in iter
ids_bucket = ids_bucket + ids_bucket * (rem // len_bucket) + ids_bucket[:(rem % len_bucket)]
ZeroDivisionError: integer division or modulo by zero
谢谢大佬的回复，在修改后出现了如下问题 Traceback (most recent call last): File "C:\Users\zzh\AppData\Local\Programs\Python\Python38\lib\site-packages\torch\utils\data\dataloader.py", line 1328, in del self._shutdown_workers() File "C:\Users\zzh\AppData\Local\Programs\Python\Python38\lib\site-packages\torch\utils\data\dataloader.py", line 1268, in _shutdown_workers if not self._shutdown: AttributeError: '_MultiProcessingDataLoaderIter' object has no attribute '_shutdown' Traceback (most recent call last): File "train.py", line 340, in main() File "train.py", line 56, in main mp.spawn(run, nprocs=n_gpus, args=(n_gpus, hps,)) File "C:\Users\zzh\AppData\Local\Programs\Python\Python38\lib\site-packages\torch\multiprocessing\spawn.py", line 230, in spawn return start_processes(fn, args, nprocs, join, daemon, start_method='spawn') File "C:\Users\zzh\AppData\Local\Programs\Python\Python38\lib\site-packages\torch\multiprocessing\spawn.py", line 188, in start_processes while not context.join(): File "C:\Users\zzh\AppData\Local\Programs\Python\Python38\lib\site-packages\torch\multiprocessing\spawn.py", line 150, in join raise ProcessRaisedException(msg, error_index, failed_process.pid) torch.multiprocessing.spawn.ProcessRaisedException:
-- Process 0 terminated with the following error: Traceback (most recent call last): File "C:\Users\zzh\AppData\Local\Programs\Python\Python38\lib\site-packages\torch\multiprocessing\spawn.py", line 59, in _wrap fn(i, *args) File "C:\Users\zzh\Desktop\ai\Sovits-main\train.py", line 137, in run train_and_evaluate(rank, epoch, hps, [net_g, net_d, F0_model], use_F0_model, [optim_g, optim_d], File "C:\Users\zzh\Desktop\ai\Sovits-main\train.py", line 161, in train_and_evaluate for batch_idx, (x, x_lengths, spec, spec_lengths, y, y_lengths) in enumerate(train_loader): File "C:\Users\zzh\AppData\Local\Programs\Python\Python38\lib\site-packages\torch\utils\data\dataloader.py", line 359, in iter return self._get_iterator() File "C:\Users\zzh\AppData\Local\Programs\Python\Python38\lib\site-packages\torch\utils\data\dataloader.py", line 305, in _get_iterator return _MultiProcessingDataLoaderIter(self) File "C:\Users\zzh\AppData\Local\Programs\Python\Python38\lib\site-packages\torch\utils\data\dataloader.py", line 878, in init super(_MultiProcessingDataLoaderIter, self).init(loader) File "C:\Users\zzh\AppData\Local\Programs\Python\Python38\lib\site-packages\torch\utils\data\dataloader.py", line 497, in init self._sampler_iter = iter(self._index_sampler) File "C:\Users\zzh\Desktop\ai\Sovits-main\data_utils.py", line 296, in iter ids_bucket = ids_bucket + ids_bucket * (rem // len_bucket) + ids_bucket[:(rem % len_bucket)] ZeroDivisionError: integer division or modulo by zero
这个可能是你的数据集有些小问题，可以试试让你的训练集中数据的条数能被batchsize整除
谢谢大佬，问题解决了，但提示显存不足
RuntimeError: CUDA out of memory. Tried to allocate 148.00 MiB (GPU 0; 6.00 GiB total capacity; 1.26 GiB already allocated; 2.53 GiB free; 1.40 GiB reserved in total by PyTorch)
根据报错代码来看显存是足够的，这是为什么呢？
Traceback (most recent call last):
File "train.py", line 340, in
main()
File "train.py", line 56, in main
mp.spawn(run, nprocs=n_gpus, args=(n_gpus, hps,))
File "C:\Users\zzh\AppData\Local\Programs\Python\Python38\lib\site-packages\torch\multiprocessing\spawn.py", line 230, in spawn
return start_processes(fn, args, nprocs, join, daemon, start_method='spawn')
File "C:\Users\zzh\AppData\Local\Programs\Python\Python38\lib\site-packages\torch\multiprocessing\spawn.py", line 188, in start_processes
while not context.join():
File "C:\Users\zzh\AppData\Local\Programs\Python\Python38\lib\site-packages\torch\multiprocessing\spawn.py", line 150, in join
raise ProcessRaisedException(msg, error_index, failed_process.pid)
torch.multiprocessing.spawn.ProcessRaisedException:
-- Process 0 terminated with the following error:
Traceback (most recent call last):
File "C:\Users\zzh\AppData\Local\Programs\Python\Python38\lib\site-packages\torch\multiprocessing\spawn.py", line 59, in _wrap
fn(i, *args)
File "C:\Users\zzh\Desktop\ai\Sovits-main\train.py", line 137, in run
train_and_evaluate(rank, epoch, hps, [net_g, net_d, F0_model], use_F0_model, [optim_g, optim_d],
File "C:\Users\zzh\Desktop\ai\Sovits-main\train.py", line 179, in train_and_evaluate
(z, z_p, m_p, logs_p, m_q, logs_q) = net_g(x, x_lengths, spec, spec_lengths, F0=F0)
File "C:\Users\zzh\AppData\Local\Programs\Python\Python38\lib\site-packages\torch\nn\modules\module.py", line 1051, in _call_impl
return forward_call(*input, **kwargs)
File "C:\Users\zzh\AppData\Local\Programs\Python\Python38\lib\site-packages\torch\nn\parallel\distributed.py", line 799, in forward
output = self.module(*inputs[0], **kwargs[0])
File "C:\Users\zzh\AppData\Local\Programs\Python\Python38\lib\site-packages\torch\nn\modules\module.py", line 1051, in _call_impl
return forward_call(*input, **kwargs)
File "C:\Users\zzh\Desktop\ai\Sovits-main\models.py", line 504, in forward
z, m_q, logs_q, y_mask = self.enc_q(y, y_lengths, g=g)
File "C:\Users\zzh\AppData\Local\Programs\Python\Python38\lib\site-packages\torch\nn\modules\module.py", line 1051, in _call_impl
return forward_call(*input, **kwargs)
File "C:\Users\zzh\Desktop\ai\Sovits-main\models.py", line 261, in forward
x = self.enc(x, x_mask, g=g)
File "C:\Users\zzh\AppData\Local\Programs\Python\Python38\lib\site-packages\torch\nn\modules\module.py", line 1051, in _call_impl
return forward_call(*input, **kwargs)
File "C:\Users\zzh\Desktop\ai\Sovits-main\modules.py", line 156, in forward
x_in = self.in_layersi
File "C:\Users\zzh\AppData\Local\Programs\Python\Python38\lib\site-packages\torch\nn\modules\module.py", line 1071, in _call_impl
result = forward_call(*input, **kwargs)
File "C:\Users\zzh\AppData\Local\Programs\Python\Python38\lib\site-packages\torch\nn\modules\conv.py", line 298, in forward
return self._conv_forward(input, self.weight, self.bias)
File "C:\Users\zzh\AppData\Local\Programs\Python\Python38\lib\site-packages\torch\nn\modules\conv.py", line 294, in _conv_forward
return F.conv1d(input, weight, bias, self.stride,
RuntimeError: CUDA out of memory. Tried to allocate 148.00 MiB (GPU 0; 6.00 GiB total capacity; 1.26 GiB already allocated; 2.53 GiB free; 1.40 GiB reserved in total by PyTorch)
不好意思回复晚了。这个问题我也遇到过，当时16G的显存只能用12G左右，多了就会报错。我暂时没有靠谱的解决方法，只能把batchsize调小了。
谢谢大佬，那看来电脑配置有限运作不了，麻烦大佬了