NotImplementedError: Support for `validation_epoch_end` has been removed in v2.0.0.
tlightsky opened this issue · comments
(taichi-nerfs) E:\source\py\taichi-nerfs>python train.py --root_dir ./Synthetic_NeRF/Lego --exp_name Lego --perf --num_epochs 20 --batch_size 8192 --lr 1e-2 --no_save_test --gui
[Taichi] version 1.6.0, llvm 15.0.1, commit 98574106, win, python 3.8.16
[Taichi] Starting on arch=cuda
[W 04/05/23 13:54:46.404 57464] [memory_pool.cpp:taichi::lang::MemoryPool::MemoryPool@43] Missing CUDA implementation
GridEncoding: Nmin=16 b=1.31951 F=2 T=2^19 L=16
per_level_scale: 1.3195079107728942
offset_: 5710032
total_hash_size: 11420064
Using 16bit Automatic Mixed Precision (AMP)
GPU available: True (cuda), used: True
TPU available: False, using: 0 TPU cores
IPU available: False, using: 0 IPUs
HPU available: False, using: 0 HPUs
Traceback (most recent call last):
File "train.py", line 307, in <module>
trainer.fit(system)
File "E:\Users\wangc\miniconda3\envs\taichi-nerfs\lib\site-packages\pytorch_lightning\trainer\trainer.py", line 520, in fit
call._call_and_handle_interrupt(
File "E:\Users\wangc\miniconda3\envs\taichi-nerfs\lib\site-packages\pytorch_lightning\trainer\call.py", line 44, in _call_and_handle_interrupt
return trainer_fn(*args, **kwargs)
File "E:\Users\wangc\miniconda3\envs\taichi-nerfs\lib\site-packages\pytorch_lightning\trainer\trainer.py", line 559, in _fit_impl
self._run(model, ckpt_path=ckpt_path)
File "E:\Users\wangc\miniconda3\envs\taichi-nerfs\lib\site-packages\pytorch_lightning\trainer\trainer.py", line 883, in _run
_verify_loop_configurations(self)
File "E:\Users\wangc\miniconda3\envs\taichi-nerfs\lib\site-packages\pytorch_lightning\trainer\configuration_validator.py", line 38, in _verify_loop_configurations
__verify_train_val_loop_configuration(trainer, model)
File "E:\Users\wangc\miniconda3\envs\taichi-nerfs\lib\site-packages\pytorch_lightning\trainer\configuration_validator.py", line 86, in __verify_train_val_loop_configuration
raise NotImplementedError(
NotImplementedError: Support for `validation_epoch_end` has been removed in v2.0.0. `NeRFSystem` implements this method. You can use the `on_validation_epoch_end` hook instead. To access outputs, save them in-memory as instance attributes. You can find migration examples in https://github.com/Lightning-AI/lightning/pull/16520.
fixed after using,
on_validation_epoch_end
seems a break change in 2.0,
maybe version of pytorch-lightning should be under 2.0?
seems not fully fixed
Epoch 19: 100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 1000/1000 [00:12<00:00, 78.52it/s, v_num=0][Taichi] version 1.6.0, llvm 15.0.1, commit 98574106, win, python 3.8.16
[Taichi] version 1.6.0, llvm 15.0.1, commit 98574106, win, python 3.8.16
[Taichi] version 1.6.0, llvm 15.0.1, commit 98574106, win, python 3.8.16
[Taichi] version 1.6.0, llvm 15.0.1, commit 98574106, win, python 3.8.16
[Taichi] version 1.6.0, llvm 15.0.1, commit 98574106, win, python 3.8.16
[Taichi] version 1.6.0, llvm 15.0.1, commit 98574106, win, python 3.8.16
[Taichi] version 1.6.0, llvm 15.0.1, commit 98574106, win, python 3.8.16
[Taichi] version 1.6.0, llvm 15.0.1, commit 98574106, win, python 3.8.16
total training time: 292.19
[Taichi] version 1.6.0, llvm 15.0.1, commit 98574106, win, python 3.8.16
[Taichi] version 1.6.0, llvm 15.0.1, commit 98574106, win, python 3.8.16
[Taichi] version 1.6.0, llvm 15.0.1, commit 98574106, win, python 3.8.16
[Taichi] version 1.6.0, llvm 15.0.1, commit 98574106, win, python 3.8.16
[Taichi] version 1.6.0, llvm 15.0.1, commit 98574106, win, python 3.8.16
[Taichi] version 1.6.0, llvm 15.0.1, commit 98574106, win, python 3.8.16
[Taichi] version 1.6.0, llvm 15.0.1, commit 98574106, win, python 3.8.16
[Taichi] version 1.6.0, llvm 15.0.1, commit 98574106, win, python 3.8.16
Traceback (most recent call last):███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 200/200 [00:10<00:00, 18.43it/s]
File "E:\Users\wangc\miniconda3\envs\taichi-nerfs\lib\site-packages\pytorch_lightning\trainer\call.py", line 44, in _call_and_handle_interrupt
return trainer_fn(*args, **kwargs)
File "E:\Users\wangc\miniconda3\envs\taichi-nerfs\lib\site-packages\pytorch_lightning\trainer\trainer.py", line 559, in _fit_impl
self._run(model, ckpt_path=ckpt_path)
File "E:\Users\wangc\miniconda3\envs\taichi-nerfs\lib\site-packages\pytorch_lightning\trainer\trainer.py", line 935, in _run
results = self._run_stage()
File "E:\Users\wangc\miniconda3\envs\taichi-nerfs\lib\site-packages\pytorch_lightning\trainer\trainer.py", line 978, in _run_stage
self.fit_loop.run()
File "E:\Users\wangc\miniconda3\envs\taichi-nerfs\lib\site-packages\pytorch_lightning\loops\fit_loop.py", line 201, in run
self.advance()
File "E:\Users\wangc\miniconda3\envs\taichi-nerfs\lib\site-packages\pytorch_lightning\loops\fit_loop.py", line 354, in advance
self.epoch_loop.run(self._data_fetcher)
File "E:\Users\wangc\miniconda3\envs\taichi-nerfs\lib\site-packages\pytorch_lightning\loops\training_epoch_loop.py", line 134, in run
self.on_advance_end()
File "E:\Users\wangc\miniconda3\envs\taichi-nerfs\lib\site-packages\pytorch_lightning\loops\training_epoch_loop.py", line 248, in on_advance_end
self.val_loop.run()
File "E:\Users\wangc\miniconda3\envs\taichi-nerfs\lib\site-packages\pytorch_lightning\loops\utilities.py", line 174, in _decorator
return loop_run(self, *args, **kwargs)
File "E:\Users\wangc\miniconda3\envs\taichi-nerfs\lib\site-packages\pytorch_lightning\loops\evaluation_loop.py", line 122, in run
return self.on_run_end()
File "E:\Users\wangc\miniconda3\envs\taichi-nerfs\lib\site-packages\pytorch_lightning\loops\evaluation_loop.py", line 244, in on_run_end
self._on_evaluation_epoch_end()
File "E:\Users\wangc\miniconda3\envs\taichi-nerfs\lib\site-packages\pytorch_lightning\loops\evaluation_loop.py", line 326, in _on_evaluation_epoch_end
call._call_lightning_module_hook(trainer, hook_name)
File "E:\Users\wangc\miniconda3\envs\taichi-nerfs\lib\site-packages\pytorch_lightning\trainer\call.py", line 142, in _call_lightning_module_hook
output = fn(*args, **kwargs)
TypeError: on_validation_epoch_end() missing 1 required positional argument: 'outputs'
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "train.py", line 307, in <module>
trainer.fit(system)
File "E:\Users\wangc\miniconda3\envs\taichi-nerfs\lib\site-packages\pytorch_lightning\trainer\trainer.py", line 520, in fit
call._call_and_handle_interrupt(
File "E:\Users\wangc\miniconda3\envs\taichi-nerfs\lib\site-packages\pytorch_lightning\trainer\call.py", line 68, in _call_and_handle_interrupt
trainer._teardown()
File "E:\Users\wangc\miniconda3\envs\taichi-nerfs\lib\site-packages\pytorch_lightning\trainer\trainer.py", line 958, in _teardown
self.strategy.teardown()
File "E:\Users\wangc\miniconda3\envs\taichi-nerfs\lib\site-packages\pytorch_lightning\strategies\strategy.py", line 475, in teardown
self.lightning_module.cpu()
File "E:\Users\wangc\miniconda3\envs\taichi-nerfs\lib\site-packages\lightning_fabric\utilities\device_dtype_mixin.py", line 78, in cpu
return super().cpu()
File "E:\Users\wangc\miniconda3\envs\taichi-nerfs\lib\site-packages\torch\nn\modules\module.py", line 954, in cpu
return self._apply(lambda t: t.cpu())
File "E:\Users\wangc\miniconda3\envs\taichi-nerfs\lib\site-packages\torch\nn\modules\module.py", line 797, in _apply
module._apply(fn)
File "E:\Users\wangc\miniconda3\envs\taichi-nerfs\lib\site-packages\torch\nn\modules\module.py", line 797, in _apply
module._apply(fn)
File "E:\Users\wangc\miniconda3\envs\taichi-nerfs\lib\site-packages\torch\nn\modules\module.py", line 844, in _apply
self._buffers[key] = fn(buf)
File "E:\Users\wangc\miniconda3\envs\taichi-nerfs\lib\site-packages\torch\nn\modules\module.py", line 954, in <lambda>
return self._apply(lambda t: t.cpu())
RuntimeError: [enforce fail at C:\actions-runner\_work\pytorch\pytorch\builder\windows\pytorch\c10\core\impl\alloc_cpu.cpp:72] data. DefaultCPUAllocator: not enough memory: you tried to allocate 1073741824 bytes.
my temporary fix-up for this, in train.py
@@ -62,6 +62,7 @@ class NeRFSystem(LightningModule):
self.test_id = 0
self.val_dir = f'results/{self.hparams.dataset_name}/{self.hparams.exp_name}/training'
os.makedirs(self.val_dir, exist_ok=True)
+ self.validation_step_outputs =[]
def forward(self, batch, split):
if split == 'train':
@@ -249,10 +250,11 @@ class NeRFSystem(LightningModule):
rgb_pred)
imageio.imsave(os.path.join(self.val_dir, f'depth_{idx:03d}.png'),
depth)
-
+ self.validation_step_outputs.append(logs)
return logs
- def validation_epoch_end(self, outputs):
+ def on_validation_epoch_end(self):
+ outputs=self.validation_step_outputs
psnrs = torch.stack([x['psnr'] for x in outputs])
mean_psnr = psnrs.mean()
self.log('test/psnr', mean_psnr, True)It seems memory allocation is also an issue:
DefaultCPUAllocator: not enough memory: you tried to allocate 1073741824 bytes.
my spec is like:
i9-12900K, 64G mem, 3090
switch pytorch to 11.7
can got some output mp4 and jpg,
still have some error:
[Taichi] version 1.6.0, llvm 15.0.1, commit 98574106, win, python 3.8.16
[Taichi] version 1.6.0, llvm 15.0.1, commit 98574106, win, python 3.8.16
[Taichi] version 1.6.0, llvm 15.0.1, commit 98574106, win, python 3.8.16
[Taichi] version 1.6.0, llvm 15.0.1, commit 98574106, win, python 3.8.16
[Taichi] version 1.6.0, llvm 15.0.1, commit 98574106, win, python 3.8.16
[Taichi] version 1.6.0, llvm 15.0.1, commit 98574106, win, python 3.8.16
[Taichi] version 1.6.0, llvm 15.0.1, commit 98574106, win, python 3.8.16
[Taichi] version 1.6.0, llvm 15.0.1, commit 98574106, win, python 3.8.16
Epoch 19: : 1003it [01:28, 11.28it/s, loss=0.000181]Traceback (most recent call last):
File "E:\Users\wangc\miniconda3\envs\taichi-nerfs-117\lib\site-packages\torch\utils\data\dataloader.py", line 1120, in _try_get_data | 3/200 [00:00<00:59, 3.30it/s]
data = self._data_queue.get(timeout=timeout)
File "E:\Users\wangc\miniconda3\envs\taichi-nerfs-117\lib\queue.py", line 178, in get
raise Empty
_queue.Empty
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "train.py", line 307, in <module>
trainer.fit(system)
File "E:\Users\wangc\miniconda3\envs\taichi-nerfs-117\lib\site-packages\pytorch_lightning\trainer\trainer.py", line 696, in fit
self._call_and_handle_interrupt(
File "E:\Users\wangc\miniconda3\envs\taichi-nerfs-117\lib\site-packages\pytorch_lightning\trainer\trainer.py", line 650, in _call_and_handle_interrupt
return trainer_fn(*args, **kwargs)
File "E:\Users\wangc\miniconda3\envs\taichi-nerfs-117\lib\site-packages\pytorch_lightning\trainer\trainer.py", line 735, in _fit_impl
results = self._run(model, ckpt_path=self.ckpt_path)
File "E:\Users\wangc\miniconda3\envs\taichi-nerfs-117\lib\site-packages\pytorch_lightning\trainer\trainer.py", line 1166, in _run
results = self._run_stage()
File "E:\Users\wangc\miniconda3\envs\taichi-nerfs-117\lib\site-packages\pytorch_lightning\trainer\trainer.py", line 1252, in _run_stage
return self._run_train()
File "E:\Users\wangc\miniconda3\envs\taichi-nerfs-117\lib\site-packages\pytorch_lightning\trainer\trainer.py", line 1283, in _run_train
self.fit_loop.run()
File "E:\Users\wangc\miniconda3\envs\taichi-nerfs-117\lib\site-packages\pytorch_lightning\loops\loop.py", line 200, in run
self.advance(*args, **kwargs)
File "E:\Users\wangc\miniconda3\envs\taichi-nerfs-117\lib\site-packages\pytorch_lightning\loops\fit_loop.py", line 271, in advance
self._outputs = self.epoch_loop.run(self._data_fetcher)
File "E:\Users\wangc\miniconda3\envs\taichi-nerfs-117\lib\site-packages\pytorch_lightning\loops\loop.py", line 201, in run
self.on_advance_end()
File "E:\Users\wangc\miniconda3\envs\taichi-nerfs-117\lib\site-packages\pytorch_lightning\loops\epoch\training_epoch_loop.py", line 241, in on_advance_end
self._run_validation()
File "E:\Users\wangc\miniconda3\envs\taichi-nerfs-117\lib\site-packages\pytorch_lightning\loops\epoch\training_epoch_loop.py", line 299, in _run_validation
self.val_loop.run()
File "E:\Users\wangc\miniconda3\envs\taichi-nerfs-117\lib\site-packages\pytorch_lightning\loops\loop.py", line 200, in run
self.advance(*args, **kwargs)
File "E:\Users\wangc\miniconda3\envs\taichi-nerfs-117\lib\site-packages\pytorch_lightning\loops\dataloader\evaluation_loop.py", line 155, in advance
dl_outputs = self.epoch_loop.run(self._data_fetcher, dl_max_batches, kwargs)
File "E:\Users\wangc\miniconda3\envs\taichi-nerfs-117\lib\site-packages\pytorch_lightning\loops\loop.py", line 200, in run
self.advance(*args, **kwargs)
File "E:\Users\wangc\miniconda3\envs\taichi-nerfs-117\lib\site-packages\pytorch_lightning\loops\epoch\evaluation_epoch_loop.py", line 127, in advance
batch = next(data_fetcher)
File "E:\Users\wangc\miniconda3\envs\taichi-nerfs-117\lib\site-packages\pytorch_lightning\utilities\fetching.py", line 184, in __next__
return self.fetching_function()
File "E:\Users\wangc\miniconda3\envs\taichi-nerfs-117\lib\site-packages\pytorch_lightning\utilities\fetching.py", line 263, in fetching_function
self._fetch_next_batch(self.dataloader_iter)
File "E:\Users\wangc\miniconda3\envs\taichi-nerfs-117\lib\site-packages\pytorch_lightning\utilities\fetching.py", line 277, in _fetch_next_batch
batch = next(iterator)
File "E:\Users\wangc\miniconda3\envs\taichi-nerfs-117\lib\site-packages\torch\utils\data\dataloader.py", line 628, in __next__
data = self._next_data()
File "E:\Users\wangc\miniconda3\envs\taichi-nerfs-117\lib\site-packages\torch\utils\data\dataloader.py", line 1316, in _next_data
idx, data = self._get_data()
File "E:\Users\wangc\miniconda3\envs\taichi-nerfs-117\lib\site-packages\torch\utils\data\dataloader.py", line 1272, in _get_data
success, data = self._try_get_data()
File "E:\Users\wangc\miniconda3\envs\taichi-nerfs-117\lib\site-packages\torch\utils\data\dataloader.py", line 1133, in _try_get_data
raise RuntimeError('DataLoader worker (pid(s) {}) exited unexpectedly'.format(pids_str)) from e
RuntimeError: DataLoader worker (pid(s) 80508, 82040, 45060, 68340) exited unexpectedly
The problem seems to relate to the dataloader. Would you may try setting the number of work of the dataloader to a smaller value, like 1.
The problem seems to relate to the dataloader. Would you may try setting the number of work of the dataloader to a smaller value, like
1.
not sure why but it did fix the issue, thanks