Issue after recent update / changes on saving captions
caniyabanci76 opened this issue · comments
Training halts on the first saving of a checkpoint. (i'm not using captions). Output shown below:
`
Adding training captions to checkpoint [Dataloader]
Here comes the checkpoint...
Adding training captions to checkpoint [Dataloader]
Training complete. max_training_steps reached or we blew up.
Traceback (most recent call last):
File "main.py", line 878, in
trainer.fit(model, data)
File "/opt/conda/lib/python3.7/site-packages/pytorch_lightning/trainer/trainer.py", line 771, in fit
self._fit_impl, model, train_dataloaders, val_dataloaders, datamodule, ckpt_path
File "/opt/conda/lib/python3.7/site-packages/pytorch_lightning/trainer/trainer.py", line 723, in _call_and_handle_interrupt
return trainer_fn(*args, **kwargs)
File "/opt/conda/lib/python3.7/site-packages/pytorch_lightning/trainer/trainer.py", line 811, in _fit_impl
results = self._run(model, ckpt_path=self.ckpt_path)
File "/opt/conda/lib/python3.7/site-packages/pytorch_lightning/trainer/trainer.py", line 1236, in _run
results = self._run_stage()
File "/opt/conda/lib/python3.7/site-packages/pytorch_lightning/trainer/trainer.py", line 1323, in _run_stage
return self._run_train()
File "/opt/conda/lib/python3.7/site-packages/pytorch_lightning/trainer/trainer.py", line 1353, in _run_train
self.fit_loop.run()
File "/opt/conda/lib/python3.7/site-packages/pytorch_lightning/loops/base.py", line 204, in run
self.advance(*args, **kwargs)
File "/opt/conda/lib/python3.7/site-packages/pytorch_lightning/loops/fit_loop.py", line 266, in advance
self._outputs = self.epoch_loop.run(self._data_fetcher)
File "/opt/conda/lib/python3.7/site-packages/pytorch_lightning/loops/base.py", line 204, in run
self.advance(*args, **kwargs)
File "/opt/conda/lib/python3.7/site-packages/pytorch_lightning/loops/epoch/training_epoch_loop.py", line 231, in advance
self.trainer._call_callback_hooks("on_train_batch_end", batch_end_outputs, batch, batch_idx, **extra_kwargs)
File "/opt/conda/lib/python3.7/site-packages/pytorch_lightning/trainer/trainer.py", line 1630, in _call_callback_hooks
self._on_train_batch_end(*args, **kwargs)
File "/opt/conda/lib/python3.7/site-packages/pytorch_lightning/trainer/trainer.py", line 1662, in _on_train_batch_end
callback.on_train_batch_end(self, self.lightning_module, outputs, batch, batch_idx)
File "/opt/conda/lib/python3.7/site-packages/pytorch_lightning/callbacks/model_checkpoint.py", line 301, in on_train_batch_end
self._save_last_checkpoint(trainer, monitor_candidates)
File "/opt/conda/lib/python3.7/site-packages/pytorch_lightning/callbacks/model_checkpoint.py", line 644, in _save_last_checkpoint
self._save_checkpoint(trainer, filepath)
File "/opt/conda/lib/python3.7/site-packages/pytorch_lightning/callbacks/model_checkpoint.py", line 384, in _save_checkpoint
trainer.save_checkpoint(filepath, self.save_weights_only)
File "/opt/conda/lib/python3.7/site-packages/pytorch_lightning/trainer/trainer.py", line 2467, in save_checkpoint
self._checkpoint_connector.save_checkpoint(filepath, weights_only=weights_only, storage_options=storage_options)
File "/opt/conda/lib/python3.7/site-packages/pytorch_lightning/trainer/connectors/checkpoint_connector.py", line 444, in save_checkpoint
_checkpoint = self.dump_checkpoint(weights_only)
File "/opt/conda/lib/python3.7/site-packages/pytorch_lightning/trainer/connectors/checkpoint_connector.py", line 422, in dump_checkpoint
self.trainer._call_callbacks_on_save_checkpoint(checkpoint)
File "/opt/conda/lib/python3.7/site-packages/pytorch_lightning/trainer/trainer.py", line 1687, in _call_callbacks_on_save_checkpoint
state = callback.on_save_checkpoint(self, self.lightning_module, checkpoint)
File "/workspace/Dreambooth-Stable-Diffusion/ldm/modules/callbacks/captions.py", line 13, in on_save_checkpoint
for image_path in data.image_paths
AttributeError: 'ConcatDataset' object has no attribute 'image_paths'
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "main.py", line 880, in
melk()
File "main.py", line 857, in melk
trainer.save_checkpoint(ckpt_path)
File "/opt/conda/lib/python3.7/site-packages/pytorch_lightning/trainer/trainer.py", line 2467, in save_checkpoint
self._checkpoint_connector.save_checkpoint(filepath, weights_only=weights_only, storage_options=storage_options)
File "/opt/conda/lib/python3.7/site-packages/pytorch_lightning/trainer/connectors/checkpoint_connector.py", line 444, in save_checkpoint
_checkpoint = self.dump_checkpoint(weights_only)
File "/opt/conda/lib/python3.7/site-packages/pytorch_lightning/trainer/connectors/checkpoint_connector.py", line 422, in dump_checkpoint
self.trainer._call_callbacks_on_save_checkpoint(checkpoint)
File "/opt/conda/lib/python3.7/site-packages/pytorch_lightning/trainer/trainer.py", line 1687, in _call_callbacks_on_save_checkpoint
state = callback.on_save_checkpoint(self, self.lightning_module, checkpoint)
File "/workspace/Dreambooth-Stable-Diffusion/ldm/modules/callbacks/captions.py", line 13, in on_save_checkpoint
for image_path in data.image_paths
AttributeError: 'ConcatDataset' object has no attribute 'image_paths'
`
Same issue here.
Same issue here.
I commented out captions_callback (lines 774,775,776) in main.py just to get it running for the time being.
Same issue here.
I commented out captions_callback (lines 774,775,776) in main.py just to get it running for the time being.
This worked for me as well! Thanks for putting that on here!
Same issue here.
I commented out captions_callback (lines 774,775,776) in main.py just to get it running for the time being.
This worked for me as well, thanks!
This should no longer be a problem now. changes have been reverted.