SpconvOps.point2voxel_cuda() error
LutaoChu opened this issue · comments
I'm getting a SpconvOps error when running project UniDistill.
Version：spconv2.1.25+cumm0.2.9. I also tried ● spconv-cu114-2.1.24, spconv-cu114-2.1.25, spconv-cu111-2.1.25, spconv-cu114-2.1.18 all didn’t work.
Related issues：megvii-research/CVPR2023-UniDistill#9
Traceback (most recent call last):
File "/usr/local/envs/unidistill/lib/python3.6/site-packages/pytorch_lightning/trainer/trainer.py", line 685, in _call_and_handle_interrupt
return trainer_fn(*args, **kwargs)
File "/usr/local/envs/unidistill/lib/python3.6/site-packages/pytorch_lightning/trainer/trainer.py", line 777, in _fit_impl
self._run(model, ckpt_path=ckpt_path)
File "/usr/local/envs/unidistill/lib/python3.6/site-packages/pytorch_lightning/trainer/trainer.py", line 1199, in _run
self._dispatch()
File "/usr/local/envs/unidistill/lib/python3.6/site-packages/pytorch_lightning/trainer/trainer.py", line 1279, in _dispatch
self.training_type_plugin.start_training(self)
File "/usr/local/envs/unidistill/lib/python3.6/site-packages/pytorch_lightning/plugins/training_type/training_type_plugin.py", line 202, in start_training
self._results = trainer.run_stage()
File "/usr/local/envs/unidistill/lib/python3.6/site-packages/pytorch_lightning/trainer/trainer.py", line 1289, in run_stage
return self._run_train()
File "/usr/local/envs/unidistill/lib/python3.6/site-packages/pytorch_lightning/trainer/trainer.py", line 1319, in _run_train
self.fit_loop.run()
File "/usr/local/envs/unidistill/lib/python3.6/site-packages/pytorch_lightning/loops/base.py", line 145, in run
self.advance(*args, **kwargs)
File "/usr/local/envs/unidistill/lib/python3.6/site-packages/pytorch_lightning/loops/fit_loop.py", line 234, in advance
self.epoch_loop.run(data_fetcher)
File "/usr/local/envs/unidistill/lib/python3.6/site-packages/pytorch_lightning/loops/base.py", line 145, in run
self.advance(*args, **kwargs)
File "/usr/local/envs/unidistill/lib/python3.6/site-packages/pytorch_lightning/loops/epoch/training_epoch_loop.py", line 193, in advance
batch_output = self.batch_loop.run(batch, batch_idx)
File "/usr/local/envs/unidistill/lib/python3.6/site-packages/pytorch_lightning/loops/base.py", line 145, in run
self.advance(*args, **kwargs)
File "/usr/local/envs/unidistill/lib/python3.6/site-packages/pytorch_lightning/loops/batch/training_batch_loop.py", line 88, in advance
outputs = self.optimizer_loop.run(split_batch, optimizers, batch_idx)
File "/usr/local/envs/unidistill/lib/python3.6/site-packages/pytorch_lightning/loops/base.py", line 145, in run
self.advance(*args, **kwargs)
File "/usr/local/envs/unidistill/lib/python3.6/site-packages/pytorch_lightning/loops/optimization/optimizer_loop.py", line 219, in advance
self.optimizer_idx,
File "/usr/local/envs/unidistill/lib/python3.6/site-packages/pytorch_lightning/loops/optimization/optimizer_loop.py", line 266, in _run_optimization
self._optimizer_step(optimizer, opt_idx, batch_idx, closure)
File "/usr/local/envs/unidistill/lib/python3.6/site-packages/pytorch_lightning/loops/optimization/optimizer_loop.py", line 386, in _optimizer_step
using_lbfgs=is_lbfgs,
File "/usr/local/envs/unidistill/lib/python3.6/site-packages/pytorch_lightning/core/lightning.py", line 1652, in optimizer_step
optimizer.step(closure=optimizer_closure)
File "/usr/local/envs/unidistill/lib/python3.6/site-packages/pytorch_lightning/core/optimizer.py", line 164, in step
trainer.accelerator.optimizer_step(self._optimizer, self._optimizer_idx, closure, **kwargs)
File "/usr/local/envs/unidistill/lib/python3.6/site-packages/pytorch_lightning/accelerators/accelerator.py", line 339, in optimizer_step
self.precision_plugin.optimizer_step(model, optimizer, opt_idx, closure, **kwargs)
File "/usr/local/envs/unidistill/lib/python3.6/site-packages/pytorch_lightning/plugins/precision/precision_plugin.py", line 163, in optimizer_step
optimizer.step(closure=closure, **kwargs)
File "/usr/local/envs/unidistill/lib/python3.6/site-packages/torch/optim/lr_scheduler.py", line 65, in wrapper
return wrapped(*args, **kwargs)
File "/usr/local/envs/unidistill/lib/python3.6/site-packages/torch/optim/optimizer.py", line 88, in wrapper
return func(*args, **kwargs)
File "/usr/local/envs/unidistill/lib/python3.6/site-packages/torch/autograd/grad_mode.py", line 28, in decorate_context
return func(*args, **kwargs)
File "/usr/local/envs/unidistill/lib/python3.6/site-packages/torch/optim/adamw.py", line 65, in step
loss = closure()
File "/usr/local/envs/unidistill/lib/python3.6/site-packages/pytorch_lightning/plugins/precision/precision_plugin.py", line 148, in _wrap_closure
closure_result = closure()
File "/usr/local/envs/unidistill/lib/python3.6/site-packages/pytorch_lightning/loops/optimization/optimizer_loop.py", line 160, in __call__
self._result = self.closure(*args, **kwargs)
File "/usr/local/envs/unidistill/lib/python3.6/site-packages/pytorch_lightning/loops/optimization/optimizer_loop.py", line 142, in closure
step_output = self._step_fn()
File "/usr/local/envs/unidistill/lib/python3.6/site-packages/pytorch_lightning/loops/optimization/optimizer_loop.py", line 435, in _training_step
training_step_output = self.trainer.accelerator.training_step(step_kwargs)
File "/usr/local/envs/unidistill/lib/python3.6/site-packages/pytorch_lightning/accelerators/accelerator.py", line 219, in training_step
return self.training_type_plugin.training_step(*step_kwargs.values())
File "/usr/local/envs/unidistill/lib/python3.6/site-packages/pytorch_lightning/plugins/training_type/ddp.py", line 439, in training_step
return self.model(*args, **kwargs)
File "/usr/local/envs/unidistill/lib/python3.6/site-packages/torch/nn/modules/module.py", line 1051, in _call_impl
return forward_call(*input, **kwargs)
File "/usr/local/envs/unidistill/lib/python3.6/site-packages/torch/nn/parallel/distributed.py", line 799, in forward
output = self.module(*inputs[0], **kwargs[0])
File "/usr/local/envs/unidistill/lib/python3.6/site-packages/torch/nn/modules/module.py", line 1051, in _call_impl
return forward_call(*input, **kwargs)
File "/usr/local/envs/unidistill/lib/python3.6/site-packages/pytorch_lightning/overrides/base.py", line 81, in forward
output = self.module.training_step(*inputs, **kwargs)
File "/root/UniDistill/unidistill/exps/multisensor_fusion/nuscenes/BEVFusion/BEVFusion_nuscenes_base_exp.py", line 376, in training_step
ret_dict, tf_dict, _, _, _, _ = self(points, imgs, metas, gt_boxes)
File "/usr/local/envs/unidistill/lib/python3.6/site-packages/torch/nn/modules/module.py", line 1051, in _call_impl
return forward_call(*input, **kwargs)
File "/root/UniDistill/unidistill/exps/multisensor_fusion/nuscenes/BEVFusion/BEVFusion_nuscenes_base_exp.py", line 360, in forward
return self.model(points, imgs, metas, gt_boxes)
File "/usr/local/envs/unidistill/lib/python3.6/site-packages/torch/nn/modules/module.py", line 1051, in _call_impl
return forward_call(*input, **kwargs)
File "/root/UniDistill/unidistill/exps/multisensor_fusion/nuscenes/BEVFusion/BEVFusion_nuscenes_centerhead_fusion_exp.py", line 144, in forward
lidar_output = self.lidar_encoder(lidar_points)
File "/usr/local/envs/unidistill/lib/python3.6/site-packages/torch/nn/modules/module.py", line 1051, in _call_impl
return forward_call(*input, **kwargs)
File "/root/UniDistill/unidistill/exps/multisensor_fusion/nuscenes/BEVFusion/BEVFusion_nuscenes_base_exp.py", line 76, in forward
voxels, voxel_coords, voxel_num_points = self.voxelizer(lidar_points)
File "/usr/local/envs/unidistill/lib/python3.6/site-packages/torch/nn/modules/module.py", line 1051, in _call_impl
return forward_call(*input, **kwargs)
File "/root/UniDistill/unidistill/data/det3d/preprocess/voxelization.py", line 54, in forward
voxel_output = self.voxel_generator(p)
File "/usr/local/envs/unidistill/lib/python3.6/site-packages/spconv/pytorch/utils.py", line 88, in __call__
res = self.generate_voxel_with_id(pc, clear_voxels, empty_mean)
File "/usr/local/envs/unidistill/lib/python3.6/site-packages/spconv/pytorch/utils.py", line 139, in generate_voxel_with_id
empty_mean, clear_voxels, stream)
RuntimeError: zero_ /tmp/pip-build-env-jm2xsh18/overlay/lib/python3.6/site-packages/cumm/include/tensorview/tensor.h 220
cuda failed with error 1 invalid argument. use CUDA_LAUNCH_BLOCKING=1 to get correct traceback.
It's urgent. Please help！