train can not run ,it start but it doesn't run anymore
kairenchen123 opened this issue · comments
loading all datasets...
using 4 threads
loading from cache file: cache/coco_trainval2014.pkl
loading annotations into memory...
Done (t=14.38s)
creating index...
index created!
loading from cache file: cache/coco_trainval2014.pkl
loading annotations into memory...
Done (t=15.31s)
creating index...
index created!
loading from cache file: cache/coco_trainval2014.pkl
loading annotations into memory...
Done (t=15.66s)
creating index...
index created!
loading from cache file: cache/coco_trainval2014.pkl
loading annotations into memory...
Done (t=13.77s)
creating index...
index created!
loading from cache file: cache/coco_minival2014.pkl
loading annotations into memory...
Done (t=0.42s)
creating index...
index created!
system config...
{'batch_size': 48,
'cache_dir': 'cache',
'chunk_sizes': [6, 6, 6, 6, 6, 6, 6, 6],
'config_dir': 'config',
'data_dir': './data',
'data_rng': <mtrand.RandomState object at 0x7f577acd47e0>,
'dataset': 'MSCOCO',
'decay_rate': 10,
'display': 5,
'learning_rate': 0.00025,
'max_iter': 480000,
'nnet_rng': <mtrand.RandomState object at 0x7f577acd4828>,
'opt_algo': 'adam',
'prefetch_size': 6,
'pretrain': None,
'result_dir': 'results',
'sampling_function': 'kp_detection',
'snapshot': 5000,
'snapshot_name': 'CenterNet-52',
'stepsize': 450000,
'test_split': 'testdev',
'train_split': 'trainval',
'val_iter': 500,
'val_split': 'minival',
'weight_decay': False,
'weight_decay_rate': 1e-05,
'weight_decay_type': 'l2'}
db config...
{'ae_threshold': 0.5,
'border': 128,
'categories': 80,
'data_aug': True,
'gaussian_bump': True,
'gaussian_iou': 0.7,
'gaussian_radius': -1,
'input_size': [511, 511],
'kp_categories': 1,
'lighting': True,
'max_per_image': 100,
'merge_bbox': False,
'nms_algorithm': 'exp_soft_nms',
'nms_kernel': 3,
'nms_threshold': 0.5,
'output_sizes': [[128, 128]],
'rand_color': True,
'rand_crop': True,
'rand_pushes': False,
'rand_samples': False,
'rand_scale_max': 1.4,
'rand_scale_min': 0.6,
'rand_scale_step': 0.1,
'rand_scales': array([0.6, 0.7, 0.8, 0.9, 1. , 1.1, 1.2, 1.3]),
'special_crop': False,
'test_scales': [1],
'top_k': 70,
'weight_exp': 8}
len of db: 118287
start prefetching data...
shuffling indices...
start prefetching data...
Traceback (most recent call last):
File "train.py", line 43, in prefetch_data
data, ind = sample_data(db, ind, data_aug=data_aug)
File "/root/autodl-tmp/CenterNet/sample/coco.py", line 199, in sample_data
return globals()[system_configs.sampling_function](db, k_ind, data_aug, debug)
File "/root/autodl-tmp/CenterNet/sample/coco.py", line 99, in kp_detection
image, detections = random_crop(image, detections, rand_scales, input_size, border=border)
File "/root/autodl-tmp/CenterNet/sample/utils.py", line 57, in random_crop
image_height, image_width = image.shape[0:2]
AttributeError: 'NoneType' object has no attribute 'shape'
Process Process-1:
Traceback (most recent call last):
File "/root/miniconda3/envs/CenterNet/lib/python3.6/multiprocessing/process.py", line 258, in _bootstrap
self.run()
File "/root/miniconda3/envs/CenterNet/lib/python3.6/multiprocessing/process.py", line 93, in run
self._target(*self._args, **self._kwargs)
File "train.py", line 47, in prefetch_data
raise e
File "train.py", line 43, in prefetch_data
data, ind = sample_data(db, ind, data_aug=data_aug)
File "/root/autodl-tmp/CenterNet/sample/coco.py", line 199, in sample_data
return globals()[system_configs.sampling_function](db, k_ind, data_aug, debug)
File "/root/autodl-tmp/CenterNet/sample/coco.py", line 99, in kp_detection
image, detections = random_crop(image, detections, rand_scales, input_size, border=border)
File "/root/autodl-tmp/CenterNet/sample/utils.py", line 57, in random_crop
image_height, image_width = image.shape[0:2]
AttributeError: 'NoneType' object has no attribute 'shape'
shuffling indices...
start prefetching data...
Traceback (most recent call last):
File "train.py", line 43, in prefetch_data
data, ind = sample_data(db, ind, data_aug=data_aug)
File "/root/autodl-tmp/CenterNet/sample/coco.py", line 199, in sample_data
return globals()[system_configs.sampling_function](db, k_ind, data_aug, debug)
File "/root/autodl-tmp/CenterNet/sample/coco.py", line 99, in kp_detection
image, detections = random_crop(image, detections, rand_scales, input_size, border=border)
File "/root/autodl-tmp/CenterNet/sample/utils.py", line 57, in random_crop
image_height, image_width = image.shape[0:2]
AttributeError: 'NoneType' object has no attribute 'shape'
Process Process-2:
Traceback (most recent call last):
File "/root/miniconda3/envs/CenterNet/lib/python3.6/multiprocessing/process.py", line 258, in _bootstrap
self.run()
File "/root/miniconda3/envs/CenterNet/lib/python3.6/multiprocessing/process.py", line 93, in run
self._target(*self._args, **self._kwargs)
File "train.py", line 47, in prefetch_data
raise e
File "train.py", line 43, in prefetch_data
data, ind = sample_data(db, ind, data_aug=data_aug)
File "/root/autodl-tmp/CenterNet/sample/coco.py", line 199, in sample_data
return globals()[system_configs.sampling_function](db, k_ind, data_aug, debug)
File "/root/autodl-tmp/CenterNet/sample/coco.py", line 99, in kp_detection
image, detections = random_crop(image, detections, rand_scales, input_size, border=border)
File "/root/autodl-tmp/CenterNet/sample/utils.py", line 57, in random_crop
image_height, image_width = image.shape[0:2]
AttributeError: 'NoneType' object has no attribute 'shape'
shuffling indices...
Traceback (most recent call last):
File "train.py", line 43, in prefetch_data
data, ind = sample_data(db, ind, data_aug=data_aug)
File "/root/autodl-tmp/CenterNet/sample/coco.py", line 199, in sample_data
return globals()[system_configs.sampling_function](db, k_ind, data_aug, debug)
File "/root/autodl-tmp/CenterNet/sample/coco.py", line 99, in kp_detection
image, detections = random_crop(image, detections, rand_scales, input_size, border=border)
File "/root/autodl-tmp/CenterNet/sample/utils.py", line 57, in random_crop
image_height, image_width = image.shape[0:2]
AttributeError: 'NoneType' object has no attribute 'shape'
Process Process-3:
Traceback (most recent call last):
File "/root/miniconda3/envs/CenterNet/lib/python3.6/multiprocessing/process.py", line 258, in _bootstrap
self.run()
File "/root/miniconda3/envs/CenterNet/lib/python3.6/multiprocessing/process.py", line 93, in run
self._target(*self._args, **self._kwargs)
File "train.py", line 47, in prefetch_data
raise e
File "train.py", line 43, in prefetch_data
data, ind = sample_data(db, ind, data_aug=data_aug)
File "/root/autodl-tmp/CenterNet/sample/coco.py", line 199, in sample_data
return globals()[system_configs.sampling_function](db, k_ind, data_aug, debug)
File "/root/autodl-tmp/CenterNet/sample/coco.py", line 99, in kp_detection
image, detections = random_crop(image, detections, rand_scales, input_size, border=border)
File "/root/autodl-tmp/CenterNet/sample/utils.py", line 57, in random_crop
image_height, image_width = image.shape[0:2]
AttributeError: 'NoneType' object has no attribute 'shape'
start prefetching data...
shuffling indices...
building model...
module_file: models.CenterNet-52
start prefetching data...
Traceback (most recent call last):
File "train.py", line 43, in prefetch_data
data, ind = sample_data(db, ind, data_aug=data_aug)
File "/root/autodl-tmp/CenterNet/sample/coco.py", line 199, in sample_data
return globals()[system_configs.sampling_function](db, k_ind, data_aug, debug)
File "/root/autodl-tmp/CenterNet/sample/coco.py", line 99, in kp_detection
image, detections = random_crop(image, detections, rand_scales, input_size, border=border)
File "/root/autodl-tmp/CenterNet/sample/utils.py", line 57, in random_crop
image_height, image_width = image.shape[0:2]
AttributeError: 'NoneType' object has no attribute 'shape'
Process Process-4:
Traceback (most recent call last):
File "/root/miniconda3/envs/CenterNet/lib/python3.6/multiprocessing/process.py", line 258, in _bootstrap
self.run()
File "/root/miniconda3/envs/CenterNet/lib/python3.6/multiprocessing/process.py", line 93, in run
self._target(*self._args, **self._kwargs)
File "train.py", line 47, in prefetch_data
raise e
File "train.py", line 43, in prefetch_data
data, ind = sample_data(db, ind, data_aug=data_aug)
File "/root/autodl-tmp/CenterNet/sample/coco.py", line 199, in sample_data
return globals()[system_configs.sampling_function](db, k_ind, data_aug, debug)
File "/root/autodl-tmp/CenterNet/sample/coco.py", line 99, in kp_detection
image, detections = random_crop(image, detections, rand_scales, input_size, border=border)
File "/root/autodl-tmp/CenterNet/sample/utils.py", line 57, in random_crop
image_height, image_width = image.shape[0:2]
AttributeError: 'NoneType' object has no attribute 'shape'
shuffling indices...
Traceback (most recent call last):
File "train.py", line 43, in prefetch_data
data, ind = sample_data(db, ind, data_aug=data_aug)
File "/root/autodl-tmp/CenterNet/sample/coco.py", line 199, in sample_data
return globals()[system_configs.sampling_function](db, k_ind, data_aug, debug)
File "/root/autodl-tmp/CenterNet/sample/coco.py", line 99, in kp_detection
image, detections = random_crop(image, detections, rand_scales, input_size, border=border)
File "/root/autodl-tmp/CenterNet/sample/utils.py", line 57, in random_crop
image_height, image_width = image.shape[0:2]
AttributeError: 'NoneType' object has no attribute 'shape'
Process Process-5:
Traceback (most recent call last):
File "/root/miniconda3/envs/CenterNet/lib/python3.6/multiprocessing/process.py", line 258, in _bootstrap
self.run()
File "/root/miniconda3/envs/CenterNet/lib/python3.6/multiprocessing/process.py", line 93, in run
self._target(*self._args, **self._kwargs)
File "train.py", line 47, in prefetch_data
raise e
File "train.py", line 43, in prefetch_data
data, ind = sample_data(db, ind, data_aug=data_aug)
File "/root/autodl-tmp/CenterNet/sample/coco.py", line 199, in sample_data
return globals()[system_configs.sampling_function](db, k_ind, data_aug, debug)
File "/root/autodl-tmp/CenterNet/sample/coco.py", line 99, in kp_detection
image, detections = random_crop(image, detections, rand_scales, input_size, border=border)
File "/root/autodl-tmp/CenterNet/sample/utils.py", line 57, in random_crop
image_height, image_width = image.shape[0:2]
AttributeError: 'NoneType' object has no attribute 'shape'
/root/miniconda3/envs/CenterNet/lib/python3.6/site-packages/torch/cuda/init.py:114: UserWarning:
Found GPU0 NVIDIA GeForce RTX 2080 Ti which requires CUDA_VERSION >= 9000 for
optimal performance and fast startup time, but your PyTorch was compiled
with CUDA_VERSION 8000. Please install the correct PyTorch binary
using instructions from http://pytorch.org
warnings.warn(incorrect_binary_warn % (d, name, 9000, CUDA_VERSION))
total parameters: 104844152
setting learning rate to: 0.00025
training start...
0%| | 0/480000 [00:00<?, ?it/s]