An error occurred while training s3dis: RuntimeError: CUDA error: device-side assert triggered CUDA kernel errors might be asynchronously reported at some other API call, so the stacktrace below might be incorrect. For debugging consider passing CUDA_LAUNCH_BLOCKING=1. Compile with `TORCH_USE_CUDA_DSA` to enable device-side assertions.
Yatoronto opened this issue · comments
detailed error as follows:
sh scripts/train.sh -p python -g 1 -d s3dis -c semseg-pt-v3m1-0-base -n semseg-pt-v3m1-0-base
Experiment name: semseg-pt-v3m1-0-base
Python interpreter dir: python
Dataset: s3dis
Config: semseg-pt-v3m1-0-base
GPU Num: 1
=========> CREATE EXP DIR <=========
Experiment dir: /media/wsc/16B68EBDB68E9CBD/PTv3/Pointcept/exp/s3dis/semseg-pt-v3m1-0-base
Loading config in: configs/s3dis/semseg-pt-v3m1-0-base.py
Running code in: exp/s3dis/semseg-pt-v3m1-0-base/code
=========> RUN TASK <=========
[2024-06-13 21:10:56,553 INFO train.py line 128 16646] => Loading config ...
[2024-06-13 21:10:56,553 INFO train.py line 130 16646] Save path: exp/s3dis/semseg-pt-v3m1-0-base
[2024-06-13 21:10:57,100 INFO train.py line 131 16646] Config:
weight = None
resume = False
evaluate = True
test_only = False
seed = 56066318
save_path = 'exp/s3dis/semseg-pt-v3m1-0-base'
num_worker = 12
batch_size = 2
batch_size_val = None
batch_size_test = None
epoch = 3000
eval_epoch = 100
sync_bn = False
enable_amp = True
empty_cache = False
empty_cache_per_epoch = False
find_unused_parameters = False
mix_prob = 0.8
param_dicts = [dict(keyword='block', lr=0.0006)]
hooks = [
dict(type='CheckpointLoader'),
dict(type='IterationTimer', warmup_iter=2),
dict(type='InformationWriter'),
dict(type='SemSegEvaluator'),
dict(type='CheckpointSaver', save_freq=None),
dict(type='PreciseEvaluator', test_last=False)
]
train = dict(type='DefaultTrainer')
test = dict(type='SemSegTester', verbose=True)
model = dict(
type='DefaultSegmentorV2',
num_classes=13,
backbone_out_channels=64,
backbone=dict(
type='PT-v3m1',
in_channels=6,
order=('z', 'z-trans', 'hilbert', 'hilbert-trans'),
stride=(2, 2, 2, 2),
enc_depths=(2, 2, 2, 6, 2),
enc_channels=(32, 64, 128, 256, 512),
enc_num_head=(2, 4, 8, 16, 32),
enc_patch_size=(128, 128, 128, 128, 128),
dec_depths=(2, 2, 2, 2),
dec_channels=(64, 64, 128, 256),
dec_num_head=(4, 4, 8, 16),
dec_patch_size=(128, 128, 128, 128),
mlp_ratio=4,
qkv_bias=True,
qk_scale=None,
attn_drop=0.0,
proj_drop=0.0,
drop_path=0.3,
shuffle_orders=True,
pre_norm=True,
enable_rpe=False,
enable_flash=True,
upcast_attention=False,
upcast_softmax=False,
cls_mode=False,
pdnorm_bn=False,
pdnorm_ln=False,
pdnorm_decouple=True,
pdnorm_adaptive=False,
pdnorm_affine=True,
pdnorm_conditions=('ScanNet', 'S3DIS', 'Structured3D')),
criteria=[
dict(type='CrossEntropyLoss', loss_weight=1.0, ignore_index=-1),
dict(
type='LovaszLoss',
mode='multiclass',
loss_weight=1.0,
ignore_index=-1)
])
optimizer = dict(type='AdamW', lr=0.006, weight_decay=0.05)
scheduler = dict(
type='OneCycleLR',
max_lr=[0.006, 0.0006],
pct_start=0.05,
anneal_strategy='cos',
div_factor=10.0,
final_div_factor=1000.0)
dataset_type = 'S3DISDataset'
data_root = 'data/s3dis'
data = dict(
num_classes=13,
ignore_index=-1,
names=[
'ceiling', 'floor', 'wall', 'beam', 'column', 'window', 'door',
'table', 'chair', 'sofa', 'bookcase', 'board', 'clutter'
],
train=dict(
type='S3DISDataset',
split=('Area_1', 'Area_2', 'Area_3', 'Area_4', 'Area_6'),
data_root='data/s3dis',
transform=[
dict(type='CenterShift', apply_z=True),
dict(
type='RandomDropout',
dropout_ratio=0.2,
dropout_application_ratio=0.2),
dict(
type='RandomRotate',
angle=[-1, 1],
axis='z',
center=[0, 0, 0],
p=0.5),
dict(
type='RandomRotate',
angle=[-0.015625, 0.015625],
axis='x',
p=0.5),
dict(
type='RandomRotate',
angle=[-0.015625, 0.015625],
axis='y',
p=0.5),
dict(type='RandomScale', scale=[0.9, 1.1]),
dict(type='RandomFlip', p=0.5),
dict(type='RandomJitter', sigma=0.005, clip=0.02),
dict(type='ChromaticAutoContrast', p=0.2, blend_factor=None),
dict(type='ChromaticTranslation', p=0.95, ratio=0.05),
dict(type='ChromaticJitter', p=0.95, std=0.05),
dict(
type='GridSample',
keys=('coord', 'color', 'segment'),
grid_size=0.02,
hash_type='fnv',
mode='train',
return_grid_coord=True),
dict(type='SphereCrop', sample_rate=0.6, mode='random'),
dict(type='SphereCrop', point_max=204800, mode='random'),
dict(type='CenterShift', apply_z=False),
dict(type='NormalizeColor'),
dict(type='ToTensor'),
dict(
type='Collect',
keys=('coord', 'grid_coord', 'segment'),
feat_keys=('coord', 'color'))
],
test_mode=False,
loop=30),
val=dict(
type='S3DISDataset',
split='Area_5',
data_root='data/s3dis',
transform=[
dict(type='CenterShift', apply_z=True),
dict(
type='Copy',
keys_dict=dict(coord='origin_coord',
segment='origin_segment')),
dict(
type='GridSample',
keys=('coord', 'color', 'segment'),
grid_size=0.02,
hash_type='fnv',
mode='train',
return_grid_coord=True),
dict(type='CenterShift', apply_z=False),
dict(type='NormalizeColor'),
dict(type='ToTensor'),
dict(
type='Collect',
keys=('coord', 'grid_coord', 'origin_coord', 'segment',
'origin_segment'),
offset_keys_dict=dict(
offset='coord', origin_offset='origin_coord'),
feat_keys=('coord', 'color'))
],
test_mode=False),
test=dict(
type='S3DISDataset',
split='Area_5',
data_root='data/s3dis',
transform=[
dict(type='CenterShift', apply_z=True),
dict(type='NormalizeColor')
],
test_mode=True,
test_cfg=dict(
voxelize=dict(
type='GridSample',
grid_size=0.02,
hash_type='fnv',
mode='test',
keys=('coord', 'color'),
return_grid_coord=True),
crop=None,
post_transform=[
dict(type='CenterShift', apply_z=False),
dict(type='ToTensor'),
dict(
type='Collect',
keys=('coord', 'grid_coord', 'index'),
feat_keys=('coord', 'color'))
],
aug_transform=[[{
'type': 'RandomScale',
'scale': [0.9, 0.9]
}], [{
'type': 'RandomScale',
'scale': [0.95, 0.95]
}], [{
'type': 'RandomScale',
'scale': [1, 1]
}], [{
'type': 'RandomScale',
'scale': [1.05, 1.05]
}], [{
'type': 'RandomScale',
'scale': [1.1, 1.1]
}],
[{
'type': 'RandomScale',
'scale': [0.9, 0.9]
}, {
'type': 'RandomFlip',
'p': 1
}],
[{
'type': 'RandomScale',
'scale': [0.95, 0.95]
}, {
'type': 'RandomFlip',
'p': 1
}],
[{
'type': 'RandomScale',
'scale': [1, 1]
}, {
'type': 'RandomFlip',
'p': 1
}],
[{
'type': 'RandomScale',
'scale': [1.05, 1.05]
}, {
'type': 'RandomFlip',
'p': 1
}],
[{
'type': 'RandomScale',
'scale': [1.1, 1.1]
}, {
'type': 'RandomFlip',
'p': 1
}]])))
num_worker_per_gpu = 12
batch_size_per_gpu = 2
batch_size_val_per_gpu = 1
batch_size_test_per_gpu = 1
[2024-06-13 21:10:57,100 INFO train.py line 132 16646] => Building model ...
[2024-06-13 21:10:57,344 INFO train.py line 216 16646] Num params: 46167117
[2024-06-13 21:10:58,085 INFO train.py line 134 16646] => Building writer ...
[2024-06-13 21:10:58,086 INFO train.py line 226 16646] Tensorboard writer logging dir: exp/s3dis/semseg-pt-v3m1-0-base
[2024-06-13 21:10:58,086 INFO train.py line 136 16646] => Building train dataset & dataloader ...
[2024-06-13 21:10:58,087 INFO defaults.py line 68 16646] Totally 204 x 30 samples in ('Area_1', 'Area_2', 'Area_3', 'Area_4', 'Area_6') set.
[2024-06-13 21:10:58,088 INFO train.py line 138 16646] => Building val dataset & dataloader ...
[2024-06-13 21:10:58,088 INFO defaults.py line 68 16646] Totally 68 x 1 samples in Area_5 set.
[2024-06-13 21:10:58,088 INFO train.py line 140 16646] => Building optimize, scheduler, scaler(amp) ...
[2024-06-13 21:10:58,090 INFO optimizer.py line 54 16646] Params Group 1 - lr: 0.006; Params: ['seg_head.weight', 'seg_head.bias', 'backbone.embedding.stem.conv.weight', 'backbone.embedding.stem.norm.weight', 'backbone.embedding.stem.norm.bias', 'backbone.enc.enc1.down.proj.weight', 'backbone.enc.enc1.down.proj.bias', 'backbone.enc.enc1.down.norm.0.weight', 'backbone.enc.enc1.down.norm.0.bias', 'backbone.enc.enc2.down.proj.weight', 'backbone.enc.enc2.down.proj.bias', 'backbone.enc.enc2.down.norm.0.weight', 'backbone.enc.enc2.down.norm.0.bias', 'backbone.enc.enc3.down.proj.weight', 'backbone.enc.enc3.down.proj.bias', 'backbone.enc.enc3.down.norm.0.weight', 'backbone.enc.enc3.down.norm.0.bias', 'backbone.enc.enc4.down.proj.weight', 'backbone.enc.enc4.down.proj.bias', 'backbone.enc.enc4.down.norm.0.weight', 'backbone.enc.enc4.down.norm.0.bias', 'backbone.dec.dec3.up.proj.0.weight', 'backbone.dec.dec3.up.proj.0.bias', 'backbone.dec.dec3.up.proj.1.weight', 'backbone.dec.dec3.up.proj.1.bias', 'backbone.dec.dec3.up.proj_skip.0.weight', 'backbone.dec.dec3.up.proj_skip.0.bias', 'backbone.dec.dec3.up.proj_skip.1.weight', 'backbone.dec.dec3.up.proj_skip.1.bias', 'backbone.dec.dec2.up.proj.0.weight', 'backbone.dec.dec2.up.proj.0.bias', 'backbone.dec.dec2.up.proj.1.weight', 'backbone.dec.dec2.up.proj.1.bias', 'backbone.dec.dec2.up.proj_skip.0.weight', 'backbone.dec.dec2.up.proj_skip.0.bias', 'backbone.dec.dec2.up.proj_skip.1.weight', 'backbone.dec.dec2.up.proj_skip.1.bias', 'backbone.dec.dec1.up.proj.0.weight', 'backbone.dec.dec1.up.proj.0.bias', 'backbone.dec.dec1.up.proj.1.weight', 'backbone.dec.dec1.up.proj.1.bias', 'backbone.dec.dec1.up.proj_skip.0.weight', 'backbone.dec.dec1.up.proj_skip.0.bias', 'backbone.dec.dec1.up.proj_skip.1.weight', 'backbone.dec.dec1.up.proj_skip.1.bias', 'backbone.dec.dec0.up.proj.0.weight', 'backbone.dec.dec0.up.proj.0.bias', 'backbone.dec.dec0.up.proj.1.weight', 'backbone.dec.dec0.up.proj.1.bias', 'backbone.dec.dec0.up.proj_skip.0.weight', 'backbone.dec.dec0.up.proj_skip.0.bias', 'backbone.dec.dec0.up.proj_skip.1.weight', 'backbone.dec.dec0.up.proj_skip.1.bias'].
[2024-06-13 21:10:58,090 INFO optimizer.py line 54 16646] Params Group 2 - lr: 0.0006; Params: ['backbone.enc.enc0.block0.cpe.0.weight', 'backbone.enc.enc0.block0.cpe.0.bias', 'backbone.enc.enc0.block0.cpe.1.weight', 'backbone.enc.enc0.block0.cpe.1.bias', 'backbone.enc.enc0.block0.cpe.2.weight', 'backbone.enc.enc0.block0.cpe.2.bias', 'backbone.enc.enc0.block0.norm1.0.weight', 'backbone.enc.enc0.block0.norm1.0.bias', 'backbone.enc.enc0.block0.attn.qkv.weight', 'backbone.enc.enc0.block0.attn.qkv.bias', 'backbone.enc.enc0.block0.attn.proj.weight', 'backbone.enc.enc0.block0.attn.proj.bias', 'backbone.enc.enc0.block0.norm2.0.weight', 'backbone.enc.enc0.block0.norm2.0.bias', 'backbone.enc.enc0.block0.mlp.0.fc1.weight', 'backbone.enc.enc0.block0.mlp.0.fc1.bias', 'backbone.enc.enc0.block0.mlp.0.fc2.weight', 'backbone.enc.enc0.block0.mlp.0.fc2.bias', 'backbone.enc.enc0.block1.cpe.0.weight', 'backbone.enc.enc0.block1.cpe.0.bias', 'backbone.enc.enc0.block1.cpe.1.weight', 'backbone.enc.enc0.block1.cpe.1.bias', 'backbone.enc.enc0.block1.cpe.2.weight', 'backbone.enc.enc0.block1.cpe.2.bias', 'backbone.enc.enc0.block1.norm1.0.weight', 'backbone.enc.enc0.block1.norm1.0.bias', 'backbone.enc.enc0.block1.attn.qkv.weight', 'backbone.enc.enc0.block1.attn.qkv.bias', 'backbone.enc.enc0.block1.attn.proj.weight', 'backbone.enc.enc0.block1.attn.proj.bias', 'backbone.enc.enc0.block1.norm2.0.weight', 'backbone.enc.enc0.block1.norm2.0.bias', 'backbone.enc.enc0.block1.mlp.0.fc1.weight', 'backbone.enc.enc0.block1.mlp.0.fc1.bias', 'backbone.enc.enc0.block1.mlp.0.fc2.weight', 'backbone.enc.enc0.block1.mlp.0.fc2.bias', 'backbone.enc.enc1.block0.cpe.0.weight', 'backbone.enc.enc1.block0.cpe.0.bias', 'backbone.enc.enc1.block0.cpe.1.weight', 'backbone.enc.enc1.block0.cpe.1.bias', 'backbone.enc.enc1.block0.cpe.2.weight', 'backbone.enc.enc1.block0.cpe.2.bias', 'backbone.enc.enc1.block0.norm1.0.weight', 'backbone.enc.enc1.block0.norm1.0.bias', 'backbone.enc.enc1.block0.attn.qkv.weight', 'backbone.enc.enc1.block0.attn.qkv.bias', 'backbone.enc.enc1.block0.attn.proj.weight', 'backbone.enc.enc1.block0.attn.proj.bias', 'backbone.enc.enc1.block0.norm2.0.weight', 'backbone.enc.enc1.block0.norm2.0.bias', 'backbone.enc.enc1.block0.mlp.0.fc1.weight', 'backbone.enc.enc1.block0.mlp.0.fc1.bias', 'backbone.enc.enc1.block0.mlp.0.fc2.weight', 'backbone.enc.enc1.block0.mlp.0.fc2.bias', 'backbone.enc.enc1.block1.cpe.0.weight', 'backbone.enc.enc1.block1.cpe.0.bias', 'backbone.enc.enc1.block1.cpe.1.weight', 'backbone.enc.enc1.block1.cpe.1.bias', 'backbone.enc.enc1.block1.cpe.2.weight', 'backbone.enc.enc1.block1.cpe.2.bias', 'backbone.enc.enc1.block1.norm1.0.weight', 'backbone.enc.enc1.block1.norm1.0.bias', 'backbone.enc.enc1.block1.attn.qkv.weight', 'backbone.enc.enc1.block1.attn.qkv.bias', 'backbone.enc.enc1.block1.attn.proj.weight', 'backbone.enc.enc1.block1.attn.proj.bias', 'backbone.enc.enc1.block1.norm2.0.weight', 'backbone.enc.enc1.block1.norm2.0.bias', 'backbone.enc.enc1.block1.mlp.0.fc1.weight', 'backbone.enc.enc1.block1.mlp.0.fc1.bias', 'backbone.enc.enc1.block1.mlp.0.fc2.weight', 'backbone.enc.enc1.block1.mlp.0.fc2.bias', 'backbone.enc.enc2.block0.cpe.0.weight', 'backbone.enc.enc2.block0.cpe.0.bias', 'backbone.enc.enc2.block0.cpe.1.weight', 'backbone.enc.enc2.block0.cpe.1.bias', 'backbone.enc.enc2.block0.cpe.2.weight', 'backbone.enc.enc2.block0.cpe.2.bias', 'backbone.enc.enc2.block0.norm1.0.weight', 'backbone.enc.enc2.block0.norm1.0.bias', 'backbone.enc.enc2.block0.attn.qkv.weight', 'backbone.enc.enc2.block0.attn.qkv.bias', 'backbone.enc.enc2.block0.attn.proj.weight', 'backbone.enc.enc2.block0.attn.proj.bias', 'backbone.enc.enc2.block0.norm2.0.weight', 'backbone.enc.enc2.block0.norm2.0.bias', 'backbone.enc.enc2.block0.mlp.0.fc1.weight', 'backbone.enc.enc2.block0.mlp.0.fc1.bias', 'backbone.enc.enc2.block0.mlp.0.fc2.weight', 'backbone.enc.enc2.block0.mlp.0.fc2.bias', 'backbone.enc.enc2.block1.cpe.0.weight', 'backbone.enc.enc2.block1.cpe.0.bias', 'backbone.enc.enc2.block1.cpe.1.weight', 'backbone.enc.enc2.block1.cpe.1.bias', 'backbone.enc.enc2.block1.cpe.2.weight', 'backbone.enc.enc2.block1.cpe.2.bias', 'backbone.enc.enc2.block1.norm1.0.weight', 'backbone.enc.enc2.block1.norm1.0.bias', 'backbone.enc.enc2.block1.attn.qkv.weight', 'backbone.enc.enc2.block1.attn.qkv.bias', 'backbone.enc.enc2.block1.attn.proj.weight', 'backbone.enc.enc2.block1.attn.proj.bias', 'backbone.enc.enc2.block1.norm2.0.weight', 'backbone.enc.enc2.block1.norm2.0.bias', 'backbone.enc.enc2.block1.mlp.0.fc1.weight', 'backbone.enc.enc2.block1.mlp.0.fc1.bias', 'backbone.enc.enc2.block1.mlp.0.fc2.weight', 'backbone.enc.enc2.block1.mlp.0.fc2.bias', 'backbone.enc.enc3.block0.cpe.0.weight', 'backbone.enc.enc3.block0.cpe.0.bias', 'backbone.enc.enc3.block0.cpe.1.weight', 'backbone.enc.enc3.block0.cpe.1.bias', 'backbone.enc.enc3.block0.cpe.2.weight', 'backbone.enc.enc3.block0.cpe.2.bias', 'backbone.enc.enc3.block0.norm1.0.weight', 'backbone.enc.enc3.block0.norm1.0.bias', 'backbone.enc.enc3.block0.attn.qkv.weight', 'backbone.enc.enc3.block0.attn.qkv.bias', 'backbone.enc.enc3.block0.attn.proj.weight', 'backbone.enc.enc3.block0.attn.proj.bias', 'backbone.enc.enc3.block0.norm2.0.weight', 'backbone.enc.enc3.block0.norm2.0.bias', 'backbone.enc.enc3.block0.mlp.0.fc1.weight', 'backbone.enc.enc3.block0.mlp.0.fc1.bias', 'backbone.enc.enc3.block0.mlp.0.fc2.weight', 'backbone.enc.enc3.block0.mlp.0.fc2.bias', 'backbone.enc.enc3.block1.cpe.0.weight', 'backbone.enc.enc3.block1.cpe.0.bias', 'backbone.enc.enc3.block1.cpe.1.weight', 'backbone.enc.enc3.block1.cpe.1.bias', 'backbone.enc.enc3.block1.cpe.2.weight', 'backbone.enc.enc3.block1.cpe.2.bias', 'backbone.enc.enc3.block1.norm1.0.weight', 'backbone.enc.enc3.block1.norm1.0.bias', 'backbone.enc.enc3.block1.attn.qkv.weight', 'backbone.enc.enc3.block1.attn.qkv.bias', 'backbone.enc.enc3.block1.attn.proj.weight', 'backbone.enc.enc3.block1.attn.proj.bias', 'backbone.enc.enc3.block1.norm2.0.weight', 'backbone.enc.enc3.block1.norm2.0.bias', 'backbone.enc.enc3.block1.mlp.0.fc1.weight', 'backbone.enc.enc3.block1.mlp.0.fc1.bias', 'backbone.enc.enc3.block1.mlp.0.fc2.weight', 'backbone.enc.enc3.block1.mlp.0.fc2.bias', 'backbone.enc.enc3.block2.cpe.0.weight', 'backbone.enc.enc3.block2.cpe.0.bias', 'backbone.enc.enc3.block2.cpe.1.weight', 'backbone.enc.enc3.block2.cpe.1.bias', 'backbone.enc.enc3.block2.cpe.2.weight', 'backbone.enc.enc3.block2.cpe.2.bias', 'backbone.enc.enc3.block2.norm1.0.weight', 'backbone.enc.enc3.block2.norm1.0.bias', 'backbone.enc.enc3.block2.attn.qkv.weight', 'backbone.enc.enc3.block2.attn.qkv.bias', 'backbone.enc.enc3.block2.attn.proj.weight', 'backbone.enc.enc3.block2.attn.proj.bias', 'backbone.enc.enc3.block2.norm2.0.weight', 'backbone.enc.enc3.block2.norm2.0.bias', 'backbone.enc.enc3.block2.mlp.0.fc1.weight', 'backbone.enc.enc3.block2.mlp.0.fc1.bias', 'backbone.enc.enc3.block2.mlp.0.fc2.weight', 'backbone.enc.enc3.block2.mlp.0.fc2.bias', 'backbone.enc.enc3.block3.cpe.0.weight', 'backbone.enc.enc3.block3.cpe.0.bias', 'backbone.enc.enc3.block3.cpe.1.weight', 'backbone.enc.enc3.block3.cpe.1.bias', 'backbone.enc.enc3.block3.cpe.2.weight', 'backbone.enc.enc3.block3.cpe.2.bias', 'backbone.enc.enc3.block3.norm1.0.weight', 'backbone.enc.enc3.block3.norm1.0.bias', 'backbone.enc.enc3.block3.attn.qkv.weight', 'backbone.enc.enc3.block3.attn.qkv.bias', 'backbone.enc.enc3.block3.attn.proj.weight', 'backbone.enc.enc3.block3.attn.proj.bias', 'backbone.enc.enc3.block3.norm2.0.weight', 'backbone.enc.enc3.block3.norm2.0.bias', 'backbone.enc.enc3.block3.mlp.0.fc1.weight', 'backbone.enc.enc3.block3.mlp.0.fc1.bias', 'backbone.enc.enc3.block3.mlp.0.fc2.weight', 'backbone.enc.enc3.block3.mlp.0.fc2.bias', 'backbone.enc.enc3.block4.cpe.0.weight', 'backbone.enc.enc3.block4.cpe.0.bias', 'backbone.enc.enc3.block4.cpe.1.weight', 'backbone.enc.enc3.block4.cpe.1.bias', 'backbone.enc.enc3.block4.cpe.2.weight', 'backbone.enc.enc3.block4.cpe.2.bias', 'backbone.enc.enc3.block4.norm1.0.weight', 'backbone.enc.enc3.block4.norm1.0.bias', 'backbone.enc.enc3.block4.attn.qkv.weight', 'backbone.enc.enc3.block4.attn.qkv.bias', 'backbone.enc.enc3.block4.attn.proj.weight', 'backbone.enc.enc3.block4.attn.proj.bias', 'backbone.enc.enc3.block4.norm2.0.weight', 'backbone.enc.enc3.block4.norm2.0.bias', 'backbone.enc.enc3.block4.mlp.0.fc1.weight', 'backbone.enc.enc3.block4.mlp.0.fc1.bias', 'backbone.enc.enc3.block4.mlp.0.fc2.weight', 'backbone.enc.enc3.block4.mlp.0.fc2.bias', 'backbone.enc.enc3.block5.cpe.0.weight', 'backbone.enc.enc3.block5.cpe.0.bias', 'backbone.enc.enc3.block5.cpe.1.weight', 'backbone.enc.enc3.block5.cpe.1.bias', 'backbone.enc.enc3.block5.cpe.2.weight', 'backbone.enc.enc3.block5.cpe.2.bias', 'backbone.enc.enc3.block5.norm1.0.weight', 'backbone.enc.enc3.block5.norm1.0.bias', 'backbone.enc.enc3.block5.attn.qkv.weight', 'backbone.enc.enc3.block5.attn.qkv.bias', 'backbone.enc.enc3.block5.attn.proj.weight', 'backbone.enc.enc3.block5.attn.proj.bias', 'backbone.enc.enc3.block5.norm2.0.weight', 'backbone.enc.enc3.block5.norm2.0.bias', 'backbone.enc.enc3.block5.mlp.0.fc1.weight', 'backbone.enc.enc3.block5.mlp.0.fc1.bias', 'backbone.enc.enc3.block5.mlp.0.fc2.weight', 'backbone.enc.enc3.block5.mlp.0.fc2.bias', 'backbone.enc.enc4.block0.cpe.0.weight', 'backbone.enc.enc4.block0.cpe.0.bias', 'backbone.enc.enc4.block0.cpe.1.weight', 'backbone.enc.enc4.block0.cpe.1.bias', 'backbone.enc.enc4.block0.cpe.2.weight', 'backbone.enc.enc4.block0.cpe.2.bias', 'backbone.enc.enc4.block0.norm1.0.weight', 'backbone.enc.enc4.block0.norm1.0.bias', 'backbone.enc.enc4.block0.attn.qkv.weight', 'backbone.enc.enc4.block0.attn.qkv.bias', 'backbone.enc.enc4.block0.attn.proj.weight', 'backbone.enc.enc4.block0.attn.proj.bias', 'backbone.enc.enc4.block0.norm2.0.weight', 'backbone.enc.enc4.block0.norm2.0.bias', 'backbone.enc.enc4.block0.mlp.0.fc1.weight', 'backbone.enc.enc4.block0.mlp.0.fc1.bias', 'backbone.enc.enc4.block0.mlp.0.fc2.weight', 'backbone.enc.enc4.block0.mlp.0.fc2.bias', 'backbone.enc.enc4.block1.cpe.0.weight', 'backbone.enc.enc4.block1.cpe.0.bias', 'backbone.enc.enc4.block1.cpe.1.weight', 'backbone.enc.enc4.block1.cpe.1.bias', 'backbone.enc.enc4.block1.cpe.2.weight', 'backbone.enc.enc4.block1.cpe.2.bias', 'backbone.enc.enc4.block1.norm1.0.weight', 'backbone.enc.enc4.block1.norm1.0.bias', 'backbone.enc.enc4.block1.attn.qkv.weight', 'backbone.enc.enc4.block1.attn.qkv.bias', 'backbone.enc.enc4.block1.attn.proj.weight', 'backbone.enc.enc4.block1.attn.proj.bias', 'backbone.enc.enc4.block1.norm2.0.weight', 'backbone.enc.enc4.block1.norm2.0.bias', 'backbone.enc.enc4.block1.mlp.0.fc1.weight', 'backbone.enc.enc4.block1.mlp.0.fc1.bias', 'backbone.enc.enc4.block1.mlp.0.fc2.weight', 'backbone.enc.enc4.block1.mlp.0.fc2.bias', 'backbone.dec.dec3.block0.cpe.0.weight', 'backbone.dec.dec3.block0.cpe.0.bias', 'backbone.dec.dec3.block0.cpe.1.weight', 'backbone.dec.dec3.block0.cpe.1.bias', 'backbone.dec.dec3.block0.cpe.2.weight', 'backbone.dec.dec3.block0.cpe.2.bias', 'backbone.dec.dec3.block0.norm1.0.weight', 'backbone.dec.dec3.block0.norm1.0.bias', 'backbone.dec.dec3.block0.attn.qkv.weight', 'backbone.dec.dec3.block0.attn.qkv.bias', 'backbone.dec.dec3.block0.attn.proj.weight', 'backbone.dec.dec3.block0.attn.proj.bias', 'backbone.dec.dec3.block0.norm2.0.weight', 'backbone.dec.dec3.block0.norm2.0.bias', 'backbone.dec.dec3.block0.mlp.0.fc1.weight', 'backbone.dec.dec3.block0.mlp.0.fc1.bias', 'backbone.dec.dec3.block0.mlp.0.fc2.weight', 'backbone.dec.dec3.block0.mlp.0.fc2.bias', 'backbone.dec.dec3.block1.cpe.0.weight', 'backbone.dec.dec3.block1.cpe.0.bias', 'backbone.dec.dec3.block1.cpe.1.weight', 'backbone.dec.dec3.block1.cpe.1.bias', 'backbone.dec.dec3.block1.cpe.2.weight', 'backbone.dec.dec3.block1.cpe.2.bias', 'backbone.dec.dec3.block1.norm1.0.weight', 'backbone.dec.dec3.block1.norm1.0.bias', 'backbone.dec.dec3.block1.attn.qkv.weight', 'backbone.dec.dec3.block1.attn.qkv.bias', 'backbone.dec.dec3.block1.attn.proj.weight', 'backbone.dec.dec3.block1.attn.proj.bias', 'backbone.dec.dec3.block1.norm2.0.weight', 'backbone.dec.dec3.block1.norm2.0.bias', 'backbone.dec.dec3.block1.mlp.0.fc1.weight', 'backbone.dec.dec3.block1.mlp.0.fc1.bias', 'backbone.dec.dec3.block1.mlp.0.fc2.weight', 'backbone.dec.dec3.block1.mlp.0.fc2.bias', 'backbone.dec.dec2.block0.cpe.0.weight', 'backbone.dec.dec2.block0.cpe.0.bias', 'backbone.dec.dec2.block0.cpe.1.weight', 'backbone.dec.dec2.block0.cpe.1.bias', 'backbone.dec.dec2.block0.cpe.2.weight', 'backbone.dec.dec2.block0.cpe.2.bias', 'backbone.dec.dec2.block0.norm1.0.weight', 'backbone.dec.dec2.block0.norm1.0.bias', 'backbone.dec.dec2.block0.attn.qkv.weight', 'backbone.dec.dec2.block0.attn.qkv.bias', 'backbone.dec.dec2.block0.attn.proj.weight', 'backbone.dec.dec2.block0.attn.proj.bias', 'backbone.dec.dec2.block0.norm2.0.weight', 'backbone.dec.dec2.block0.norm2.0.bias', 'backbone.dec.dec2.block0.mlp.0.fc1.weight', 'backbone.dec.dec2.block0.mlp.0.fc1.bias', 'backbone.dec.dec2.block0.mlp.0.fc2.weight', 'backbone.dec.dec2.block0.mlp.0.fc2.bias', 'backbone.dec.dec2.block1.cpe.0.weight', 'backbone.dec.dec2.block1.cpe.0.bias', 'backbone.dec.dec2.block1.cpe.1.weight', 'backbone.dec.dec2.block1.cpe.1.bias', 'backbone.dec.dec2.block1.cpe.2.weight', 'backbone.dec.dec2.block1.cpe.2.bias', 'backbone.dec.dec2.block1.norm1.0.weight', 'backbone.dec.dec2.block1.norm1.0.bias', 'backbone.dec.dec2.block1.attn.qkv.weight', 'backbone.dec.dec2.block1.attn.qkv.bias', 'backbone.dec.dec2.block1.attn.proj.weight', 'backbone.dec.dec2.block1.attn.proj.bias', 'backbone.dec.dec2.block1.norm2.0.weight', 'backbone.dec.dec2.block1.norm2.0.bias', 'backbone.dec.dec2.block1.mlp.0.fc1.weight', 'backbone.dec.dec2.block1.mlp.0.fc1.bias', 'backbone.dec.dec2.block1.mlp.0.fc2.weight', 'backbone.dec.dec2.block1.mlp.0.fc2.bias', 'backbone.dec.dec1.block0.cpe.0.weight', 'backbone.dec.dec1.block0.cpe.0.bias', 'backbone.dec.dec1.block0.cpe.1.weight', 'backbone.dec.dec1.block0.cpe.1.bias', 'backbone.dec.dec1.block0.cpe.2.weight', 'backbone.dec.dec1.block0.cpe.2.bias', 'backbone.dec.dec1.block0.norm1.0.weight', 'backbone.dec.dec1.block0.norm1.0.bias', 'backbone.dec.dec1.block0.attn.qkv.weight', 'backbone.dec.dec1.block0.attn.qkv.bias', 'backbone.dec.dec1.block0.attn.proj.weight', 'backbone.dec.dec1.block0.attn.proj.bias', 'backbone.dec.dec1.block0.norm2.0.weight', 'backbone.dec.dec1.block0.norm2.0.bias', 'backbone.dec.dec1.block0.mlp.0.fc1.weight', 'backbone.dec.dec1.block0.mlp.0.fc1.bias', 'backbone.dec.dec1.block0.mlp.0.fc2.weight', 'backbone.dec.dec1.block0.mlp.0.fc2.bias', 'backbone.dec.dec1.block1.cpe.0.weight', 'backbone.dec.dec1.block1.cpe.0.bias', 'backbone.dec.dec1.block1.cpe.1.weight', 'backbone.dec.dec1.block1.cpe.1.bias', 'backbone.dec.dec1.block1.cpe.2.weight', 'backbone.dec.dec1.block1.cpe.2.bias', 'backbone.dec.dec1.block1.norm1.0.weight', 'backbone.dec.dec1.block1.norm1.0.bias', 'backbone.dec.dec1.block1.attn.qkv.weight', 'backbone.dec.dec1.block1.attn.qkv.bias', 'backbone.dec.dec1.block1.attn.proj.weight', 'backbone.dec.dec1.block1.attn.proj.bias', 'backbone.dec.dec1.block1.norm2.0.weight', 'backbone.dec.dec1.block1.norm2.0.bias', 'backbone.dec.dec1.block1.mlp.0.fc1.weight', 'backbone.dec.dec1.block1.mlp.0.fc1.bias', 'backbone.dec.dec1.block1.mlp.0.fc2.weight', 'backbone.dec.dec1.block1.mlp.0.fc2.bias', 'backbone.dec.dec0.block0.cpe.0.weight', 'backbone.dec.dec0.block0.cpe.0.bias', 'backbone.dec.dec0.block0.cpe.1.weight', 'backbone.dec.dec0.block0.cpe.1.bias', 'backbone.dec.dec0.block0.cpe.2.weight', 'backbone.dec.dec0.block0.cpe.2.bias', 'backbone.dec.dec0.block0.norm1.0.weight', 'backbone.dec.dec0.block0.norm1.0.bias', 'backbone.dec.dec0.block0.attn.qkv.weight', 'backbone.dec.dec0.block0.attn.qkv.bias', 'backbone.dec.dec0.block0.attn.proj.weight', 'backbone.dec.dec0.block0.attn.proj.bias', 'backbone.dec.dec0.block0.norm2.0.weight', 'backbone.dec.dec0.block0.norm2.0.bias', 'backbone.dec.dec0.block0.mlp.0.fc1.weight', 'backbone.dec.dec0.block0.mlp.0.fc1.bias', 'backbone.dec.dec0.block0.mlp.0.fc2.weight', 'backbone.dec.dec0.block0.mlp.0.fc2.bias', 'backbone.dec.dec0.block1.cpe.0.weight', 'backbone.dec.dec0.block1.cpe.0.bias', 'backbone.dec.dec0.block1.cpe.1.weight', 'backbone.dec.dec0.block1.cpe.1.bias', 'backbone.dec.dec0.block1.cpe.2.weight', 'backbone.dec.dec0.block1.cpe.2.bias', 'backbone.dec.dec0.block1.norm1.0.weight', 'backbone.dec.dec0.block1.norm1.0.bias', 'backbone.dec.dec0.block1.attn.qkv.weight', 'backbone.dec.dec0.block1.attn.qkv.bias', 'backbone.dec.dec0.block1.attn.proj.weight', 'backbone.dec.dec0.block1.attn.proj.bias', 'backbone.dec.dec0.block1.norm2.0.weight', 'backbone.dec.dec0.block1.norm2.0.bias', 'backbone.dec.dec0.block1.mlp.0.fc1.weight', 'backbone.dec.dec0.block1.mlp.0.fc1.bias', 'backbone.dec.dec0.block1.mlp.0.fc2.weight', 'backbone.dec.dec0.block1.mlp.0.fc2.bias'].
[2024-06-13 21:10:58,091 INFO train.py line 144 16646] => Building hooks ...
[2024-06-13 21:10:58,091 INFO misc.py line 214 16646] => Loading checkpoint & weight ...
[2024-06-13 21:10:58,091 INFO misc.py line 250 16646] No weight found at: None
[2024-06-13 21:10:58,091 INFO train.py line 151 16646] >>>>>>>>>>>>>>>> Start Training >>>>>>>>>>>>>>>>
[2024-06-13 21:11:05,799 INFO misc.py line 118 16646] Train: [1/100][1/3060] Data 2.525 (2.525) Batch 7.462 (7.462) Remain 634:17:19 loss: 4.2320 Lr: 0.00060
[2024-06-13 21:11:05,984 INFO misc.py line 118 16646] Train: [1/100][2/3060] Data 0.001 (0.001) Batch 0.184 (0.184) Remain 15:40:20 loss: 3.3508 Lr: 0.00060
[2024-06-13 21:11:06,203 INFO misc.py line 118 16646] Train: [1/100][3/3060] Data 0.001 (0.001) Batch 0.218 (0.218) Remain 18:32:12 loss: 3.2581 Lr: 0.00060
[2024-06-13 21:11:06,383 INFO misc.py line 118 16646] Train: [1/100][4/3060] Data 0.002 (0.002) Batch 0.181 (0.181) Remain 15:21:04 loss: 2.7867 Lr: 0.00060
[2024-06-13 21:11:06,653 INFO misc.py line 118 16646] Train: [1/100][5/3060] Data 0.002 (0.002) Batch 0.269 (0.225) Remain 19:05:17 loss: 2.4009 Lr: 0.00060
[2024-06-13 21:11:06,807 INFO misc.py line 118 16646] Train: [1/100][6/3060] Data 0.004 (0.002) Batch 0.154 (0.201) Remain 17:04:29 loss: 2.4601 Lr: 0.00060
[2024-06-13 21:11:07,052 INFO misc.py line 118 16646] Train: [1/100][7/3060] Data 0.004 (0.003) Batch 0.246 (0.212) Remain 18:01:58 loss: 2.1156 Lr: 0.00060
[2024-06-13 21:11:07,337 INFO misc.py line 118 16646] Train: [1/100][8/3060] Data 0.003 (0.003) Batch 0.286 (0.227) Remain 19:17:14 loss: 1.8292 Lr: 0.00060
[2024-06-13 21:11:07,497 INFO misc.py line 118 16646] Train: [1/100][9/3060] Data 0.002 (0.003) Batch 0.160 (0.216) Remain 18:20:14 loss: 1.5878 Lr: 0.00060
[2024-06-13 21:11:07,668 INFO misc.py line 118 16646] Train: [1/100][10/3060] Data 0.003 (0.003) Batch 0.171 (0.209) Remain 17:47:41 loss: 2.1592 Lr: 0.00060
[2024-06-13 21:11:07,878 INFO misc.py line 118 16646] Train: [1/100][11/3060] Data 0.002 (0.003) Batch 0.210 (0.209) Remain 17:47:50 loss: 2.6694 Lr: 0.00060
[2024-06-13 21:11:08,066 INFO misc.py line 118 16646] Train: [1/100][12/3060] Data 0.003 (0.003) Batch 0.188 (0.207) Remain 17:35:32 loss: 2.5363 Lr: 0.00060
[2024-06-13 21:11:08,319 INFO misc.py line 118 16646] Train: [1/100][13/3060] Data 0.002 (0.003) Batch 0.253 (0.212) Remain 17:58:47 loss: 2.5076 Lr: 0.00060
[2024-06-13 21:11:08,581 INFO misc.py line 118 16646] Train: [1/100][14/3060] Data 0.003 (0.003) Batch 0.261 (0.216) Remain 18:21:45 loss: 1.8056 Lr: 0.00060
[2024-06-13 21:11:08,758 INFO misc.py line 118 16646] Train: [1/100][15/3060] Data 0.004 (0.003) Batch 0.178 (0.213) Remain 18:05:31 loss: 2.0780 Lr: 0.00060
[2024-06-13 21:11:08,992 INFO misc.py line 118 16646] Train: [1/100][16/3060] Data 0.003 (0.003) Batch 0.235 (0.215) Remain 18:14:14 loss: 2.2995 Lr: 0.00060
[2024-06-13 21:11:09,208 INFO misc.py line 118 16646] Train: [1/100][17/3060] Data 0.003 (0.003) Batch 0.215 (0.215) Remain 18:14:19 loss: 1.9800 Lr: 0.00060
[2024-06-13 21:11:09,482 INFO misc.py line 118 16646] Train: [1/100][18/3060] Data 0.004 (0.003) Batch 0.275 (0.219) Remain 18:34:52 loss: 2.4829 Lr: 0.00060
[2024-06-13 21:11:09,665 INFO misc.py line 118 16646] Train: [1/100][19/3060] Data 0.002 (0.003) Batch 0.183 (0.216) Remain 18:23:32 loss: 2.3146 Lr: 0.00060
[2024-06-13 21:11:09,945 INFO misc.py line 118 16646] Train: [1/100][20/3060] Data 0.002 (0.003) Batch 0.280 (0.220) Remain 18:42:40 loss: 1.5082 Lr: 0.00060
[2024-06-13 21:11:10,228 INFO misc.py line 118 16646] Train: [1/100][21/3060] Data 0.002 (0.003) Batch 0.282 (0.224) Remain 19:00:15 loss: 2.5455 Lr: 0.00060
[2024-06-13 21:11:10,520 INFO misc.py line 118 16646] Train: [1/100][22/3060] Data 0.003 (0.003) Batch 0.291 (0.227) Remain 19:18:20 loss: 2.8932 Lr: 0.00060
[2024-06-13 21:11:10,759 INFO misc.py line 118 16646] Train: [1/100][23/3060] Data 0.004 (0.003) Batch 0.239 (0.228) Remain 19:21:26 loss: 2.1961 Lr: 0.00060
[2024-06-13 21:11:10,964 INFO misc.py line 118 16646] Train: [1/100][24/3060] Data 0.004 (0.003) Batch 0.206 (0.227) Remain 19:16:02 loss: 2.0675 Lr: 0.00060
[2024-06-13 21:11:11,159 INFO misc.py line 118 16646] Train: [1/100][25/3060] Data 0.003 (0.003) Batch 0.195 (0.225) Remain 19:08:34 loss: 1.3820 Lr: 0.00060
[2024-06-13 21:11:11,378 INFO misc.py line 118 16646] Train: [1/100][26/3060] Data 0.004 (0.003) Batch 0.220 (0.225) Remain 19:07:29 loss: 2.1714 Lr: 0.00060
[2024-06-13 21:11:11,587 INFO misc.py line 118 16646] Train: [1/100][27/3060] Data 0.002 (0.003) Batch 0.208 (0.224) Remain 19:03:52 loss: 1.9616 Lr: 0.00060
[2024-06-13 21:11:11,829 INFO misc.py line 118 16646] Train: [1/100][28/3060] Data 0.003 (0.003) Batch 0.242 (0.225) Remain 19:07:27 loss: 1.3955 Lr: 0.00060
[2024-06-13 21:11:12,042 INFO misc.py line 118 16646] Train: [1/100][29/3060] Data 0.003 (0.003) Batch 0.213 (0.225) Remain 19:05:05 loss: 1.7584 Lr: 0.00060
[2024-06-13 21:11:12,277 INFO misc.py line 118 16646] Train: [1/100][30/3060] Data 0.004 (0.003) Batch 0.234 (0.225) Remain 19:06:51 loss: 2.0954 Lr: 0.00060
[2024-06-13 21:11:12,486 INFO misc.py line 118 16646] Train: [1/100][31/3060] Data 0.005 (0.003) Batch 0.210 (0.224) Remain 19:04:12 loss: 2.0444 Lr: 0.00060
[2024-06-13 21:11:12,688 INFO misc.py line 118 16646] Train: [1/100][32/3060] Data 0.003 (0.003) Batch 0.202 (0.224) Remain 19:00:14 loss: 1.6240 Lr: 0.00060
/opt/conda/conda-bld/pytorch_1678402379298/work/aten/src/ATen/native/cuda/Loss.cu:240: nll_loss_forward_reduce_cuda_kernel_2d: block: [0,0,0], thread: [20,0,0] Assertion `t >= 0 && t < n_classes` failed.
Traceback (most recent call last):
File "exp/s3dis/semseg-pt-v3m1-0-base/code/tools/train.py", line 38, in <module>
main()
File "exp/s3dis/semseg-pt-v3m1-0-base/code/tools/train.py", line 27, in main
launch(
File "/media/wsc/16B68EBDB68E9CBD/PTv3/Pointcept/exp/s3dis/semseg-pt-v3m1-0-base/code/pointcept/engines/launch.py", line 89, in launch
main_func(*cfg)
File "exp/s3dis/semseg-pt-v3m1-0-base/code/tools/train.py", line 20, in main_worker
trainer.train()
File "/media/wsc/16B68EBDB68E9CBD/PTv3/Pointcept/exp/s3dis/semseg-pt-v3m1-0-base/code/pointcept/engines/train.py", line 168, in train
self.run_step()
File "/media/wsc/16B68EBDB68E9CBD/PTv3/Pointcept/exp/s3dis/semseg-pt-v3m1-0-base/code/pointcept/engines/train.py", line 182, in run_step
output_dict = self.model(input_dict)
File "/home/wsc/anaconda3/envs/ptv3/lib/python3.8/site-packages/torch/nn/modules/module.py", line 1501, in _call_impl
return forward_call(*args, **kwargs)
File "/media/wsc/16B68EBDB68E9CBD/PTv3/Pointcept/exp/s3dis/semseg-pt-v3m1-0-base/code/pointcept/models/default.py", line 65, in forward
loss = self.criteria(seg_logits, input_dict["segment"])
File "/media/wsc/16B68EBDB68E9CBD/PTv3/Pointcept/exp/s3dis/semseg-pt-v3m1-0-base/code/pointcept/models/losses/builder.py", line 26, in __call__
loss += c(pred, target)
File "/home/wsc/anaconda3/envs/ptv3/lib/python3.8/site-packages/torch/nn/modules/module.py", line 1501, in _call_impl
return forward_call(*args, **kwargs)
File "/media/wsc/16B68EBDB68E9CBD/PTv3/Pointcept/exp/s3dis/semseg-pt-v3m1-0-base/code/pointcept/models/losses/lovasz.py", line 248, in forward
loss = _lovasz_softmax(
File "/media/wsc/16B68EBDB68E9CBD/PTv3/Pointcept/exp/s3dis/semseg-pt-v3m1-0-base/code/pointcept/models/losses/lovasz.py", line 111, in _lovasz_softmax
*_flatten_probas(probas, labels, ignore),
File "/media/wsc/16B68EBDB68E9CBD/PTv3/Pointcept/exp/s3dis/semseg-pt-v3m1-0-base/code/pointcept/models/losses/lovasz.py", line 182, in _flatten_probas
vprobas = probas[valid]
RuntimeError: CUDA error: device-side assert triggered
CUDA kernel errors might be asynchronously reported at some other API call, so the stacktrace below might be incorrect.
For debugging consider passing CUDA_LAUNCH_BLOCKING=1.
Compile with `TORCH_USE_CUDA_DSA` to enable device-side assertions.
there are my GPU and pytorch's information.
Thanks for your work and look forward to your reply!
Hello @Yatoronto
did you run a specific command to solve the issue ?