Question of validation result in step1, 15-5s
kona419 opened this issue · comments
Hello. I am running 15-5s with VOC dataset.
During the validation in step1(after the step0), my result is like,
Class IoU:
class 0: 0.8864838886464639
class 1: X
class 2: X
class 3: X
class 4: X
class 5: X
class 6: X
class 7: X
class 8: X
class 9: X
class 10: X
class 11: X
class 12: X
class 13: X
class 14: X
class 15: X
class 16: 0.5655430208625937
Class Acc:
class 0: 0.9604431435003239
class 1: X
class 2: X
class 3: X
class 4: X
class 5: X
class 6: X
class 7: X
class 8: X
class 9: X
class 10: X
class 11: X
class 12: X
class 13: X
class 14: X
class 15: X
class 16: 0.6672904492468744
Is this appropriate result..??
All of the old classes are X.
If this is not appropriate, could you tell me some advice of it?
Thank you :)
Hey! There are some issue in the data processing. X is output when you don't have any GT pixels for the classes. Please check if you modified something in the dataset or please report the run command.
Hey! There are some issue in the data processing. X is output when you don't have any GT pixels for the classes. Please check if you modified something in the dataset or please report the run command.
Thanks for the reply!!
This is my dataset code of voc.py. The only thing I changed is lambda transform part, because I got some error of lambda function.
I found out if I use target_transform_func, class 1~15 are X, or if I use reorder_transform_func, some classes(1,4,10,13) are X.
Could this be the problem?
import os
import random
import torch
import torch.utils.data as data
from torch import distributed
import torchvision as tv
import numpy as np
def group_images(dataset, labels):
# Group images based on the label in LABELS (using labels not reordered)
idxs = {lab: [] for lab in labels}
labels_cum = labels + [0, 255]
for i in range(len(dataset)):
cls = np.unique(np.array(dataset[i][1]))
if all(x in labels_cum for x in cls):
for x in cls:
if x in labels:
idxs[x].append(i)
return idxs
def filter_images(dataset, labels, labels_old=None, overlap=True):
# Filter images without any label in LABELS (using labels not reordered)
idxs = []
if 0 in labels:
labels.remove(0)
print(f"Filtering images...")
if labels_old is None:
labels_old = []
labels_cum = labels + labels_old + [0, 255]
if overlap:
fil = lambda c: any(x in labels for x in cls)
else:
fil = lambda c: any(x in labels for x in cls) and all(x in labels_cum for x in c)
for i in range(len(dataset)):
cls = np.unique(np.array(dataset[i][1]))
if fil(cls):
idxs.append(i)
if i % 1000 == 0:
print(f"\t{i}/{len(dataset)} ...")
return idxs
class Subset(torch.utils.data.Dataset):
"""
Subset of a dataset at specified indices.
Arguments:
dataset (Dataset): The whole Dataset
indices (sequence): Indices in the whole set selected for subset
transform (callable): way to transform the images and the targets
target_transform(callable): way to transform the target labels
"""
def __init__(self, dataset, indices, transform=None, target_transform=None):
self.dataset = dataset
self.indices = indices
self.transform = transform
self.target_transform = target_transform
def __getitem__(self, idx):
sample, target = self.dataset[self.indices[idx]]
if self.transform is not None:
sample, target = self.transform(sample, target)
if self.target_transform is not None:
target = self.target_transform(target)
return sample, target
def __len__(self):
return len(self.indices)
class MaskLabels:
"""
Use this class to mask labels that you don't want in your dataset.
Arguments:
labels_to_keep (list): The list of labels to keep in the target images
mask_value (int): The value to replace ignored values (def: 0)
"""
def __init__(self, labels_to_keep, mask_value=0):
self.labels = labels_to_keep
self.value = torch.tensor(mask_value, dtype=torch.uint8)
def __call__(self, sample):
# sample must be a tensor
assert isinstance(sample, torch.Tensor), "Sample must be a tensor"
sample.apply_(lambda t: t.apply_(lambda x: x if x in self.labels else self.value))
return sample
from PIL import Image
classes = {
0: 'background',
1: 'aeroplane',
2: 'bicycle',
3: 'bird',
4: 'boat',
5: 'bottle',
6: 'bus',
7: 'car',
8: 'cat',
9: 'chair',
10: 'cow',
11: 'diningtable',
12: 'dog',
13: 'horse',
14: 'motorbike',
15: 'person',
16: 'pottedplant',
17: 'sheep',
18: 'sofa',
19: 'train',
20: 'tvmonitor'
}
class VOCSegmentation(data.Dataset):
"""`Pascal VOC <http://host.robots.ox.ac.uk/pascal/VOC/>`_ Segmentation Dataset.
Args:
root (string): Root directory of the VOC Dataset.
image_set (string, optional): Select the image_set to use, ``train``, ``trainval`` or ``val``
is_aug (bool, optional): If you want to use the augmented train set or not (default is True)
transform (callable, optional): A function/transform that takes in an PIL image
and returns a transformed version. E.g, ``transforms.RandomCrop``
"""
def __init__(self,
root,
image_set='train',
is_aug=True,
transform=None):
self.root = os.path.expanduser(root)
self.year = "2012"
self.transform = transform
self.image_set = image_set
base_dir = "PascalVOC12"
voc_root = os.path.join(self.root, base_dir)
splits_dir = os.path.join(voc_root, 'splits')
if not os.path.isdir(voc_root):
raise RuntimeError('Dataset not found or corrupted.' +
' You can use download=True to download it')
if is_aug and image_set == 'train':
mask_dir = os.path.join(voc_root, 'SegmentationClassAug')
assert os.path.exists(
mask_dir), "SegmentationClassAug not found"
split_f = os.path.join(splits_dir, 'train_aug.txt')
else:
split_f = os.path.join(splits_dir, image_set.rstrip('\n') + '.txt')
if not os.path.exists(split_f):
raise ValueError(
'Wrong image_set entered! Please use image_set="train" '
'or image_set="trainval" or image_set="val"')
# remove leading \n
with open(os.path.join(split_f), "r") as f:
file_names = [x[:-1].split(' ') for x in f.readlines()]
# REMOVE FIRST SLASH OTHERWISE THE JOIN WILL start from root
self.images = [(os.path.join(voc_root, x[0][1:]), os.path.join(voc_root, x[1][1:])) for x in file_names]
def __getitem__(self, index):
"""
Args:
index (int): Index
Returns:
tuple: (image, target) where target is the image segmentation.
"""
img = Image.open(self.images[index][0]).convert('RGB')
target = Image.open(self.images[index][1])
if self.transform is not None:
img, target = self.transform(img, target)
return img, target
def __len__(self):
return len(self.images)
class VOCSegmentationIncremental(data.Dataset):
def __init__(self,
root,
train=True,
transform=None,
labels=None,
labels_old=None,
idxs_path=None,
masking=True,
overlap=True):
full_voc = VOCSegmentation(root, 'train' if train else 'val', is_aug=True, transform=None)
self.labels = []
self.labels_old = []
self.train = train
if labels is not None:
# store the labels
labels_old = labels_old if labels_old is not None else []
self.__strip_zero(labels)
self.__strip_zero(labels_old)
assert not any(l in labels_old for l in labels), "labels and labels_old must be disjoint sets"
self.labels = [0] + labels
self.labels_old = [0] + labels_old
self.order = [0] + labels_old + labels
# take index of images with at least one class in labels and all classes in labels+labels_old+[0,255]
if idxs_path is not None and os.path.exists(idxs_path):
idxs = np.load(idxs_path).tolist()
else:
idxs = filter_images(full_voc, labels, labels_old, overlap=overlap)
if idxs_path is not None and distributed.get_rank() == 0:
np.save(idxs_path, np.array(idxs, dtype=int))
if masking:
target_transform = tv.transforms.Lambda(self.target_transform_func)
else:
target_transform = tv.transforms.Lambda(self.reorder_transform_func)
# make the subset of the dataset
self.dataset = Subset(full_voc, idxs, transform, target_transform)
else:
self.dataset = full_voc
def __getitem__(self, index):
"""
Args:
index (int): Index
Returns:
tuple: (image, target) where target is the image segmentation.
"""
return self.dataset[index]
def __len__(self):
return len(self.dataset)
def target_transform_func(self, t):
tmp_labels = self.labels + [255]
masking_value = 0 if self.train else 255
inverted_order = {label: self.order.index(label) for label in self.order}
inverted_order[255] = masking_value
return t.apply_(lambda x: inverted_order[x] if x in tmp_labels else masking_value)
def reorder_transform_func(self, t):
masking_value = 0 if self.train else 255
inverted_order = {label: self.order.index(label) for label in self.order}
inverted_order[255] = masking_value
return t.apply_(lambda x: inverted_order[x] if x in inverted_order else masking_value)
@staticmethod
def __strip_zero(labels):
while 0 in labels:
labels.remove(0)
Hey! There are some issue in the data processing. X is output when you don't have any GT pixels for the classes. Please check if you modified something in the dataset or please report the run command.
What I asked about was Validation result, and I found out it is correct with validation results.
I don't get any Xs in Test result.
By the way, thank you for your apply :)