import sys from torchvision import datasets, transforms from base import BaseDataLoader from data_loader.cifar10 import get_cifar10 from data_loader.cifar100 import get_cifar100 from data_loader.clothing1m import get_clothing from data_loader.webvision import get_webvision from parse_config import ConfigParser import numpy as np import torch import torch.nn.functional as F from PIL import Image class CIFAR10DataLoader(BaseDataLoader): def __init__(self, data_dir, batch_size, shuffle=True, validation_split=0.0, num_batches=0, training=True, num_workers=4, pin_memory=True): config = ConfigParser.get_instance() cfg_trainer = config['trainer'] transform_train = transforms.Compose([ transforms.RandomCrop(32, padding=4), transforms.RandomHorizontalFlip(), transforms.ToTensor(), transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)), ]) transform_val = transforms.Compose([ transforms.ToTensor(), transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)), ]) self.data_dir = data_dir noise_file='%sCIFAR10_%.1f_Asym_%s.json'%(config['data_loader']['args']['data_dir'],cfg_trainer['percent'],cfg_trainer['asym']) self.train_dataset, self.val_dataset = get_cifar10(config['data_loader']['args']['data_dir'], cfg_trainer, train=training, transform_train=transform_train, transform_val=transform_val, noise_file = noise_file) super().__init__(self.train_dataset, batch_size, shuffle, validation_split, num_workers, pin_memory, val_dataset = self.val_dataset) class CIFAR100DataLoader(BaseDataLoader): def __init__(self, data_dir, batch_size, shuffle=True, validation_split=0.0, num_batches=0, training=True,num_workers=4, pin_memory=True): config = ConfigParser.get_instance() cfg_trainer = config['trainer'] transform_train = transforms.Compose([ #transforms.ColorJitter(brightness= 0.4, contrast= 0.4, saturation= 0.4, hue= 0.1), transforms.RandomCrop(32, padding=4), transforms.RandomHorizontalFlip(), transforms.ToTensor(), transforms.Normalize((0.5071, 0.4867, 0.4408), (0.2675, 0.2565, 0.2761)), ]) transform_val = transforms.Compose([ transforms.ToTensor(), transforms.Normalize((0.5071, 0.4867, 0.4408), (0.2675, 0.2565, 0.2761)), ]) self.data_dir = data_dir noise_file='%sCIFAR100_%.1f_Asym_%s.json'%(config['data_loader']['args']['data_dir'],cfg_trainer['percent'],cfg_trainer['asym']) self.train_dataset, self.val_dataset = get_cifar100(config['data_loader']['args']['data_dir'], cfg_trainer, train=training, transform_train=transform_train, transform_val=transform_val, noise_file = noise_file) super().__init__(self.train_dataset, batch_size, shuffle, validation_split, num_workers, pin_memory, val_dataset = self.val_dataset) self.batch_size_ = int(batch_size) class Clothing1MDataLoader(BaseDataLoader): def __init__(self, data_dir, batch_size, shuffle=True, validation_split=0.0, num_batches=0, training=True, num_workers=4, pin_memory=True): self.batch_size = batch_size self.num_workers = num_workers self.num_batches = num_batches self.training = training self.transform_train = transforms.Compose([ transforms.Resize(256), transforms.RandomCrop(224), transforms.RandomHorizontalFlip(), transforms.ToTensor(), transforms.Normalize((0.6959, 0.6537, 0.6371),(0.3113, 0.3192, 0.3214)), ]) self.transform_val = transforms.Compose([ transforms.Resize(256), transforms.CenterCrop(224), transforms.ToTensor(), transforms.Normalize((0.6959, 0.6537, 0.6371),(0.3113, 0.3192, 0.3214)), ]) self.data_dir = data_dir config = ConfigParser.get_instance() cfg_trainer = config['trainer'] self.train_dataset, self.val_dataset = get_clothing(config['data_loader']['args']['data_dir'], cfg_trainer, num_samples=self.num_batches*self.batch_size, train=training, transform_train=self.transform_train, transform_val=self.transform_val) super().__init__(self.train_dataset, batch_size, shuffle, validation_split, num_workers, pin_memory, val_dataset = self.val_dataset) class WebvisionDataLoader(BaseDataLoader): def __init__(self, data_dir, batch_size, shuffle=True, validation_split=0.0, num_batches=0, training=True, num_workers=4, pin_memory=True, num_class = 50): self.batch_size = batch_size self.num_workers = num_workers self.num_batches = num_batches self.training = training self.transform_train = transforms.Compose([ transforms.RandomCrop(227), transforms.RandomHorizontalFlip(), transforms.ToTensor(), transforms.Normalize((0.485, 0.456, 0.406),(0.229, 0.224, 0.225)), ]) self.transform_val = transforms.Compose([ transforms.CenterCrop(227), transforms.ToTensor(), transforms.Normalize((0.485, 0.456, 0.406),(0.229, 0.224, 0.225)), ]) self.transform_imagenet = transforms.Compose([ transforms.Resize(256), transforms.CenterCrop(227), transforms.ToTensor(), transforms.Normalize((0.485, 0.456, 0.406),(0.229, 0.224, 0.225)), ]) self.data_dir = data_dir config = ConfigParser.get_instance() cfg_trainer = config['trainer'] self.train_dataset, self.val_dataset = get_webvision(config['data_loader']['args']['data_dir'], cfg_trainer, num_samples=self.num_batches*self.batch_size, train=training, transform_train=self.transform_train, transform_val=self.transform_val, num_class = num_class) super().__init__(self.train_dataset, batch_size, shuffle, validation_split, num_workers, pin_memory, val_dataset = self.val_dataset)