Update code library

Author: Chengcheng-Guo

deepcore/__init__.py

@@ -0,0 +1 @@
+# __init__.py

deepcore/datasets/__init__.py

@@ -0,0 +1,8 @@
+from .cifar10 import *
+from .cifar100 import *
+from .fashionmnist import *
+from .imagenet import *
+from .mnist import *
+from .qmnist import *
+from .svhn import *
+from .tinyimagenet import *

deepcore/datasets/cifar10.py

@@ -0,0 +1,18 @@
+from torchvision import datasets, transforms
+from torch import tensor, long
+
+
+def CIFAR10(data_path):
+    channel = 3
+    im_size = (32, 32)
+    num_classes = 10
+    mean = [0.4914, 0.4822, 0.4465]
+    std = [0.2470, 0.2435, 0.2616]
+
+    transform = transforms.Compose([transforms.ToTensor(), transforms.Normalize(mean=mean, std=std)])
+    dst_train = datasets.CIFAR10(data_path, train=True, download=True, transform=transform)
+    dst_test = datasets.CIFAR10(data_path, train=False, download=True, transform=transform)
+    class_names = dst_train.classes
+    dst_train.targets = tensor(dst_train.targets, dtype=long)
+    dst_test.targets = tensor(dst_test.targets, dtype=long)
+    return channel, im_size, num_classes, class_names, mean, std, dst_train, dst_test

deepcore/datasets/cifar100.py

@@ -0,0 +1,17 @@
+from torchvision import datasets, transforms
+from torch import tensor, long
+
+
+def CIFAR100(data_path):
+    channel = 3
+    im_size = (32, 32)
+    num_classes = 100
+    mean = [0.5071, 0.4865, 0.4409]
+    std = [0.2673, 0.2564, 0.2762]
+    transform = transforms.Compose([transforms.ToTensor(), transforms.Normalize(mean=mean, std=std)])
+    dst_train = datasets.CIFAR100(data_path, train=True, download=True, transform=transform)
+    dst_test = datasets.CIFAR100(data_path, train=False, download=True, transform=transform)
+    class_names = dst_train.classes
+    dst_train.targets = tensor(dst_train.targets, dtype=long)
+    dst_test.targets = tensor(dst_test.targets, dtype=long)
+    return channel, im_size, num_classes, class_names, mean, std, dst_train, dst_test

deepcore/datasets/fashionmnist.py

@@ -0,0 +1,14 @@
+from torchvision import datasets, transforms
+
+
+def FashionMNIST(data_path):
+    channel = 1
+    im_size = (28, 28)
+    num_classes = 10
+    mean = [0.2861]
+    std = [0.3530]
+    transform = transforms.Compose([transforms.ToTensor(), transforms.Normalize(mean=mean, std=std)])
+    dst_train = datasets.FashionMNIST(data_path, train=True, download=True, transform=transform)
+    dst_test = datasets.FashionMNIST(data_path, train=False, download=True, transform=transform)
+    class_names = dst_train.classes
+    return channel, im_size, num_classes, class_names, mean, std, dst_train, dst_test

deepcore/datasets/imagenet.py

@@ -0,0 +1,27 @@
+from torchvision import datasets, transforms
+from torch import tensor, long
+
+
+def ImageNet(data_path):
+    channel = 3
+    im_size = (224, 224)
+    num_classes = 1000
+    mean = [0.485, 0.456, 0.406]
+    std = [0.229, 0.224, 0.225]
+    normalize = transforms.Normalize(mean, std)
+    dst_train = datasets.ImageNet(data_path, split="train", transform=transforms.Compose([
+            transforms.Resize(256),
+            transforms.CenterCrop(224),
+            transforms.ToTensor(),
+            normalize,
+        ]))
+    dst_test = datasets.ImageNet(data_path, split="val", transform=transforms.Compose([
+            transforms.Resize(256),
+            transforms.CenterCrop(224),
+            transforms.ToTensor(),
+            normalize,
+        ]))
+    class_names = dst_train.classes
+    dst_train.targets = tensor(dst_train.targets, dtype=long)
+    dst_test.targets = tensor(dst_test.targets, dtype=long)
+    return channel, im_size, num_classes, class_names, mean, std, dst_train, dst_test

deepcore/datasets/mnist.py

@@ -0,0 +1,25 @@
+from torchvision import datasets, transforms
+import numpy as np
+
+
+def MNIST(data_path, permuted=False, permutation_seed=None):
+    channel = 1
+    im_size = (28, 28)
+    num_classes = 10
+    mean = [0.1307]
+    std = [0.3081]
+    transform = transforms.Compose([transforms.ToTensor(), transforms.Normalize(mean=mean, std=std)])
+    if permuted:
+        np.random.seed(permutation_seed)
+        pixel_permutation = np.random.permutation(28 * 28)
+        transform = transforms.Compose(
+            [transform, transforms.Lambda(lambda x: x.view(-1, 1)[pixel_permutation].view(1, 28, 28))])
+
+    dst_train = datasets.MNIST(data_path, train=True, download=True, transform=transform)
+    dst_test = datasets.MNIST(data_path, train=False, download=True, transform=transform)
+    class_names = [str(c) for c in range(num_classes)]
+    return channel, im_size, num_classes, class_names, mean, std, dst_train, dst_test
+
+
+def permutedMNIST(data_path, permutation_seed=None):
+    return MNIST(data_path, True, permutation_seed)

deepcore/datasets/qmnist.py

@@ -0,0 +1,18 @@
+from torchvision import datasets, transforms
+
+
+def QMNIST(data_path):
+    channel = 1
+    im_size = (28, 28)
+    num_classes = 10
+    mean = [0.1308]
+    std = [0.3088]
+    transform = transforms.Compose([transforms.ToTensor(), transforms.Normalize(mean=mean, std=std)])
+    dst_train = datasets.QMNIST(data_path, train=True, download=True, transform=transform)
+    dst_test = datasets.QMNIST(data_path, train=False, download=True, transform=transform)
+    class_names = [str(c) for c in range(num_classes)]
+    dst_train.targets = dst_train.targets[:, 0]
+    dst_test.targets = dst_test.targets[:, 0]
+    dst_train.compat = False
+    dst_test.compat = False
+    return channel, im_size, num_classes, class_names, mean, std, dst_train, dst_test

deepcore/datasets/svhn.py

@@ -0,0 +1,14 @@
+from torchvision import datasets, transforms
+
+
+def SVHN(data_path):
+    channel = 3
+    im_size = (32, 32)
+    num_classes = 10
+    mean = [0.4377, 0.4438, 0.4728]
+    std = [0.1980, 0.2010, 0.1970]
+    transform = transforms.Compose([transforms.ToTensor(), transforms.Normalize(mean=mean, std=std)])
+    dst_train = datasets.SVHN(data_path, split='train', download=True, transform=transform)
+    dst_test = datasets.SVHN(data_path, split='test', download=True, transform=transform)
+    class_names = [str(c) for c in range(num_classes)]
+    return channel, im_size, num_classes, class_names, mean, std, dst_train, dst_test

deepcore/datasets/tinyimagenet.py

@@ -0,0 +1,35 @@
+from torchvision import datasets, transforms
+import os
+import requests
+import zipfile
+
+
+def TinyImageNet(data_path, downsize=True):
+    if not os.path.exists(os.path.join(data_path, "tiny-imagenet-200")):
+        url = "http://cs231n.stanford.edu/tiny-imagenet-200.zip"  # 248MB
+        print("Downloading Tiny-ImageNet")
+        r = requests.get(url, stream=True)
+        with open(os.path.join(data_path, "tiny-imagenet-200.zip"), "wb") as f:
+            for chunk in r.iter_content(chunk_size=1024):
+                if chunk:
+                    f.write(chunk)
+
+        print("Unziping Tiny-ImageNet")
+        with zipfile.ZipFile(os.path.join(data_path, "tiny-imagenet-200.zip")) as zf:
+            zf.extractall(path=data_path)
+
+    channel = 3
+    im_size = (32, 32) if downsize else (64, 64)
+    num_classes = 200
+    mean = (0.4802, 0.4481, 0.3975)
+    std = (0.2770, 0.2691, 0.2821)
+
+    transform = transforms.Compose([transforms.ToTensor(), transforms.Normalize(mean=mean, std=std)])
+    if downsize:
+        transform = transforms.Compose([transforms.Resize(32), transform])
+
+    dst_train = datasets.ImageFolder(root=os.path.join(data_path, 'tiny-imagenet-200/train'), transform=transform)
+    dst_test = datasets.ImageFolder(root=os.path.join(data_path, 'tiny-imagenet-200/test'), transform=transform)
+
+    class_names = dst_train.classes
+    return channel, im_size, num_classes, class_names, mean, std, dst_train, dst_test

deepcore/methods/__init__.py

@@ -0,0 +1,17 @@
+from .cal import *
+from .contextualdiversity import *
+from .coresetmethod import *
+from .craig import *
+from .deepfool import *
+from .earlytrain import *
+from .forgetting import *
+from .full import *
+from .glister import *
+from .grand import *
+from .gradmatch import *
+from .herding import *
+from .kcentergreedy import *
+from .submodular import *
+from .uncertainty import *
+from .uniform import *
+

deepcore/methods/cal.py

@@ -0,0 +1,140 @@
+from .earlytrain import EarlyTrain
+from .methods_utils.euclidean import euclidean_dist_pair_np
+from .methods_utils.cossim import cossim_pair_np
+import numpy as np
+import torch
+from .. import nets
+from copy import deepcopy
+from torchvision import transforms
+
+
+class Cal(EarlyTrain):
+    def __init__(self, dst_train, args, fraction=0.5, random_seed=None, epochs=200, specific_model=None,
+                 balance=True, metric="euclidean", neighbors: int = 10, pretrain_model: str = "ResNet18", **kwargs):
+        super().__init__(dst_train, args, fraction, random_seed, epochs, specific_model, **kwargs)
+
+        self.balance = balance
+
+        assert neighbors > 0 and neighbors < 100
+        self.neighbors = neighbors
+
+        if metric == "euclidean":
+            self.metric = euclidean_dist_pair_np
+        elif metric == "cossim":
+            self.metric = lambda a, b: -1. * cossim_pair_np(a, b)
+        elif callable(metric):
+            self.metric = metric
+        else:
+            self.metric = euclidean_dist_pair_np
+
+        self.pretrain_model = pretrain_model
+
+    def num_classes_mismatch(self):
+        raise ValueError("num_classes of pretrain dataset does not match that of the training dataset.")
+
+    def while_update(self, outputs, loss, targets, epoch, batch_idx, batch_size):
+        if batch_idx % self.args.print_freq == 0:
+            print('| Epoch [%3d/%3d] Iter[%3d/%3d]\t\tLoss: %.4f' % (
+                epoch, self.epochs, batch_idx + 1, (self.n_pretrain_size // batch_size) + 1, loss.item()))
+
+    def find_knn(self):
+        """
+        Find k-nearest-neighbor data points with the pretrained embedding model
+        :return: knn matrix
+        """
+
+        # Initialize pretrained model
+        model = nets.__dict__[self.pretrain_model](channel=self.args.channel, num_classes=self.args.num_classes,
+                                                   im_size=(224, 224), record_embedding=True, no_grad=True,
+                                                   pretrained=True).to(self.args.device)
+        model.eval()
+
+        # Resize dst_train to 224*224
+        if self.args.im_size[0] != 224 or self.args.im_size[1] != 224:
+            dst_train = deepcopy(self.dst_train)
+            dst_train.transform = transforms.Compose([dst_train.transform, transforms.Resize(224)])
+        else:
+            dst_train = self.dst_train
+
+        # Calculate the distance matrix and return knn results
+        if self.balance:
+            knn = []
+            for c in range(self.args.num_classes):
+                class_index = np.arange(self.n_train)[self.dst_train.targets == c]
+
+                # Start recording embedding vectors
+                embdeddings = []
+                batch_loader = torch.utils.data.DataLoader(torch.utils.data.Subset(dst_train, class_index),
+                                                           batch_size=self.args.selection_batch,
+                                                           num_workers=self.args.workers)
+                batch_num = len(batch_loader)
+                for i, (aa, _) in enumerate(batch_loader):
+                    if i % self.args.print_freq == 0:
+                        print("| Caculating embeddings for batch [%3d/%3d]" % (i + 1, batch_num))
+                    model(aa.to(self.args.device))
+                    embdeddings.append(model.embedding_recorder.embedding.flatten(1).cpu().numpy())
+
+                embdeddings = np.concatenate(embdeddings, axis=0)
+
+                knn.append(np.argsort(self.metric(embdeddings), axis=1)[:, 1:(self.neighbors + 1)])
+            return knn
+        else:
+            # Start recording embedding vectors
+            embdeddings = []
+            batch_loader = torch.utils.data.DataLoader(dst_train, batch_size=self.args.selection_batch
+                                                       ,num_workers=self.args.workers)
+            batch_num = len(batch_loader)
+
+            for i, (aa, _) in enumerate(batch_loader):
+                if i % self.args.print_freq == 0:
+                    print("| Caculating embeddings for batch [%3d/%3d]" % (i + 1, batch_num))
+                model(aa.to(self.args.device))
+                embdeddings.append(model.embedding_recorder.embedding.flatten(1).cpu().numpy())
+            embdeddings = np.concatenate(embdeddings, axis=0)
+
+            return np.argsort(self.metric(embdeddings), axis=1)[:, 1:(self.neighbors + 1)]
+
+    def calc_kl(self, knn, index=None):
+        self.model.eval()
+        self.model.no_grad = True
+        sample_num = self.n_train if index is None else len(index)
+        probs = np.zeros([sample_num, self.args.num_classes])
+
+        batch_loader = torch.utils.data.DataLoader(
+            self.dst_train if index is None else torch.utils.data.Subset(self.dst_train, index),
+            batch_size=self.args.selection_batch, num_workers=self.args.workers)
+        batch_num = len(batch_loader)
+
+        for i, (inputs, _) in enumerate(batch_loader):
+            probs[i * self.args.selection_batch:(i + 1) * self.args.selection_batch] = torch.nn.functional.softmax(
+                self.model(inputs.to(self.args.device)), dim=1).detach().cpu()
+
+        s = np.zeros(sample_num)
+        for i in range(0, sample_num, self.args.selection_batch):
+            if i % self.args.print_freq == 0:
+                print("| Caculating KL-divergence for batch [%3d/%3d]" % (i // self.args.selection_batch + 1, batch_num))
+            aa = np.expand_dims(probs[i:(i + self.args.selection_batch)], 1).repeat(self.neighbors, 1)
+            bb = probs[knn[i:(i + self.args.selection_batch)], :]
+            s[i:(i + self.args.selection_batch)] = np.mean(
+                np.sum(0.5 * aa * np.log(aa / bb) + 0.5 * bb * np.log(bb / aa), axis=2), axis=1)
+        self.model.no_grad = False
+        return s
+
+    def finish_run(self):
+        scores=[]
+        if self.balance:
+            selection_result = np.array([], dtype=np.int32)
+            for c, knn in zip(range(self.args.num_classes), self.knn):
+                class_index = np.arange(self.n_train)[self.dst_train.targets == c]
+                scores.append(self.calc_kl(knn, class_index))
+                selection_result = np.append(selection_result, class_index[np.argsort(
+                    #self.calc_kl(knn, class_index))[::1][:round(self.fraction * len(class_index))]])
+                    scores[-1])[::1][:round(self.fraction * len(class_index))]])
+        else:
+            selection_result = np.argsort(self.calc_kl(self.knn))[::1][:self.coreset_size]
+        return {"indices": selection_result, "scores":scores}
+
+    def select(self, **kwargs):
+        self.knn = self.find_knn()
+        selection_result = self.run()
+        return selection_result

deepcore/methods/contextualdiversity.py

@@ -0,0 +1,33 @@
+from .kcentergreedy import kCenterGreedy
+import torch
+
+
+# Acknowlegement to:
+# https://github.com/sharat29ag/CDAL
+
+
+class ContextualDiversity(kCenterGreedy):
+    def __init__(self, dst_train, args, fraction=0.5, random_seed=None, epochs=200,
+                 specific_model=None, balance=True, already_selected=[], torchvision_pretrain: bool = False, **kwargs):
+        super(ContextualDiversity, self).__init__(dst_train, args, fraction, random_seed, epochs=epochs, specific_model=specific_model, balance=balance, already_selected=already_selected, torchvision_pretrain=torchvision_pretrain, **kwargs)
+        self.metric = self._metric
+
+    def _metric(self, a_output, b_output):
+        with torch.no_grad():
+            # Overload self.metric function for kCenterGreedy Algorithm
+            aa = a_output.view(a_output.shape[0], 1, a_output.shape[1]).repeat(1, b_output.shape[0], 1)
+            bb = b_output.view(1, b_output.shape[0], b_output.shape[1]).repeat(a_output.shape[0], 1, 1)
+            return torch.sum(0.5 * aa * torch.log(aa / bb) + 0.5 * bb * torch.log(bb / aa), dim=2)
+
+    def construct_matrix(self, index=None):
+        self.model.eval()
+        self.model.no_grad = True
+        sample_num = self.n_train if index is None else len(index)
+        matrix = torch.zeros([sample_num, self.args.num_classes], requires_grad=False).to(self.args.device)
+        batch_loader = torch.utils.data.DataLoader(self.dst_train if index is None else
+                            torch.utils.data.Subset(self.dst_train, index), batch_size=self.args.selection_batch
+                                                   ,num_workers=self.args.workers)
+        for i, (inputs, _) in enumerate(batch_loader):
+            matrix[i * self.args.selection_batch:min((i + 1) * self.args.selection_batch, sample_num)] = torch.nn.functional.softmax(self.model(inputs.to(self.args.device)), dim=1)
+        self.model.no_grad = False
+        return matrix

deepcore/methods/coresetmethod.py

@@ -0,0 +1,17 @@
+class CoresetMethod(object):
+    def __init__(self, dst_train, args, fraction=0.5, random_seed=None, **kwargs):
+        if fraction <= 0.0 or fraction > 1.0:
+            raise ValueError("Illegal Coreset Size.")
+        self.dst_train = dst_train
+        self.num_classes = len(dst_train.classes)
+        self.fraction = fraction
+        self.random_seed = random_seed
+        self.index = []
+        self.args = args
+
+        self.n_train = len(dst_train)
+        self.coreset_size = round(self.n_train * fraction)
+
+    def select(self, **kwargs):
+        return
+