remove

-# run train.py --dataset cifar10 --model resnet18 --data_augmentation --cutout --length 16
-# run train.py --dataset cifar100 --model resnet18 --data_augmentation --cutout --length 8
-# run train.py --dataset svhn --model wideresnet --learning_rate 0.01 --epochs 160 --cutout --length 20
-
-import pdb
-import argparse
-import numpy as np
-from tqdm import tqdm
-import os
-
-import torch
-import torch.nn as nn
-from torch.autograd import Variable
-import torch.backends.cudnn as cudnn
-from torch.optim.lr_scheduler import MultiStepLR
-
-from torchvision.utils import make_grid, save_image
-from torchvision import datasets, transforms
-
-from torch.utils.data.dataloader import RandomSampler
-from util.misc import CSVLogger
-from util.cutout import Cutout
-
-from model.resnet import ResNet18
-from model.wide_resnet import WideResNet
-
-model_options = ['resnet18', 'wideresnet']
-dataset_options = ['cifar10', 'cifar100', 'svhn']
-
-parser = argparse.ArgumentParser(description='CNN')
-parser.add_argument('--dataset', '-d', default='cifar10',
-                    choices=dataset_options)
-parser.add_argument('--model', '-a', default='resnet18',
-                    choices=model_options)
-parser.add_argument('--batch_size', type=int, default=1,
-                    help='input batch size for training (default: 128)')
-parser.add_argument('--epochs', type=int, default=200,
-                    help='number of epochs to train (default: 20)')
-parser.add_argument('--learning_rate', type=float, default=0.1,
-                    help='learning rate')
-parser.add_argument('--data_augmentation', action='store_true', default=False,
-                    help='augment data by flipping and cropping')
-parser.add_argument('--cutout', action='store_true', default=False,
-                    help='apply cutout')
-parser.add_argument('--n_holes', type=int, default=1,
-                    help='number of holes to cut out from image')
-parser.add_argument('--length', type=int, default=16,
-                    help='length of the holes')
-parser.add_argument('--no-cuda', action='store_true', default=False,
-                    help='enables CUDA training')
-parser.add_argument('--seed', type=int, default=0,
-                    help='random seed (default: 1)')
-
-args = parser.parse_args()
-args.cuda = not args.no_cuda and torch.cuda.is_available()
-cudnn.benchmark = True  # Should make training should go faster for large models
-
-torch.manual_seed(args.seed)
-if args.cuda:
-    torch.cuda.manual_seed(args.seed)
-
-test_id = args.dataset + '_' + args.model
-
-print(args)
-
-# Image Preprocessing
-if args.dataset == 'svhn':
-    normalize = transforms.Normalize(mean=[x / 255.0 for x in[109.9, 109.7, 113.8]],
-                                     std=[x / 255.0 for x in [50.1, 50.6, 50.8]])
-else:
-    normalize = transforms.Normalize(mean=[x / 255.0 for x in [125.3, 123.0, 113.9]],
-                                     std=[x / 255.0 for x in [63.0, 62.1, 66.7]])
-
-train_transform = transforms.Compose([])
-if args.data_augmentation:
-    train_transform.transforms.append(transforms.RandomCrop(32, padding=4))
-    train_transform.transforms.append(transforms.RandomHorizontalFlip())
-train_transform.transforms.append(transforms.ToTensor())
-train_transform.transforms.append(normalize)
-if args.cutout:
-    train_transform.transforms.append(Cutout(n_holes=args.n_holes, length=args.length))
-
-
-test_transform = transforms.Compose([
-    transforms.ToTensor(),
-    normalize])
-
-if args.dataset == 'cifar10':
-    num_classes = 10
-    train_dataset = datasets.CIFAR10(root='data/',
-                                     train=True,
-                                     transform=train_transform,
-                                     download=True)
-
-    test_dataset = datasets.CIFAR10(root='data/',
-                                    train=False,
-                                    transform=test_transform,
-                                    download=True)
-elif args.dataset == 'cifar100':
-    num_classes = 100
-    train_dataset = datasets.CIFAR100(root='data/',
-                                      train=True,
-                                      transform=train_transform,
-                                      download=True)
-
-    test_dataset = datasets.CIFAR100(root='data/',
-                                     train=False,
-                                     transform=test_transform,
-                                     download=True)
-elif args.dataset == 'svhn':
-    num_classes = 10
-    train_dataset = datasets.SVHN(root='data/',
-                                  split='train',
-                                  transform=train_transform,
-                                  download=True)
-
-    extra_dataset = datasets.SVHN(root='data/',
-                                  split='extra',
-                                  transform=train_transform,
-                                  download=True)
-
-    # Combine both training splits (https://arxiv.org/pdf/1605.07146.pdf)
-    data = np.concatenate([train_dataset.data, extra_dataset.data], axis=0)
-    labels = np.concatenate([train_dataset.labels, extra_dataset.labels], axis=0)
-    train_dataset.data = data
-    train_dataset.labels = labels
-
-    test_dataset = datasets.SVHN(root='data/',
-                                 split='test',
-                                 transform=test_transform,
-                                 download=True)
-
-# Data Loader (Input Pipeline)
-train_loader = torch.utils.data.DataLoader(dataset=train_dataset,
-                                           batch_size=args.batch_size,
-                                           shuffle=False,
-                                        #    sampler=RandomSampler(train_dataset, True, 40000),
-                                           pin_memory=True,
-                                           num_workers=0)
-
-test_loader = torch.utils.data.DataLoader(dataset=test_dataset,
-                                          batch_size=args.batch_size,
-                                          shuffle=False,
-                                          pin_memory=True,
-                                          num_workers=0)
-
-if args.model == 'resnet18':
-    cnn = ResNet18(num_classes=num_classes)
-elif args.model == 'wideresnet':
-    if args.dataset == 'svhn':
-        cnn = WideResNet(depth=16, num_classes=num_classes, widen_factor=8,
-                         dropRate=0.4)
-    else:
-        cnn = WideResNet(depth=28, num_classes=num_classes, widen_factor=10,
-                         dropRate=0.3)
-
-cnn = cnn.cuda()
-
-criterion = nn.CrossEntropyLoss().cuda()
-cnn_optimizer = torch.optim.SGD(cnn.parameters(), lr=args.learning_rate,
-                                momentum=0.9, nesterov=True, weight_decay=5e-4)
-
-if args.dataset == 'svhn':
-    scheduler = MultiStepLR(cnn_optimizer, milestones=[80, 120], gamma=0.1)
-else:
-    scheduler = MultiStepLR(cnn_optimizer, milestones=[60, 120, 160], gamma=0.2)
-
-filename = 'logs/' + test_id + '.csv'
-csv_logger = CSVLogger(args=args, fieldnames=['epoch', 'train_acc', 'test_acc', 'xentropy', 'var', 'avg_var', 'arg_var', 'index', 'labels'], filename=filename)
-
-
-def test(loader):
-    cnn.eval()    # Change model to 'eval' mode (BN uses moving mean/var).
-    correct = 0.
-    total = 0.
-    for images, labels in loader:
-        images = images.cuda()
-        labels = labels.cuda()
-
-        with torch.no_grad():
-            pred = cnn(images)
-
-        pred = torch.max(pred.data, 1)[1]
-        total += labels.size(0)
-        correct += (pred == labels).sum().item()
-
-    val_acc = correct / total
-    cnn.train()
-    return val_acc
-
-kl_sum = 0
-y_bar = torch.zeros(8, 10).detach().cuda()
-
-# y_bar 구하는 epoch
-for epoch in range(1):
-    checkpoint = torch.load('C:/Users/82109/Desktop/캡디/캡디자료들/논문모델/cutout/checkpoints/sampling/sampling_{0}.pt'.format(8), map_location = torch.device('cuda:0'))
-    cnn.load_state_dict(checkpoint)
-    cnn.eval()
-    xentropy_loss_avg = 0.
-    correct = 0.
-    total = 0.
-    norm_const = 0
-
-    kldiv = 0
-    # pred_sum = torch.Tensor([0] * 10).detach().cuda()
-    count = 0
-    label_list = []
-    progress_bar = tqdm(train_loader)
-    for i, (images, labels) in enumerate(progress_bar):
-        progress_bar.set_description('Epoch ' + str(epoch))
-
-        images = images.cuda()
-        labels = labels.cuda()
-        label_list.append(labels.item())
-        save_image(images[0], os.path.join('C:/Users/82109/Desktop/캡디/캡디자료들/논문모델/cutout/augmented_images/{0}/'.format(8), 'img{0}.png'.format(i)))
-
-        cnn.zero_grad()
-        pred = cnn(images)
-        xentropy_loss = criterion(pred, labels)
-        # xentropy_loss.backward()
-        # cnn_optimizer.step()
-
-        xentropy_loss_avg += xentropy_loss.item()
-
-        pred_softmax = nn.functional.softmax(pred).cuda()
-        # Calculate running average of accuracy
-        pred = torch.max(pred.data, 1)[1]
-        total += labels.size(0)
-        correct += (pred == labels.data).sum().item()
-        accuracy = correct / total
-        for a in range(pred_softmax.data.size()[0]):
-            for b in range(y_bar.size()[1]):
-                y_bar[epoch][b] += torch.log(pred_softmax.data[a][b])
-        
-
-        progress_bar.set_postfix(
-            xentropy='%.3f' % (xentropy_loss_avg / (i + 1)),
-            acc='%.3f' % accuracy)
-        count += 1
-    xentropy = xentropy_loss_avg / count
-    y_bar[epoch] = torch.Tensor([x / 50000 for x in y_bar[epoch]]).cuda()
-    y_bar[epoch] = torch.exp(y_bar[epoch])
-    for index in range(y_bar.size()[1]):
-        norm_const += y_bar[epoch][index]
-    for index in range(y_bar.size()[1]):
-        y_bar[epoch][index] = y_bar[epoch][index] / norm_const
-    print("y_bar[{0}] : ".format(epoch), y_bar[epoch])
-    test_acc = test(test_loader)
-    # print(pred, labels.data)
-    tqdm.write('test_acc: %.3f' % (test_acc))
-
-    scheduler.step(epoch)  # Use this line for PyTorch <1.4
-    # scheduler.step()     # Use this line for PyTorch >=1.4
-
-    row = {'epoch': str(epoch), 'train_acc': str(accuracy), 'test_acc': str(test_acc), 'xentropy' : float(xentropy)
-    }
-    csv_logger.writerow(row)
-    # del pred
-    # torch.cuda.empty_cache()
-
-
-var_tensor = torch.zeros(8, 50000).detach().cuda()
-var_addeachcol = torch.zeros(1, 50000).detach().cuda()
-
-# kl_div 구하는 epoch
-for epoch in range(1):
-    checkpoint = torch.load('C:/Users/82109/Desktop/캡디/캡디자료들/논문모델/cutout/checkpoints/sampling/sampling_{0}.pt'.format(8), map_location = torch.device('cuda:0'))
-    cnn.load_state_dict(checkpoint)
-    cnn.eval()
-    kldiv = 0
-    for i, (images, labels) in enumerate(progress_bar):
-        progress_bar.set_description('Epoch ' + str(epoch) + ': Calculate kl_div')
-        
-        images = images.cuda()
-        labels = labels.cuda()
-
-        cnn.zero_grad()
-        pred = cnn(images)
-
-        pred_softmax = nn.functional.softmax(pred).cuda()
-
-        # 입력 두 개의 shape이 다르면 batchsize로 평균을 내서 반환.
-        kldiv = torch.nn.functional.kl_div(y_bar[epoch], pred_softmax, reduction='sum')
-        # 1 * 50000에 한 모델의 데이터별 variance 저장
-        var_tensor[epoch][i] += abs(kldiv).detach()
-        var_addeachcol[0][i] += var_tensor[epoch][i]
-        kl_sum += kldiv.detach()
-        # print(y_bar_copy.size(), pred_softmax.size())
-        # print(kl_sum)
-    var = abs(kl_sum.item() / 50000)
-    print("Variance : ", var)
-    csv_logger.writerow({'var' : float(var)})
-    # print(var_tensor)
-for i in range(var_addeachcol.size()[1]):
-    var_addeachcol[0][i] = var_addeachcol[0][i] / 8
-
-print(var_addeachcol)
-# var_addeachcol[0] = torch.Tensor([x / 8 for x in var_addeachcol]).cuda()
-var_sorted = torch.argsort(var_addeachcol)
-print(var_sorted)
-for i in range(var_addeachcol.size()[1]):
-    csv_logger.writerow({'avg_var' : float(var_addeachcol[0][i]), 'arg_var' : float(var_sorted[0][i]), 'index' : float(i + 1), 'labels' : float(label_list[i])})
-torch.save(cnn.state_dict(), 'checkpoints/' + test_id + '.pt')
-csv_logger.close()

-# run train.py --dataset cifar10 --model resnet18 --data_augmentation --cutout --length 16
-# run train.py --dataset cifar100 --model resnet18 --data_augmentation --cutout --length 8
-# run train.py --dataset svhn --model wideresnet --learning_rate 0.01 --epochs 160 --cutout --length 20
-
-import pdb
-import argparse
-import numpy as np
-from tqdm import tqdm
-
-import torch
-import torch.nn as nn
-from torch.autograd import Variable
-import torch.backends.cudnn as cudnn
-from torch.optim.lr_scheduler import MultiStepLR
-
-from torchvision.utils import make_grid
-from torchvision import datasets, transforms
-
-from torch.utils.data.dataloader import RandomSampler
-from util.misc import CSVLogger
-from util.cutout import Cutout
-
-from model.resnet import ResNet18
-from model.wide_resnet import WideResNet
-
-model_options = ['resnet18', 'wideresnet']
-dataset_options = ['cifar10', 'cifar100', 'svhn']
-
-parser = argparse.ArgumentParser(description='CNN')
-parser.add_argument('--dataset', '-d', default='cifar10',
-                    choices=dataset_options)
-parser.add_argument('--model', '-a', default='resnet18',
-                    choices=model_options)
-parser.add_argument('--batch_size', type=int, default=128,
-                    help='input batch size for training (default: 128)')
-parser.add_argument('--epochs', type=int, default=200,
-                    help='number of epochs to train (default: 20)')
-parser.add_argument('--learning_rate', type=float, default=0.1,
-                    help='learning rate')
-parser.add_argument('--data_augmentation', action='store_true', default=False,
-                    help='augment data by flipping and cropping')
-parser.add_argument('--cutout', action='store_true', default=False,
-                    help='apply cutout')
-parser.add_argument('--n_holes', type=int, default=1,
-                    help='number of holes to cut out from image')
-parser.add_argument('--length', type=int, default=16,
-                    help='length of the holes')
-parser.add_argument('--no-cuda', action='store_true', default=False,
-                    help='enables CUDA training')
-parser.add_argument('--seed', type=int, default=0,
-                    help='random seed (default: 1)')
-
-args = parser.parse_args()
-args.cuda = not args.no_cuda and torch.cuda.is_available()
-cudnn.benchmark = True  # Should make training should go faster for large models
-
-torch.manual_seed(args.seed)
-if args.cuda:
-    torch.cuda.manual_seed(args.seed)
-
-test_id = args.dataset + '_' + args.model
-
-print(args)
-
-# Image Preprocessing
-if args.dataset == 'svhn':
-    normalize = transforms.Normalize(mean=[x / 255.0 for x in[109.9, 109.7, 113.8]],
-                                     std=[x / 255.0 for x in [50.1, 50.6, 50.8]])
-else:
-    normalize = transforms.Normalize(mean=[x / 255.0 for x in [125.3, 123.0, 113.9]],
-                                     std=[x / 255.0 for x in [63.0, 62.1, 66.7]])
-
-train_transform = transforms.Compose([])
-if args.data_augmentation:
-    train_transform.transforms.append(transforms.RandomCrop(32, padding=4))
-    train_transform.transforms.append(transforms.RandomHorizontalFlip())
-train_transform.transforms.append(transforms.ToTensor())
-train_transform.transforms.append(normalize)
-if args.cutout:
-    train_transform.transforms.append(Cutout(n_holes=args.n_holes, length=args.length))
-
-
-test_transform = transforms.Compose([
-    transforms.ToTensor(),
-    normalize])
-
-if args.dataset == 'cifar10':
-    num_classes = 10
-    train_dataset = datasets.CIFAR10(root='data/',
-                                     train=True,
-                                     transform=train_transform,
-                                     download=True)
-
-    test_dataset = datasets.CIFAR10(root='data/',
-                                    train=False,
-                                    transform=test_transform,
-                                    download=True)
-elif args.dataset == 'cifar100':
-    num_classes = 100
-    train_dataset = datasets.CIFAR100(root='data/',
-                                      train=True,
-                                      transform=train_transform,
-                                      download=True)
-
-    test_dataset = datasets.CIFAR100(root='data/',
-                                     train=False,
-                                     transform=test_transform,
-                                     download=True)
-elif args.dataset == 'svhn':
-    num_classes = 10
-    train_dataset = datasets.SVHN(root='data/',
-                                  split='train',
-                                  transform=train_transform,
-                                  download=True)
-
-    extra_dataset = datasets.SVHN(root='data/',
-                                  split='extra',
-                                  transform=train_transform,
-                                  download=True)
-
-    # Combine both training splits (https://arxiv.org/pdf/1605.07146.pdf)
-    data = np.concatenate([train_dataset.data, extra_dataset.data], axis=0)
-    labels = np.concatenate([train_dataset.labels, extra_dataset.labels], axis=0)
-    train_dataset.data = data
-    train_dataset.labels = labels
-
-    test_dataset = datasets.SVHN(root='data/',
-                                 split='test',
-                                 transform=test_transform,
-                                 download=True)
-
-# Data Loader (Input Pipeline)
-train_loader = torch.utils.data.DataLoader(dataset=train_dataset,
-                                           batch_size=args.batch_size,
-                                           shuffle=False,
-                                        #    sampler=RandomSampler(train_dataset, True, 40000),
-                                           pin_memory=True,
-                                           num_workers=0)
-
-test_loader = torch.utils.data.DataLoader(dataset=test_dataset,
-                                          batch_size=args.batch_size,
-                                          shuffle=False,
-                                          pin_memory=True,
-                                          num_workers=0)
-
-if args.model == 'resnet18':
-    cnn = ResNet18(num_classes=num_classes)
-elif args.model == 'wideresnet':
-    if args.dataset == 'svhn':
-        cnn = WideResNet(depth=16, num_classes=num_classes, widen_factor=8,
-                         dropRate=0.4)
-    else:
-        cnn = WideResNet(depth=28, num_classes=num_classes, widen_factor=10,
-                         dropRate=0.3)
-checkpoint = torch.load('/content/drive/MyDrive/capstone/Cutout/checkpoints/baseline_cifar10_resnet18.pt', map_location = torch.device('cuda:0'))
-cnn = cnn.cuda()
-cnn.load_state_dict(checkpoint)
-criterion = nn.CrossEntropyLoss().cuda()
-cnn_optimizer = torch.optim.SGD(cnn.parameters(), lr=args.learning_rate,
-                                momentum=0.9, nesterov=True, weight_decay=5e-4)
-
-if args.dataset == 'svhn':
-    scheduler = MultiStepLR(cnn_optimizer, milestones=[80, 120], gamma=0.1)
-else:
-    scheduler = MultiStepLR(cnn_optimizer, milestones=[60, 120, 160], gamma=0.2)
-
-filename = 'logs/' + test_id + '.csv'
-csv_logger = CSVLogger(args=args, fieldnames=['epoch', 'train_acc', 'test_acc'], filename=filename)
-
-
-def test(loader):
-    cnn.eval()    # Change model to 'eval' mode (BN uses moving mean/var).
-    correct = 0.
-    total = 0.
-    for images, labels in loader:
-        images = images.cuda()
-        labels = labels.cuda()
-
-        with torch.no_grad():
-            pred = cnn(images)
-
-        pred = torch.max(pred.data, 1)[1]
-        
-        total += labels.size(0)
-        correct += (pred == labels).sum().item()
-
-    val_acc = correct / total
-    cnn.train()
-    return val_acc
-
-kl_sum = 0
-y_bar = torch.Tensor([0] * 10).detach().cuda()
-
-# y_bar 구하는 epoch
-for epoch in range(args.epochs):
-
-    cnn.eval()
-    xentropy_loss_avg = 0.
-    correct = 0.
-    total = 0.
-    norm_const = 0
-
-    kldiv = 0
-    # pred_sum = torch.Tensor([0] * 10).detach().cuda()
-    
-    progress_bar = tqdm(train_loader)
-    for i, (images, labels) in enumerate(progress_bar):
-        progress_bar.set_description('Epoch ' + str(epoch))
-
-        images = images.cuda()
-        labels = labels.cuda()
-
-        cnn.zero_grad()
-        pred = cnn(images)
-        xentropy_loss = criterion(pred, labels)
-        # xentropy_loss.backward()
-        # cnn_optimizer.step()
-
-        xentropy_loss_avg += xentropy_loss.item()
-
-        pred_softmax = nn.functional.softmax(pred).cuda()
-        # Calculate running average of accuracy
-        pred = torch.max(pred.data, 1)[1]
-        total += labels.size(0)
-        correct += (pred == labels.data).sum().item()
-        accuracy = correct / total
-        for a in range(pred_softmax.data.size()[0]):
-            for b in range(y_bar.size()[0]):
-                y_bar[b] += torch.log(pred_softmax.data[a][b])
-        
-
-        # expectation(log y_hat)        
-        # y_bar = [x / pred.data.size()[0] for x in y_bar]
-
-        # print(pred.data.size()[0], y_bar.size()[0]) # 128, 10
-        
-
-        # print(pred)
-        # y_hat : 모델별 예측값  --> pred_softmax
-        # y_bar : 예측값들 평균값  -- > pred / total : pred_sum
-        # labes.data : ground_truth
-
-        # y_bar = pred_sum / (i+1)
-        # kl = torch.nn.functional.kl_div(pred, y_bar)
-        # kl_sum += kl
-        
-
-        
-        # for문 추가안하면 epoch별 iter마다 xentropy_loss_avg값의 1/iter이 xentropy값으로 출력
-        # for문 추가하면 epoch 별 iter 마다 xentropy_loss_avg 값은 동일하나 xentropy값 출력이 x_l_avg 값의 1/10으로 출력
-        # for문 상관 없이 pred, labels 값은 동일하게 확인됨.
-
-        # for a in range(list(pred_sum.size())[0]):
-        #     for b in range(list(pred.size())[0]):
-        #         if pred[b] == a:
-        #             pred_sum[a] += 1
-        
-        # variance calculate : E[KL_div(y_bar, y_hat)] -> expectation of KLDivLoss(pred_sum, pred)
-        # 한 epoch마다 계산해서 출력해야 할듯
-        # nn.functional.kl_div(pred_sum, pred)
-        
-        
-        # print('\n',i, ' ', xentropy_loss_avg)
-        progress_bar.set_postfix(
-            # y_hat = '%.5f' % pred,
-            # y_bar = '%.5f' % y_bar,
-            # groun_truth = '%.5f' % labels.data,
-            # kl = '%.3f' % kl.item(),
-            # kl_sum = '%.3f' % (kl_sum.item()),
-            # kl_div = '%.3f' % (kl_sum.item() / (i + 1)), # kl_div 호출
-            xentropy='%.3f' % (xentropy_loss_avg / (i + 1)),
-            acc='%.3f' % accuracy)
-    # pred_sum = [x / 40000 for x in pred_sum]
-    y_bar = torch.Tensor([x / 50000 for x in y_bar]).cuda()
-    y_bar = torch.exp(y_bar)
-    # print(y_bar)
-    for index in range(y_bar.size()[0]):
-        norm_const += y_bar[index]
-    print(y_bar)
-    print(norm_const)
-    # print(norm_const)
-    for index in range(y_bar.size()[0]):
-        y_bar[index] = y_bar[index] / norm_const
-    print(y_bar)
-    # print(y_bar)
-    # print(pred_softmax)
-    # print(y_bar)
-    # kldiv = torch.nn.functional.kl_div(y_bar, pred_softmax, reduction='batchmean')
-    # kl_sum += kldiv
-    # print(kldiv, kl_sum)
-    y_bar_copy = y_bar.clone().detach()
-    test_acc = test(test_loader)
-    # print(pred, labels.data)
-    tqdm.write('test_acc: %.3f' % (test_acc))
-
-    scheduler.step(epoch)  # Use this line for PyTorch <1.4
-    # scheduler.step()     # Use this line for PyTorch >=1.4
-
-    row = {'epoch': str(epoch), 'train_acc': str(accuracy), 'test_acc': str(test_acc)
-    }
-    csv_logger.writerow(row)
-    del pred
-    torch.cuda.empty_cache()
-
-# kl_div 구하는 epoch
-for epoch in range(args.epochs):
-    cnn.eval()
-    kldiv = 0
-    for i, (images, labels) in enumerate(progress_bar):
-        progress_bar.set_description('Epoch ' + str(epoch) + ': Calculate kl_div')
-
-        images = images.cuda()
-        labels = labels.cuda()
-
-        cnn.zero_grad()
-        pred = cnn(images)
-
-        pred_softmax = nn.functional.softmax(pred).cuda()
-
-        # 입력 두 개의 shape이 다르면 batchsize로 평균을 내서 반환.
-        kldiv = torch.nn.functional.kl_div(y_bar_copy, pred_softmax, reduction='sum')
-        kl_sum += kldiv.detach()
-        # print(y_bar_copy.size(), pred_softmax.size())
-        # print(kl_sum)
-    print("Average KL_div : ", abs(kl_sum / 50000))
-        # y_bar = torch.Tensor([x / 40000 for x in y_bar]).cuda()
-        # y_bar = torch.exp(y_bar)
-        # # print(y_bar)
-        # for index in range(y_bar.size()[0]):
-        #     norm_const += y_bar[index]
-        # # print(norm_const)
-        # for index in range(y_bar.size()[0]):
-        #     y_bar[index] = y_bar[index] / norm_const
-        # # print(y_bar)
-        # # print(pred_softmax)
-        # # print(y_bar)
-        # kldiv = torch.nn.functional.kl_div(y_bar, pred_softmax, reduction='batchmean')
-        # kl_sum += kldiv
-        # print(kldiv, kl_sum)
-
-torch.save(cnn.state_dict(), 'checkpoints/' + test_id + '.pt')
-csv_logger.close()

-# run train.py --dataset cifar10 --model resnet18 --data_augmentation --cutout --length 16
-# run train.py --dataset cifar100 --model resnet18 --data_augmentation --cutout --length 8
-# run train.py --dataset svhn --model wideresnet --learning_rate 0.01 --epochs 160 --cutout --length 20
-
-import pdb
-import argparse
-import numpy as np
-from tqdm import tqdm
-import os
-
-import torch
-import torch.nn as nn
-from torch.autograd import Variable
-import torch.backends.cudnn as cudnn
-from torch.optim.lr_scheduler import MultiStepLR
-
-from torchvision.utils import make_grid, save_image
-from torchvision import datasets, transforms
-from torchvision.transforms.transforms import ToTensor
-from torchvision.datasets import ImageFolder
-from torch.utils.data import Dataset, DataLoader
-
-from torch.utils.data.dataloader import RandomSampler
-from util.misc import CSVLogger
-from util.cutout import Cutout
-
-from model.resnet import ResNet18
-from model.wide_resnet import WideResNet
-
-from PIL import Image
-from matplotlib.pyplot import imshow
-import time
-
-def csv2list(filename):
-    lists = []
-    file = open(filename, 'r', encoding='utf-8-sig')
-    while True:
-        line = file.readline().strip("\n")
-        # int_list = [int(i) for i in line]
-        if line:
-            line = line.split(",")
-            lists.append(line)
-        else:
-            break
-    return lists
-
-# variance순으로 정렬된 logs파일에서 읽어오기    
-filelist = csv2list("C:/Users/82109/Desktop/캡디/캡디자료들/논문모델/cutout/logs/image_save/1_5000_deleted.csv")
-for i in range(len(filelist)):
-    for j in range(len(filelist[0])):
-        filelist[i][j] = float(filelist[i][j])
-transposelist = np.transpose(filelist)
-
-# print(list)
-list_tensor = torch.tensor(transposelist, dtype=torch.long)
-target = list(list_tensor[2])
-train_img_list = list()
-
-for img_idx in transposelist[1]:
-    img_path = "C:/Users/82109/Desktop/model1/img" + str(int(img_idx)) + ".png"
-    train_img_list.append(img_path)
-
-
-class Img_Dataset(Dataset):
-
-    def __init__(self,file_list,transform):
-        self.file_list = file_list
-        self.transform = transform
-    
-    def __len__(self):
-        return len(self.file_list)
-    
-    def __getitem__(self, index):
-        img_path = self.file_list[index]
-        images = np.array(Image.open(img_path))
-        # img_transformed = self.transform(images)
-
-        labels = target[index]
-
-        return images, labels
-# print(list_tensor)
-# topk 갯수 설정
-# k = 5000
-# values, indices = torch.topk(list_tensor[0], k)
-# # print(values)
-# image_list = []
-# for i in range(k):
-#     image_list.append(int(list_tensor[1][49999-i].item()))
-
-# for i in image_list:
-#     file = "C:/Users/82109/Desktop/1/1/img{0}.png".format(i)
-#     if os.path.isfile(file):
-#         os.remove(file)
-
-# transform_train = transforms.Compose([ transforms.ToTensor(), ])
-
-
-
-
-# normalize = transforms.Normalize(mean=[x / 255.0 for x in [125.3, 123.0, 113.9]],
-#                                      std=[x / 255.0 for x in [63.0, 62.1, 66.7]])
-
-
-
-
-model_options = ['resnet18', 'wideresnet']
-dataset_options = ['cifar10', 'cifar100', 'svhn']
-
-parser = argparse.ArgumentParser(description='CNN')
-parser.add_argument('--dataset', '-d', default='cifar10',
-                    choices=dataset_options)
-parser.add_argument('--model', '-a', default='resnet18',
-                    choices=model_options)
-parser.add_argument('--batch_size', type=int, default=100,
-                    help='input batch size for training (default: 128)')
-parser.add_argument('--epochs', type=int, default=200,
-                    help='number of epochs to train (default: 20)')
-parser.add_argument('--learning_rate', type=float, default=0.1,
-                    help='learning rate')
-parser.add_argument('--data_augmentation', action='store_true', default=False,
-                    help='augment data by flipping and cropping')
-parser.add_argument('--cutout', action='store_true', default=False,
-                    help='apply cutout')
-parser.add_argument('--n_holes', type=int, default=0,
-                    help='number of holes to cut out from image')
-parser.add_argument('--length', type=int, default=0,
-                    help='length of the holes')
-parser.add_argument('--no-cuda', action='store_true', default=False,
-                    help='enables CUDA training')
-parser.add_argument('--seed', type=int, default=0,
-                    help='random seed (default: 1)')
-
-args = parser.parse_args()
-args.cuda = not args.no_cuda and torch.cuda.is_available()
-cudnn.benchmark = True  # Should make training should go faster for large models
-
-torch.manual_seed(args.seed)
-if args.cuda:
-    torch.cuda.manual_seed(args.seed)
-
-test_id = args.dataset + '_' + args.model
-
-print(args)
-
-# Image Preprocessing
-if args.dataset == 'svhn':
-    normalize = transforms.Normalize(mean=[x / 255.0 for x in[109.9, 109.7, 113.8]],
-                                     std=[x / 255.0 for x in [50.1, 50.6, 50.8]])
-else:
-    normalize = transforms.Normalize(mean=[x / 255.0 for x in [125.3, 123.0, 113.9]],
-                                     std=[x / 255.0 for x in [63.0, 62.1, 66.7]])
-
-train_transform = transforms.Compose([])
-if args.data_augmentation:
-    train_transform.transforms.append(transforms.RandomCrop(32, padding=4))
-    train_transform.transforms.append(transforms.RandomHorizontalFlip())
-train_transform.transforms.append(transforms.ToTensor())
-train_transform.transforms.append(normalize)
-if args.cutout:
-    train_transform.transforms.append(Cutout(n_holes=args.n_holes, length=args.length))
-
-test_transform = transforms.Compose([
-    transforms.ToTensor(),
-    normalize])
-
-
-if args.dataset == 'cifar10':
-    num_classes = 10
-    train_dataset = Img_Dataset(file_list = train_img_list,
-                                transform=train_transform)
-    # custom_dataset = ImageFolder(root='C:/Users/82109/Desktop/1/', transform = transform_train)
-
-    # train_dataset = datasets.CIFAR10(root='data/',
-    #                                  train=True,
-    #                                  transform=train_transform,
-    #                                  download=True)
-
-    test_dataset = datasets.CIFAR10(root='data/',
-                                    train=False,
-                                    transform=test_transform,
-                                    download=True)
-# elif args.dataset == 'cifar100':
-#     num_classes = 100
-#     train_dataset = datasets.CIFAR100(root='data/',
-#                                       train=True,
-#                                       transform=train_transform,
-#                                       download=True)
-
-#     test_dataset = datasets.CIFAR100(root='data/',
-#                                      train=False,
-#                                      transform=test_transform,
-#                                      download=True)
-# elif args.dataset == 'svhn':
-#     num_classes = 10
-#     train_dataset = datasets.SVHN(root='data/',
-#                                   split='train',
-#                                   transform=train_transform,
-#                                   download=True)
-
-#     extra_dataset = datasets.SVHN(root='data/',
-#                                   split='extra',
-#                                   transform=train_transform,
-#                                   download=True)
-
-#     # Combine both training splits (https://arxiv.org/pdf/1605.07146.pdf)
-#     data = np.concatenate([train_dataset.data, extra_dataset.data], axis=0)
-#     labels = np.concatenate([train_dataset.labels, extra_dataset.labels], axis=0)
-#     train_dataset.data = data
-#     train_dataset.labels = labels
-
-#     test_dataset = datasets.SVHN(root='data/',
-#                                  split='test',
-#                                  transform=test_transform,
-#                                  download=True)
-
-# # Data Loader (Input Pipeline)
-# train_loader = torch.utils.data.DataLoader(custom_dataset, batch_size=args.batch_size, shuffle=False, pin_memory=True,num_workers=0)
-train_loader = torch.utils.data.DataLoader(dataset=train_dataset,
-                                           batch_size=args.batch_size,
-                                           shuffle=False,
-                                        #    sampler=RandomSampler(train_dataset, True, 40000),
-                                           pin_memory=True,
-                                           num_workers=0)
-
-test_loader = torch.utils.data.DataLoader(dataset=test_dataset,
-                                          batch_size=args.batch_size,
-                                          shuffle=False,
-                                          pin_memory=True,
-                                          num_workers=0)
-
-if args.model == 'resnet18':
-    cnn = ResNet18(num_classes=num_classes)
-elif args.model == 'wideresnet':
-    if args.dataset == 'svhn':
-        cnn = WideResNet(depth=16, num_classes=num_classes, widen_factor=8,
-                         dropRate=0.4)
-    else:
-        cnn = WideResNet(depth=28, num_classes=num_classes, widen_factor=10,
-                         dropRate=0.3)
-
-
-cnn = cnn.cuda()
-
-criterion = nn.CrossEntropyLoss().cuda()
-cnn_optimizer = torch.optim.SGD(cnn.parameters(), lr=args.learning_rate,
-                                momentum=0.9, nesterov=True, weight_decay=5e-4)
-
-# scheduler = MultiStepLR(cnn_optimizer, milestones=[60, 120, 160], gamma=0.2)
-if args.dataset == 'svhn':
-    scheduler = MultiStepLR(cnn_optimizer, milestones=[80, 120], gamma=0.1)
-else:
-    scheduler = MultiStepLR(cnn_optimizer, milestones=[60, 120, 160], gamma=0.2)
-
-test_id = 'custom_dataset_resnet18'
-
-filename = 'logs/' + test_id + '.csv'
-csv_logger = CSVLogger(args=args, fieldnames=['epoch', 'train_acc', 'test_acc'], filename=filename)
-
-
-def test(loader):
-    cnn.eval()    # Change model to 'eval' mode (BN uses moving mean/var).
-    correct = 0.
-    total = 0.
-    count = 0
-    for images, labels in loader:
-        images = images.cuda()
-        labels = labels.cuda()
-
-        with torch.no_grad():
-            pred = cnn(images)
-
-        pred = torch.max(pred.data, 1)[1]
-
-        total += labels.size(0)
-        # if (pred == labels).sum().item():
-        #     print('match')
-        #     count +=1
-        correct += (pred == labels).sum().item()
-    val_acc = correct / total
-    cnn.train()
-    return val_acc
-
-# kl_sum = 0
-# y_bar = torch.zeros(8, 10).detach().cuda()
-
-# y_bar 구하는 epoch
-for epoch in range(1):
-    xentropy_loss_avg = 0.
-    correct = 0.
-    total = 0.
-    norm_const = 0
-    
-    kldiv = 0
-    # pred_sum = torch.Tensor([0] * 10).detach().cuda()
-    count = 0
-    progress_bar = tqdm(train_loader)
-    for i, (images, labels) in enumerate(progress_bar):
-        progress_bar.set_description('Epoch ' + str(epoch))
-        
-        images = Variable(images.view([args.batch_size,3,32,32]).float().cuda())
-        labels = Variable(labels.float().cuda())
-        labels = torch.tensor(labels, dtype=torch.long, device=torch.device('cuda:0'))
-        cnn.zero_grad()
-        pred = cnn(images)
-        xentropy_loss = criterion(pred, labels)
-        xentropy_loss.backward()
-        cnn_optimizer.step()
-
-        xentropy_loss_avg += xentropy_loss.item()
-
-        pred_softmax = nn.functional.softmax(pred).cuda()
-        # Calculate running average of accuracy
-        pred = torch.max(pred.data, 1)[1]
-
-        total += labels.size(0)
-        correct += (pred == labels.data).sum().item()
-        accuracy = correct / total
-        # for a in range(pred_softmax.data.size()[0]):
-        #     for b in range(y_bar.size()[1]):
-        #         y_bar[epoch][b] += torch.log(pred_softmax.data[a][b])
-        
-
-        progress_bar.set_postfix(
-            xentropy='%.3f' % (xentropy_loss_avg / (i + 1)),
-            acc='%.3f' % accuracy)
-    #     count += 1
-    # xentropy = xentropy_loss_avg / count
-    # y_bar[epoch] = torch.Tensor([x / 50000 for x in y_bar[epoch]]).cuda()
-    # y_bar[epoch] = torch.exp(y_bar[epoch])
-    # for index in range(y_bar.size()[1]):
-    #     norm_const += y_bar[epoch][index]
-    # for index in range(y_bar.size()[1]):
-    #     y_bar[epoch][index] = y_bar[epoch][index] / norm_const
-    # print("y_bar[{0}] : ".format(epoch), y_bar[epoch])
-    test_acc = test(test_loader)
-    # print(pred, labels.data)
-    tqdm.write('test_acc: %.3f' % (test_acc))
-
-    scheduler.step()  # Use this line for PyTorch <1.4
-    # scheduler.step()     # Use this line for PyTorch >=1.4
-
-    row = {'epoch': str(epoch), 'train_acc': str(accuracy), 'test_acc': str(test_acc)}
-    csv_logger.writerow(row)
-    # del pred
-    # torch.cuda.empty_cache()
-
-
-# var_tensor = torch.zeros(8, 50000).detach().cuda()
-# var_addeachcol = torch.zeros(1, 50000).detach().cuda()
-
-# # kl_div 구하는 epoch
-# for epoch in range(1):
-#     checkpoint = torch.load('C:/Users/82109/Desktop/캡디/캡디자료들/논문모델/Cutout/checkpoints/sampling/sampling_{0}.pt'.format(8), map_location = torch.device('cuda:0'))
-#     cnn.load_state_dict(checkpoint)
-#     cnn.eval()
-#     kldiv = 0
-#     for i, (images, labels) in enumerate(progress_bar):
-#         progress_bar.set_description('Epoch ' + str(epoch) + ': Calculate kl_div')
-
-#         images = images.cuda()
-#         labels = labels.cuda()
-
-#         cnn.zero_grad()
-#         pred = cnn(images)
-
-#         pred_softmax = nn.functional.softmax(pred).cuda()
-
-#         # 입력 두 개의 shape이 다르면 batchsize로 평균을 내서 반환.
-#         kldiv = torch.nn.functional.kl_div(y_bar[epoch], pred_softmax, reduction='sum')
-#         # 1 * 50000에 한 모델의 데이터별 variance 저장
-#         var_tensor[epoch][i] += abs(kldiv).detach()
-#         var_addeachcol[0][i] += var_tensor[epoch][i]
-#         kl_sum += kldiv.detach()
-#         # print(y_bar_copy.size(), pred_softmax.size())
-#         # print(kl_sum)
-#     var = abs(kl_sum.item() / 50000)
-#     print("Variance : ", var)
-#     csv_logger.writerow({'var' : float(var)})
-#     # print(var_tensor)
-# for i in range(var_addeachcol.size()[1]):
-#     var_addeachcol[0][i] = var_addeachcol[0][i] / 8
-
-# print(var_addeachcol)
-# # var_addeachcol[0] = torch.Tensor([x / 8 for x in var_addeachcol]).cuda()
-# var_sorted = torch.argsort(var_addeachcol)
-# print(var_sorted)
-# for i in range(var_addeachcol.size()[1]):
-#     csv_logger.writerow({'avg_var' : float(var_addeachcol[0][i]), 'arg_var' : float(var_sorted[0][i]), 'index' : float(i + 1)})
-torch.save(cnn.state_dict(), 'checkpoints/' + test_id + '.pt')
-csv_logger.close()