2020-1-capstone-design2 / 2016104167

Go to a project
Toggle navigation Toggle navigation pinning
조현아
cb2d38d2 1 parent cad58740

get 1 random policy

Expand all
Inline Side-by-side
Showing 5 changed files with 208 additions and 272 deletions
import os
import fire
import json
from pprint import pprint
import pickle
import torch
import torch.nn as nn
from torch.utils.tensorboard import SummaryWriter
from utils import *
# command
# python getAugmented.py --model_path='logs/April_24_21:05:15__resnet50__None/'
def eval(model_path):
print('\n[+] Parse arguments')
kwargs_path = os.path.join(model_path, 'kwargs.json')
kwargs = json.loads(open(kwargs_path).read())
args, kwargs = parse_args(kwargs)
pprint(args)
device = torch.device('cuda' if args.use_cuda else 'cpu')
cp_path = os.path.join(model_path, 'augmentation.cp')
writer = SummaryWriter(log_dir=model_path)
print('\n[+] Load transform')
# list
with open(cp_path, 'rb') as f:
aug_transform_list = pickle.load(f)
augmented_image_list = [torch.Tensor(240,0)] * len(get_dataset(args, None, 'test'))
print('\n[+] Load dataset')
for aug_idx, aug_transform in enumerate(aug_transform_list):
dataset = get_dataset(args, aug_transform, 'test')
loader = iter(get_aug_dataloader(args, dataset))
for i, (images, target) in enumerate(loader):
images = images.view(240, 240)
# concat image
augmented_image_list[i] = torch.cat([augmented_image_list[i], images], dim = 1)
if i % 1000 == 0:
print("\n images size: ", augmented_image_list[i].size()) # [240, 240]
break
# break
# print(augmented_image_list)
print('\n[+] Write on tensorboard')
if writer:
for i, data in enumerate(augmented_image_list):
tag = 'img/' + str(i)
writer.add_image(tag, data.view(1, 240, -1), global_step=0)
break
writer.close()
# if writer:
# for j in range():
# tag = 'img/' + str(img_count) + '_' + str(j)
# # writer.add_image(tag,
# # concat_image_features(images[j], first[j]), global_step=step)
# # if j > 0:
# # fore = concat_image_features(fore, images[j])
# writer.add_image(tag, fore, global_step=0)
# img_count = img_count + 1
# writer.close()
if __name__ == '__main__':
fire.Fire(eval)
import os
import fire
import json
from pprint import pprint
import pickle
import torch
import torch.nn as nn
from torch.utils.tensorboard import SummaryWriter
from utils import *
# command
# python getAugmented.py --model_path='logs/April_24_21:05:15__resnet50__None/'
def eval(model_path):
print('\n[+] Parse arguments')
kwargs_path = os.path.join(model_path, 'kwargs.json')
kwargs = json.loads(open(kwargs_path).read())
args, kwargs = parse_args(kwargs)
pprint(args)
device = torch.device('cuda' if args.use_cuda else 'cpu')
cp_path = os.path.join(model_path, 'augmentation.cp')
writer = SummaryWriter(log_dir=model_path)
print('\n[+] Load transform')
# list
with open(cp_path, 'rb') as f:
aug_transform_list = pickle.load(f)
augmented_image_list = [torch.Tensor(240,0)] * len(get_dataset(args, None, 'train'))
print('\n[+] Load dataset')
for aug_idx, aug_transform in enumerate(aug_transform_list):
dataset = get_dataset(args, aug_transform, 'train')
loader = iter(get_aug_dataloader(args, dataset))
for i, (images, target) in enumerate(loader):
images = images.view(240, 240)
# concat image
augmented_image_list[i] = torch.cat([augmented_image_list[i], images], dim = 1)
print('\n[+] Write on tensorboard')
if writer:
for i, data in enumerate(augmented_image_list):
tag = 'img/' + str(i)
writer.add_image(tag, data.view(1, 240, -1), global_step=0)
writer.close()
if __name__ == '__main__':
fire.Fire(eval)
import os
import fire
import json
from pprint import pprint
import pickle
import random
import torch
import torch.nn as nn
from torchvision.utils import save_image
from torch.utils.tensorboard import SummaryWriter
from utils import *
# command
# python getAugmented_saveimg.py --model_path='logs/April_26_00:55:16__resnet50__None/'
def eval(model_path):
print('\n[+] Parse arguments')
kwargs_path = os.path.join(model_path, 'kwargs.json')
kwargs = json.loads(open(kwargs_path).read())
args, kwargs = parse_args(kwargs)
pprint(args)
device = torch.device('cuda' if args.use_cuda else 'cpu')
cp_path = os.path.join(model_path, 'augmentation.cp')
writer = SummaryWriter(log_dir=model_path)
print('\n[+] Load transform')
# list to tensor
with open(cp_path, 'rb') as f:
aug_transform_list = pickle.load(f)
transform = transforms.RandomChoice(aug_transform_list)
print('\n[+] Load dataset')
dataset = get_dataset(args, transform, 'train')
loader = iter(get_aug_dataloader(args, dataset))
print('\n[+] Save 1 random policy')
os.makedirs(os.path.join(model_path, 'augmented_imgs'))
save_dir = os.path.join(model_path, 'augmented_imgs')
for i, (image, target) in enumerate(loader):
image = image.view(240, 240)
# save img
save_image(image, os.path.join(save_dir, 'aug_'+ str(i) + '.png'))
if(i % 100 == 0):
print("\n saved images: ", i)
print('\n[+] Finished to save')
if __name__ == '__main__':
fire.Fire(eval)
This diff is collapsed. Click to expand it.
import os
import time
import importlib
import collections
import pickle as cp
import glob
import numpy as np
import pandas as pd
from natsort import natsorted
from PIL import Image
import torch
import torchvision
import torch.nn.functional as F
import torchvision.models as models
import torchvision.transforms as transforms
from torch.utils.data import Subset
from torch.utils.data import Dataset, DataLoader
from sklearn.model_selection import StratifiedShuffleSplit
from sklearn.model_selection import train_test_split
from sklearn.model_selection import KFold
from networks import *
TRAIN_DATASET_PATH = '/content/drive/My Drive/CD2 Project/data/nonaug+Normal_train/'
TRAIN_TARGET_PATH = '/content/drive/My Drive/CD2 Project/data/train_nonaug_classify_target.csv'
# VAL_DATASET_PATH = '../../data/MICCAI_BraTS_2019_Data_Training/ce_valid/'
# VAL_TARGET_PATH = '../../data/MICCAI_BraTS_2019_Data_Training/ce_valid_targets.csv'
current_epoch = 0
def split_dataset(args, dataset, k):
# load dataset
X = list(range(len(dataset)))
Y = dataset.targets
# split to k-fold
assert len(X) == len(Y)
def _it_to_list(_it):
return list(zip(*list(_it)))
sss = StratifiedShuffleSplit(n_splits=k, random_state=args.seed, test_size=0.1)
Dm_indexes, Da_indexes = _it_to_list(sss.split(X, Y))
return Dm_indexes, Da_indexes
def get_model_name(args):
from datetime import datetime, timedelta, timezone
now = datetime.now(timezone.utc)
tz = timezone(timedelta(hours=9))
now = now.astimezone(tz)
date_time = now.strftime("%B_%d_%H:%M:%S")
model_name = '__'.join([date_time, args.network, str(args.seed)])
return model_name
def dict_to_namedtuple(d):
Args = collections.namedtuple('Args', sorted(d.keys()))
for k,v in d.items():
if type(v) is dict:
d[k] = dict_to_namedtuple(v)
elif type(v) is str:
try:
d[k] = eval(v)
except:
d[k] = v
args = Args(**d)
return args
def parse_args(kwargs):
# combine with default args
kwargs['dataset'] = kwargs['dataset'] if 'dataset' in kwargs else 'BraTS'
kwargs['network'] = kwargs['network'] if 'network' in kwargs else 'resnet50'
kwargs['optimizer'] = kwargs['optimizer'] if 'optimizer' in kwargs else 'adam'
kwargs['learning_rate'] = kwargs['learning_rate'] if 'learning_rate' in kwargs else 0.0001
kwargs['seed'] = kwargs['seed'] if 'seed' in kwargs else None
kwargs['use_cuda'] = kwargs['use_cuda'] if 'use_cuda' in kwargs else True
kwargs['use_cuda'] = kwargs['use_cuda'] and torch.cuda.is_available()
kwargs['num_workers'] = kwargs['num_workers'] if 'num_workers' in kwargs else 4
kwargs['print_step'] = kwargs['print_step'] if 'print_step' in kwargs else 500
kwargs['val_step'] = kwargs['val_step'] if 'val_step' in kwargs else 500
kwargs['scheduler'] = kwargs['scheduler'] if 'scheduler' in kwargs else 'exp'
kwargs['batch_size'] = kwargs['batch_size'] if 'batch_size' in kwargs else 128
kwargs['start_step'] = kwargs['start_step'] if 'start_step' in kwargs else 0
kwargs['max_step'] = kwargs['max_step'] if 'max_step' in kwargs else 5000
kwargs['augment_path'] = kwargs['augment_path'] if 'augment_path' in kwargs else None
# to named tuple
args = dict_to_namedtuple(kwargs)
return args, kwargs
def select_model(args):
# grayResNet2
resnet_dict = {'resnet18':grayResNet2.resnet18(), 'resnet34':grayResNet2.resnet34(),
'resnet50':grayResNet2.resnet50(), 'resnet101':grayResNet2.resnet101(), 'resnet152':grayResNet2.resnet152()}
if args.network in resnet_dict:
backbone = resnet_dict[args.network]
model = basenet.BaseNet(backbone, args)
else:
Net = getattr(importlib.import_module('networks.{}'.format(args.network)), 'Net')
model = Net(args)
#print(model) # print model architecture
return model
def select_optimizer(args, model):
if args.optimizer == 'sgd':
optimizer = torch.optim.SGD(model.parameters(), lr=args.learning_rate, momentum=0.9, weight_decay=0.0001)
elif args.optimizer == 'rms':
optimizer = torch.optim.RMSprop(model.parameters(), lr=args.learning_rate)
elif args.optimizer == 'adam':
optimizer = torch.optim.Adam(model.parameters(), lr=args.learning_rate)
else:
raise Exception('Unknown Optimizer')
return optimizer
def select_scheduler(args, optimizer):
if not args.scheduler or args.scheduler == 'None':
return None
elif args.scheduler =='clr':
return torch.optim.lr_scheduler.CyclicLR(optimizer, 0.01, 0.015, mode='triangular2', step_size_up=250000, cycle_momentum=False)
elif args.scheduler =='exp':
return torch.optim.lr_scheduler.ExponentialLR(optimizer, gamma=0.9999283, last_epoch=-1)
else:
raise Exception('Unknown Scheduler')
class CustomDataset(Dataset):
def __init__(self, data_path, csv_path):
self.len = len(self.imgs)
self.path = data_path
self.imgs = natsorted(os.listdir(data_path))
df = pd.read_csv(csv_path)
targets_list = []
for fname in self.imgs:
row = df.loc[df['filename'] == fname]
targets_list.append(row.iloc[0, 1])
self.targets = targets_list
def __len__(self):
return self.len
def __getitem__(self, idx):
img_loc = os.path.join(self.path, self.imgs[idx])
targets = self.targets[idx]
image = Image.open(img_loc)
return image, targets
def get_dataset(args, transform, split='train'):
assert split in ['train', 'val', 'test', 'trainval']
if split in ['train']:
dataset = CustomDataset(TRAIN_DATASET_PATH, TRAIN_TARGET_PATH, transform=transform)
else: #test
dataset = CustomDataset(VAL_DATASET_PATH, VAL_TARGET_PATH, transform=transform)
return dataset
def get_dataloader(args, dataset, shuffle=False, pin_memory=True):
data_loader = torch.utils.data.DataLoader(dataset,
batch_size=args.batch_size,
shuffle=shuffle,
num_workers=args.num_workers,
pin_memory=pin_memory)
return data_loader
def get_aug_dataloader(args, dataset, shuffle=False, pin_memory=True):
data_loader = torch.utils.data.DataLoader(dataset,
batch_size=args.batch_size,
shuffle=shuffle,
num_workers=args.num_workers,
pin_memory=pin_memory)
return data_loader
def get_inf_dataloader(args, dataset):
global current_epoch
data_loader = iter(get_dataloader(args, dataset, shuffle=True))
while True:
try:
batch = next(data_loader)
except StopIteration:
current_epoch += 1
data_loader = iter(get_dataloader(args, dataset, shuffle=True))
batch = next(data_loader)
yield batch
def train_step(args, model, optimizer, scheduler, criterion, batch, step, writer, device=None):
model.train()
images, target = batch
if device:
images = images.to(device)
target = target.to(device)
elif args.use_cuda:
images = images.cuda(non_blocking=True)
target = target.cuda(non_blocking=True)
# compute output
start_t = time.time()
output, first = model(images)
forward_t = time.time() - start_t
loss = criterion(output, target)
# measure accuracy and record loss
acc1, acc5 = accuracy(output, target, topk=(1, 5))
acc1 /= images.size(0)
acc5 /= images.size(0)
# compute gradient and do SGD step
optimizer.zero_grad()
start_t = time.time()
loss.backward()
backward_t = time.time() - start_t
optimizer.step()
if scheduler: scheduler.step()
if writer and step % args.print_step == 0:
n_imgs = min(images.size(0), 10)
tag = 'train/' + str(step)
for j in range(n_imgs):
writer.add_image(tag,
concat_image_features(images[j], first[j]), global_step=step)
return acc1, acc5, loss, forward_t, backward_t
#_acc1, _acc5 = accuracy(output, target, topk=(1, 5))
def accuracy(output, target, topk=(1,)):
"""Computes the accuracy over the k top predictions for the specified values of k"""
with torch.no_grad():
maxk = max(topk)
batch_size = target.size(0)
_, pred = output.topk(maxk, 1, True, True)
pred = pred.t()
correct = pred.eq(target.view(1, -1).expand_as(pred))
res = []
for k in topk:
correct_k = correct[:k].view(-1).float().sum(0, keepdim=True)
res.append(correct_k)
return res