rm a few aug policies

@@ -17,14 +17,14 @@ from utils import *
 DEFALUT_CANDIDATES = [
     ShearXY,
     TranslateXY,
-    Rotate,
-    AutoContrast,
-    Invert,
+    # Rotate,
+    # AutoContrast,
+    # Invert,
     Equalize,
     Solarize,
     Posterize,
     Contrast,
-    Color,
+    # Color,
     Brightness,
     Sharpness,
     Cutout,
@@ -87,7 +87,7 @@ def validate_child(args, model, dataset, subset_indx, transform, device=None):
 
     dataset.transform = transform
     subset = Subset(dataset, subset_indx)
-    data_loader = get_dataloader(args, subset, pin_memory=False)
+    data_loader = get_dataloader(args, subset, pin_memory=False) ##
 
     return validate(args, model, criterion, data_loader, 0, None, device)
 

+{"use_cuda": true, "network": "resnet50", "dataset": "BraTS", "optimizer": "adam", "fast_auto_augment": true, "learning_rate": 0.01, "seed": null, "num_workers": 4, "print_step": 500, "val_step": 500, "scheduler": "exp", "batch_size": 128, "start_step": 0, "max_step": 10000, "augment_path": null}
\ No newline at end of file

+
+04:28
\ No newline at end of file

+{"use_cuda": true, "network": "resnet50", "dataset": "BraTS", "optimizer": "adam", "fast_auto_augment": true, "learning_rate": 0.0001, "seed": null, "num_workers": 4, "print_step": 500, "val_step": 500, "scheduler": "exp", "batch_size": 128, "start_step": 0, "max_step": 10000, "augment_path": null}
\ No newline at end of file

+
+05:45

+{"use_cuda": true, "network": "resnet50", "dataset": "BraTS", "optimizer": "adam", "fast_auto_augment": true, "print_step": 300, "batch_size": 4, "val_step": 300, "max_step": 300, "learning_rate": 0.0001, "seed": null, "num_workers": 4, "scheduler": "exp", "start_step": 0, "augment_path": null}
\ No newline at end of file

+max step = 5000
+step = 500
+lr = 0.0001
+batch_size = 128
+
+
+DEFALUT_CANDIDATES = [
+    ShearXY,
+    TranslateXY,
+    Rotate,
+    AutoContrast,
+    Invert,
+    Equalize,
+    Solarize,
+    Posterize,
+    Contrast,
+    Color,
+    Brightness,
+    Sharpness,
+    Cutout
+]
+
+
+
+
+[+] Parse arguments
+Args(augment_path=None, batch_size=64, dataset='BraTS', fast_auto_augment=True, learning_rate=0.0001, max_step=5000, network='resnet50', num_workers=4, optimizer='adam', print_step=500, scheduler='exp', seed=None, start_step=0, use_cuda=True, val_step=500)
+
+[+] Create log dir
+2020-04-06 10:55:55.010279: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.1
+
+[+] Create network
+
+[+] Load dataset
+[+] Child 0 training started (GPU: 0)
+
+[+] Training step: 0/5000	Elapsed time: 0.11min	Learning rate: 9.999283e-05	Device name: Tesla P4
+  Acc@1 : 0.000%
+  Acc@5 : 1.562%
+  Loss : 6.452464580535889
+
+[+] Training step: 500/5000	Elapsed time: 4.94min	Learning rate: 9.647145853624023e-05	Device name: Tesla P4
+  Acc@1 : 100.000%
+  Acc@5 : 100.000%
+  Loss : 0.0003164634108543396
+
+[+] Training step: 1000/5000	Elapsed time: 9.78min	Learning rate: 9.307409653381692e-05	Device name: Tesla P4
+  Acc@1 : 100.000%
+  Acc@5 : 100.000%
+  Loss : 0.00015767663717269897
+
+[+] Training step: 1500/5000	Elapsed time: 14.55min	Learning rate: 8.979637684582141e-05	Device name: Tesla P4
+  Acc@1 : 100.000%
+  Acc@5 : 100.000%
+  Loss : 7.193535566329956e-05
+
+[+] Training step: 2000/5000	Elapsed time: 19.42min	Learning rate: 8.663408611983758e-05	Device name: Tesla P4
+  Acc@1 : 100.000%
+  Acc@5 : 100.000%
+  Loss : 3.548711538314819e-05
+
+[+] Training step: 2500/5000	Elapsed time: 24.18min	Learning rate: 8.358315938187757e-05	Device name: Tesla P4
+  Acc@1 : 100.000%
+  Acc@5 : 100.000%
+  Loss : 2.549588680267334e-05
+
+[+] Training step: 3000/5000	Elapsed time: 29.06min	Learning rate: 8.063967481105164e-05	Device name: Tesla P4
+  Acc@1 : 100.000%
+  Acc@5 : 100.000%
+  Loss : 2.321600914001465e-05
+
+[+] Training step: 3500/5000	Elapsed time: 33.86min	Learning rate: 7.779984869825457e-05	Device name: Tesla P4
+  Acc@1 : 100.000%
+  Acc@5 : 100.000%
+  Loss : 2.5473142159171402e-05
+
+[+] Training step: 4000/5000	Elapsed time: 38.75min	Learning rate: 7.506003058238717e-05	Device name: Tesla P4
+  Acc@1 : 100.000%
+  Acc@5 : 100.000%
+  Loss : 8.58306884765625e-06
+
+[+] Training step: 4500/5000	Elapsed time: 43.64min	Learning rate: 7.24166985578635e-05	Device name: Tesla P4
+  Acc@1 : 100.000%
+  Acc@5 : 100.000%
+  Loss : 6.780028343200684e-06
+100% 1/1 [00:06<00:00,  6.08s/it, best loss: 5.96450536249904e-06]
+100% 1/1 [00:04<00:00,  4.25s/it, best loss: 8.08198547019856e-06]
+[+] Child 1 training started (GPU: 0)
+
+[+] Training step: 0/5000	Elapsed time: 0.06min	Learning rate: 9.999283e-05	Device name: Tesla P4
+  Acc@1 : 0.000%
+  Acc@5 : 1.562%
+  Loss : 6.547057628631592
+
+[+] Training step: 500/5000	Elapsed time: 4.90min	Learning rate: 9.647145853624023e-05	Device name: Tesla P4
+  Acc@1 : 100.000%
+  Acc@5 : 100.000%
+  Loss : 0.000509507954120636
+
+[+] Training step: 1000/5000	Elapsed time: 9.79min	Learning rate: 9.307409653381692e-05	Device name: Tesla P4
+  Acc@1 : 100.000%
+  Acc@5 : 100.000%
+  Loss : 0.0001100301742553711
+
+[+] Training step: 1500/5000	Elapsed time: 14.60min	Learning rate: 8.979637684582141e-05	Device name: Tesla P4
+  Acc@1 : 100.000%
+  Acc@5 : 100.000%
+  Loss : 7.142126560211182e-05
+
+[+] Training step: 2000/5000	Elapsed time: 19.46min	Learning rate: 8.663408611983758e-05	Device name: Tesla P4
+  Acc@1 : 100.000%
+  Acc@5 : 100.000%
+  Loss : 5.066394805908203e-05
+
+[+] Training step: 2500/5000	Elapsed time: 24.26min	Learning rate: 8.358315938187757e-05	Device name: Tesla P4
+  Acc@1 : 100.000%
+  Acc@5 : 100.000%
+  Loss : 2.6144087314605713e-05
+
+[+] Training step: 3000/5000	Elapsed time: 29.14min	Learning rate: 8.063967481105164e-05	Device name: Tesla P4
+  Acc@1 : 100.000%
+  Acc@5 : 100.000%
+  Loss : 1.6264617443084717e-05
+
+[+] Training step: 3500/5000	Elapsed time: 33.96min	Learning rate: 7.779984869825457e-05	Device name: Tesla P4
+  Acc@1 : 100.000%
+  Acc@5 : 100.000%
+  Loss : 2.374147152295336e-05
+
+[+] Training step: 4000/5000	Elapsed time: 38.83min	Learning rate: 7.506003058238717e-05	Device name: Tesla P4
+  Acc@1 : 100.000%
+  Acc@5 : 100.000%
+  Loss : 1.138448715209961e-05
+
+[+] Training step: 4500/5000	Elapsed time: 43.67min	Learning rate: 7.24166985578635e-05	Device name: Tesla P4
+  Acc@1 : 100.000%
+  Acc@5 : 100.000%
+  Loss : 6.288290023803711e-06
+100% 1/1 [00:04<00:00,  4.13s/it, best loss: 4.1864686863846146e-06]
+100% 1/1 [00:04<00:00,  4.07s/it, best loss: 8.486084880132694e-06]
+[+] Child 2 training started (GPU: 0)
+
+[+] Training step: 0/5000	Elapsed time: 0.07min	Learning rate: 9.999283e-05	Device name: Tesla P4
+  Acc@1 : 1.562%
+  Acc@5 : 1.562%
+  Loss : 6.490279674530029
+
+[+] Training step: 500/5000	Elapsed time: 4.82min	Learning rate: 9.647145853624023e-05	Device name: Tesla P4
+  Acc@1 : 100.000%
+  Acc@5 : 100.000%
+  Loss : 0.0005068182945251465
+
+[+] Training step: 1000/5000	Elapsed time: 9.63min	Learning rate: 9.307409653381692e-05	Device name: Tesla P4
+  Acc@1 : 100.000%
+  Acc@5 : 100.000%
+  Loss : 0.00012455880641937256
+
+[+] Training step: 1500/5000	Elapsed time: 14.36min	Learning rate: 8.979637684582141e-05	Device name: Tesla P4
+  Acc@1 : 100.000%
+  Acc@5 : 100.000%
+  Loss : 6.882846355438232e-05
+
+[+] Training step: 2000/5000	Elapsed time: 19.16min	Learning rate: 8.663408611983758e-05	Device name: Tesla P4
+  Acc@1 : 100.000%
+  Acc@5 : 100.000%
+  Loss : 2.9802322387695312e-05
+
+[+] Training step: 2500/5000	Elapsed time: 23.87min	Learning rate: 8.358315938187757e-05	Device name: Tesla P4
+  Acc@1 : 100.000%
+  Acc@5 : 100.000%
+  Loss : 1.917034387588501e-05
+
+[+] Training step: 3000/5000	Elapsed time: 28.66min	Learning rate: 8.063967481105164e-05	Device name: Tesla P4
+  Acc@1 : 100.000%
+  Acc@5 : 100.000%
+  Loss : 1.6361474990844727e-05
+
+[+] Training step: 3500/5000	Elapsed time: 33.37min	Learning rate: 7.779984869825457e-05	Device name: Tesla P4
+  Acc@1 : 100.000%
+  Acc@5 : 100.000%
+  Loss : 1.0741384357970674e-05
+
+[+] Training step: 4000/5000	Elapsed time: 38.21min	Learning rate: 7.506003058238717e-05	Device name: Tesla P4
+  Acc@1 : 100.000%
+  Acc@5 : 100.000%
+  Loss : 7.048249244689941e-06
+
+[+] Training step: 4500/5000	Elapsed time: 43.05min	Learning rate: 7.24166985578635e-05	Device name: Tesla P4
+  Acc@1 : 100.000%
+  Acc@5 : 100.000%
+  Loss : 7.450580596923828e-06
+100% 1/1 [00:04<00:00,  4.30s/it, best loss: 3.3297781101282453e-06]
+100% 1/1 [00:04<00:00,  4.30s/it, best loss: 7.823362466297112e-06]
+[+] Child 3 training started (GPU: 0)
+
+[+] Training step: 0/5000	Elapsed time: 0.06min	Learning rate: 9.999283e-05	Device name: Tesla P4
+  Acc@1 : 0.000%
+  Acc@5 : 0.000%
+  Loss : 6.425505638122559
+
+[+] Training step: 500/5000	Elapsed time: 4.89min	Learning rate: 9.647145853624023e-05	Device name: Tesla P4
+  Acc@1 : 100.000%
+  Acc@5 : 100.000%
+  Loss : 0.0004970133304595947
+
+[+] Training step: 1000/5000	Elapsed time: 9.81min	Learning rate: 9.307409653381692e-05	Device name: Tesla P4
+  Acc@1 : 100.000%
+  Acc@5 : 100.000%
+  Loss : 0.00017832964658737183
+
+[+] Training step: 1500/5000	Elapsed time: 14.64min	Learning rate: 8.979637684582141e-05	Device name: Tesla P4
+  Acc@1 : 100.000%
+  Acc@5 : 100.000%
+  Loss : 6.403028964996338e-05
+
+[+] Training step: 2000/5000	Elapsed time: 19.53min	Learning rate: 8.663408611983758e-05	Device name: Tesla P4
+  Acc@1 : 100.000%
+  Acc@5 : 100.000%
+  Loss : 7.484853267669678e-05
+
+[+] Training step: 2500/5000	Elapsed time: 24.35min	Learning rate: 8.358315938187757e-05	Device name: Tesla P4
+  Acc@1 : 100.000%
+  Acc@5 : 100.000%
+  Loss : 3.053247928619385e-05
+
+[+] Training step: 3000/5000	Elapsed time: 29.26min	Learning rate: 8.063967481105164e-05	Device name: Tesla P4
+  Acc@1 : 100.000%
+  Acc@5 : 100.000%
+  Loss : 1.741945743560791e-05
+
+[+] Training step: 3500/5000	Elapsed time: 34.08min	Learning rate: 7.779984869825457e-05	Device name: Tesla P4
+  Acc@1 : 100.000%
+  Acc@5 : 100.000%
+  Loss : 2.042870801233221e-05
+
+[+] Training step: 4000/5000	Elapsed time: 38.94min	Learning rate: 7.506003058238717e-05	Device name: Tesla P4
+  Acc@1 : 100.000%
+  Acc@5 : 100.000%
+  Loss : 7.957220077514648e-06
+
+[+] Training step: 4500/5000	Elapsed time: 43.80min	Learning rate: 7.24166985578635e-05	Device name: Tesla P4
+  Acc@1 : 100.000%
+  Acc@5 : 100.000%
+  Loss : 6.332993507385254e-06
+100% 1/1 [00:04<00:00,  4.14s/it, best loss: 6.594900241907453e-06]
+100% 1/1 [00:04<00:00,  4.14s/it, best loss: 2.6993832307198318e-06]
+RandomChoice(
+    Compose(
+    Pad(padding=4, fill=0, padding_mode=constant)
+    RandomCrop(size=(32, 32), padding=None)
+    RandomHorizontalFlip(p=0.5)
+    ShearXY(prob=0.67, magnitude=0.71)
+    Posterize(prob=0.38, magnitude=0.90)
+    ToTensor()
+)
+    Compose(
+    Pad(padding=4, fill=0, padding_mode=constant)
+    RandomCrop(size=(32, 32), padding=None)
+    RandomHorizontalFlip(p=0.5)
+    Color(prob=0.90, magnitude=0.30)
+    Rotate(prob=0.41, magnitude=0.51)
+    ToTensor()
+)
+    Compose(
+    Pad(padding=4, fill=0, padding_mode=constant)
+    RandomCrop(size=(32, 32), padding=None)
+    RandomHorizontalFlip(p=0.5)
+    Sharpness(prob=0.84, magnitude=0.21)
+    Contrast(prob=0.88, magnitude=0.16)
+    ToTensor()
+)
+    Compose(
+    Pad(padding=4, fill=0, padding_mode=constant)
+    RandomCrop(size=(32, 32), padding=None)
+    RandomHorizontalFlip(p=0.5)
+    ShearXY(prob=0.24, magnitude=0.39)
+    TranslateXY(prob=0.47, magnitude=0.55)
+    ToTensor()
+)
+    Compose(
+    Pad(padding=4, fill=0, padding_mode=constant)
+    RandomCrop(size=(32, 32), padding=None)
+    RandomHorizontalFlip(p=0.5)
+    Sharpness(prob=0.38, magnitude=0.48)
+    Equalize(prob=0.72, magnitude=0.25)
+    ToTensor()
+)
+    Compose(
+    Pad(padding=4, fill=0, padding_mode=constant)
+    RandomCrop(size=(32, 32), padding=None)
+    RandomHorizontalFlip(p=0.5)
+    Rotate(prob=0.84, magnitude=0.36)
+    AutoContrast(prob=0.12, magnitude=0.05)
+    ToTensor()
+)
+    Compose(
+    Pad(padding=4, fill=0, padding_mode=constant)
+    RandomCrop(size=(32, 32), padding=None)
+    RandomHorizontalFlip(p=0.5)
+    Contrast(prob=0.26, magnitude=0.12)
+    ShearXY(prob=0.32, magnitude=0.13)
+    ToTensor()
+)
+    Compose(
+    Pad(padding=4, fill=0, padding_mode=constant)
+    RandomCrop(size=(32, 32), padding=None)
+    RandomHorizontalFlip(p=0.5)
+    Solarize(prob=0.05, magnitude=0.03)
+    Sharpness(prob=0.97, magnitude=0.83)
+    ToTensor()
+)
+)
+
+[+] Start training
+
+[+] Use 1 GPUs
+
+[+] Using GPU: Tesla P4 
+/usr/local/lib/python3.6/dist-packages/torch/nn/functional.py:2457: UserWarning: nn.functional.upsample is deprecated. Use nn.functional.interpolate instead.
+  warnings.warn("nn.functional.upsample is deprecated. Use nn.functional.interpolate instead.")
+/usr/local/lib/python3.6/dist-packages/torch/nn/functional.py:2539: UserWarning: Default upsampling behavior when mode=bilinear is changed to align_corners=False since 0.4.0. Please specify align_corners=True if the old behavior is desired. See the documentation of nn.Upsample for details.
+  "See the documentation of nn.Upsample for details.".format(mode))
+
+[+] Training step: 0/5000	Training epoch: 0/31	Elapsed time: 0.62min	Learning rate: 9.999283e-05
+  Acc@1 : 0.000%
+  Acc@5 : 0.000%
+  Loss : 6.816585063934326
+  FW Time : 33.351ms
+  BW Time : 10.075ms
+
+[+] Valid results
+  Acc@1 : 100.000%
+  Acc@5 : 100.000%
+  Loss : 5.762917498941533e-05
+
+[+] Model saved
+
+[+] Training step: 500/5000	Training epoch: 0/31	Elapsed time: 6.52min	Learning rate: 9.647145853624023e-05
+  Acc@1 : 100.000%
+  Acc@5 : 100.000%
+  Loss : 0.00011359900236129761
+  FW Time : 19.469ms
+  BW Time : 14.039ms
+
+[+] Valid results
+  Acc@1 : 100.000%
+  Acc@5 : 100.000%
+  Loss : 0.0
+
+[+] Model saved
+
+[+] Training step: 1000/5000	Training epoch: 0/31	Elapsed time: 8.88min	Learning rate: 9.307409653381692e-05
+  Acc@1 : 100.000%
+  Acc@5 : 100.000%
+  Loss : 8.489564061164856e-05
+  FW Time : 23.165ms
+  BW Time : 13.944ms
+
+[+] Valid results
+  Acc@1 : 100.000%
+  Acc@5 : 100.000%
+  Loss : 0.0
+
+[+] Model saved
+
+[+] Training step: 1500/5000	Training epoch: 0/31	Elapsed time: 11.14min	Learning rate: 8.979637684582141e-05
+  Acc@1 : 100.000%
+  Acc@5 : 100.000%
+  Loss : 6.42240047454834e-06
+  FW Time : 14.039ms
+  BW Time : 8.235ms
+
+[+] Valid results
+  Acc@1 : 100.000%
+  Acc@5 : 100.000%
+  Loss : 0.0
+
+[+] Model saved
+
+[+] Training step: 2000/5000	Training epoch: 0/31	Elapsed time: 13.49min	Learning rate: 8.663408611983758e-05
+  Acc@1 : 100.000%
+  Acc@5 : 100.000%
+  Loss : 2.175569534301758e-06
+  FW Time : 20.693ms
+  BW Time : 14.328ms
+
+[+] Valid results
+  Acc@1 : 100.000%
+  Acc@5 : 100.000%
+  Loss : 0.0
+
+[+] Model saved
+
+[+] Training step: 2500/5000	Training epoch: 0/31	Elapsed time: 15.84min	Learning rate: 8.358315938187757e-05
+  Acc@1 : 100.000%
+  Acc@5 : 100.000%
+  Loss : 0.0
+  FW Time : 21.502ms
+  BW Time : 11.221ms
+
+[+] Valid results
+  Acc@1 : 100.000%
+  Acc@5 : 100.000%
+  Loss : 0.0
+
+[+] Model saved
+
+[+] Training step: 3000/5000	Training epoch: 0/31	Elapsed time: 18.10min	Learning rate: 8.063967481105164e-05
+  Acc@1 : 100.000%
+  Acc@5 : 100.000%
+  Loss : 0.0
+  FW Time : 24.350ms
+  BW Time : 13.500ms
+
+[+] Valid results
+  Acc@1 : 100.000%
+  Acc@5 : 100.000%
+  Loss : 0.0
+
+[+] Model saved
+
+[+] Training step: 3500/5000	Training epoch: 0/31	Elapsed time: 20.44min	Learning rate: 7.779984869825457e-05
+  Acc@1 : 100.000%
+  Acc@5 : 100.000%
+  Loss : 0.0
+  FW Time : 24.146ms
+  BW Time : 12.905ms
+
+[+] Valid results
+  Acc@1 : 100.000%
+  Acc@5 : 100.000%
+  Loss : 0.0
+
+[+] Model saved
+
+[+] Training step: 4000/5000	Training epoch: 0/31	Elapsed time: 22.77min	Learning rate: 7.506003058238717e-05
+  Acc@1 : 100.000%
+  Acc@5 : 100.000%
+  Loss : 0.0
+  FW Time : 64.314ms
+  BW Time : 17.536ms
+
+[+] Valid results
+  Acc@1 : 100.000%
+  Acc@5 : 100.000%
+  Loss : 0.0
+
+[+] Model saved
+
+[+] Training step: 4500/5000	Training epoch: 0/31	Elapsed time: 25.01min	Learning rate: 7.24166985578635e-05
+  Acc@1 : 100.000%
+  Acc@5 : 100.000%
+  Loss : 0.0
+  FW Time : 17.300ms
+  BW Time : 12.326ms
+
+[+] Valid results
+  Acc@1 : 100.000%
+  Acc@5 : 100.000%
+  Loss : 0.0
+
+[+] Model saved
\ No newline at end of file

+max step = 5000
+step = 500
+lr = 0.0001
+batch_size = 128
+
+DEFALUT_CANDIDATES = [
+    ShearXY,
+    TranslateXY,
+    # Rotate,
+    AutoContrast,
+    # Invert,
+    Equalize,
+    Solarize,
+    Posterize,
+    Contrast,
+    # Color,
+    Brightness,
+    Sharpness,
+    Cutout,
+#     SamplePairing,
+]
+
+[+] Parse arguments
+Args(augment_path=None, batch_size=64, dataset='BraTS', fast_auto_augment=True, learning_rate=0.0001, max_step=5000, network='resnet50', num_workers=4, optimizer='adam', print_step=500, scheduler='exp', seed=None, start_step=0, use_cuda=True, val_step=500)
+
+[+] Create log dir
+2020-04-13 06:56:00.492033: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.1
+
+[+] Create network
+
+[+] Load dataset
+[+] Child 0 training started (GPU: 0)
+
+[+] Training step: 0/5000	Elapsed time: 0.55min	Learning rate: 9.999283e-05	Device name: Tesla P100-PCIE-16GB
+  Acc@1 : 0.000%
+  Acc@5 : 0.000%
+  Loss : 7.331308841705322
+
+[+] Training step: 500/5000	Elapsed time: 7.61min	Learning rate: 9.647145853624023e-05	Device name: Tesla P100-PCIE-16GB
+  Acc@1 : 100.000%
+  Acc@5 : 100.000%
+  Loss : 0.0003173351287841797
+
+[+] Training step: 1000/5000	Elapsed time: 13.69min	Learning rate: 9.307409653381692e-05	Device name: Tesla P100-PCIE-16GB
+  Acc@1 : 100.000%
+  Acc@5 : 100.000%
+  Loss : 0.00012065470218658447
+
+[+] Training step: 1500/5000	Elapsed time: 19.60min	Learning rate: 8.979637684582141e-05	Device name: Tesla P100-PCIE-16GB
+  Acc@1 : 100.000%
+  Acc@5 : 100.000%
+  Loss : 8.422136306762695e-05
+
+[+] Training step: 2000/5000	Elapsed time: 25.56min	Learning rate: 8.663408611983758e-05	Device name: Tesla P100-PCIE-16GB
+  Acc@1 : 100.000%
+  Acc@5 : 100.000%
+  Loss : 5.1371753215789795e-05
+
+[+] Training step: 2500/5000	Elapsed time: 31.51min	Learning rate: 8.358315938187757e-05	Device name: Tesla P100-PCIE-16GB
+  Acc@1 : 100.000%
+  Acc@5 : 100.000%
+  Loss : 2.7231872081756592e-05
+
+[+] Training step: 3000/5000	Elapsed time: 37.30min	Learning rate: 8.063967481105164e-05	Device name: Tesla P100-PCIE-16GB
+  Acc@1 : 100.000%
+  Acc@5 : 100.000%
+  Loss : 1.662224531173706e-05
+
+[+] Training step: 3500/5000	Elapsed time: 42.91min	Learning rate: 7.779984869825457e-05	Device name: Tesla P100-PCIE-16GB
+  Acc@1 : 100.000%
+  Acc@5 : 100.000%
+  Loss : 1.1795445061579812e-05
+
+[+] Training step: 4000/5000	Elapsed time: 48.72min	Learning rate: 7.506003058238717e-05	Device name: Tesla P100-PCIE-16GB
+  Acc@1 : 100.000%
+  Acc@5 : 100.000%
+  Loss : 6.079673767089844e-06
+
+[+] Training step: 4500/5000	Elapsed time: 54.77min	Learning rate: 7.24166985578635e-05	Device name: Tesla P100-PCIE-16GB
+  Acc@1 : 100.000%
+  Acc@5 : 100.000%
+  Loss : 1.3053417205810547e-05
+100% 1/1 [00:31<00:00, 31.34s/it, best loss: 2.456923766658292e-06]
+100% 1/1 [00:05<00:00,  5.32s/it, best loss: 6.659556220256491e-06]
+[+] Child 1 training started (GPU: 0)
+
+[+] Training step: 0/5000	Elapsed time: 0.08min	Learning rate: 9.999283e-05	Device name: Tesla P100-PCIE-16GB
+  Acc@1 : 0.000%
+  Acc@5 : 0.000%
+  Loss : 7.408554553985596
+
+[+] Training step: 500/5000	Elapsed time: 6.04min	Learning rate: 9.647145853624023e-05	Device name: Tesla P100-PCIE-16GB
+  Acc@1 : 100.000%
+  Acc@5 : 100.000%
+  Loss : 0.0002929195761680603
+
+[+] Training step: 1000/5000	Elapsed time: 12.01min	Learning rate: 9.307409653381692e-05	Device name: Tesla P100-PCIE-16GB
+  Acc@1 : 100.000%
+  Acc@5 : 100.000%
+  Loss : 0.00017809122800827026
+
+[+] Training step: 1500/5000	Elapsed time: 17.93min	Learning rate: 8.979637684582141e-05	Device name: Tesla P100-PCIE-16GB
+  Acc@1 : 100.000%
+  Acc@5 : 100.000%
+  Loss : 9.942799806594849e-05
+
+[+] Training step: 2000/5000	Elapsed time: 23.73min	Learning rate: 8.663408611983758e-05	Device name: Tesla P100-PCIE-16GB
+  Acc@1 : 100.000%
+  Acc@5 : 100.000%
+  Loss : 4.045665264129639e-05
+
+[+] Training step: 2500/5000	Elapsed time: 29.59min	Learning rate: 8.358315938187757e-05	Device name: Tesla P100-PCIE-16GB
+  Acc@1 : 100.000%
+  Acc@5 : 100.000%
+  Loss : 2.6069581508636475e-05
+
+[+] Training step: 3000/5000	Elapsed time: 35.49min	Learning rate: 8.063967481105164e-05	Device name: Tesla P100-PCIE-16GB
+  Acc@1 : 100.000%
+  Acc@5 : 100.000%
+  Loss : 2.197176218032837e-05
+
+[+] Training step: 3500/5000	Elapsed time: 41.35min	Learning rate: 7.779984869825457e-05	Device name: Tesla P100-PCIE-16GB
+  Acc@1 : 100.000%
+  Acc@5 : 100.000%
+  Loss : 7.2278476181963924e-06
+
+[+] Training step: 4000/5000	Elapsed time: 47.20min	Learning rate: 7.506003058238717e-05	Device name: Tesla P100-PCIE-16GB
+  Acc@1 : 100.000%
+  Acc@5 : 100.000%
+  Loss : 8.530914783477783e-06
+
+[+] Training step: 4500/5000	Elapsed time: 52.98min	Learning rate: 7.24166985578635e-05	Device name: Tesla P100-PCIE-16GB
+  Acc@1 : 100.000%
+  Acc@5 : 100.000%
+  Loss : 5.826354026794434e-06
+100% 1/1 [00:05<00:00,  5.11s/it, best loss: 8.13047790870769e-06]
+100% 1/1 [00:05<00:00,  5.03s/it, best loss: 6.885851689730771e-06]
+[+] Child 2 training started (GPU: 0)
+
+[+] Training step: 0/5000	Elapsed time: 0.08min	Learning rate: 9.999283e-05	Device name: Tesla P100-PCIE-16GB
+  Acc@1 : 0.000%
+  Acc@5 : 0.000%
+  Loss : 7.380028247833252
+
+[+] Training step: 500/5000	Elapsed time: 5.62min	Learning rate: 9.647145853624023e-05	Device name: Tesla P100-PCIE-16GB
+  Acc@1 : 100.000%
+  Acc@5 : 100.000%
+  Loss : 0.0003273114562034607
+
+[+] Training step: 1000/5000	Elapsed time: 11.24min	Learning rate: 9.307409653381692e-05	Device name: Tesla P100-PCIE-16GB
+  Acc@1 : 100.000%
+  Acc@5 : 100.000%
+  Loss : 0.00010266900062561035
+
+[+] Training step: 1500/5000	Elapsed time: 16.86min	Learning rate: 8.979637684582141e-05	Device name: Tesla P100-PCIE-16GB
+  Acc@1 : 100.000%
+  Acc@5 : 100.000%
+  Loss : 6.0267746448516846e-05
+
+[+] Training step: 2000/5000	Elapsed time: 22.77min	Learning rate: 8.663408611983758e-05	Device name: Tesla P100-PCIE-16GB
+  Acc@1 : 100.000%
+  Acc@5 : 100.000%
+  Loss : 3.4943222999572754e-05
+
+[+] Training step: 2500/5000	Elapsed time: 28.61min	Learning rate: 8.358315938187757e-05	Device name: Tesla P100-PCIE-16GB
+  Acc@1 : 100.000%
+  Acc@5 : 100.000%
+  Loss : 3.550201654434204e-05
+
+[+] Training step: 3000/5000	Elapsed time: 34.66min	Learning rate: 8.063967481105164e-05	Device name: Tesla P100-PCIE-16GB
+  Acc@1 : 100.000%
+  Acc@5 : 100.000%
+  Loss : 2.2232532501220703e-05
+
+[+] Training step: 3500/5000	Elapsed time: 40.65min	Learning rate: 7.779984869825457e-05	Device name: Tesla P100-PCIE-16GB
+  Acc@1 : 100.000%
+  Acc@5 : 100.000%
+  Loss : 2.436888826196082e-05
+
+[+] Training step: 4000/5000	Elapsed time: 46.78min	Learning rate: 7.506003058238717e-05	Device name: Tesla P100-PCIE-16GB
+  Acc@1 : 100.000%
+  Acc@5 : 100.000%
+  Loss : 8.17328691482544e-06
+
+[+] Training step: 4500/5000	Elapsed time: 52.87min	Learning rate: 7.24166985578635e-05	Device name: Tesla P100-PCIE-16GB
+  Acc@1 : 100.000%
+  Acc@5 : 100.000%
+  Loss : 8.38935375213623e-06
+100% 1/1 [00:05<00:00,  5.43s/it, best loss: 9.423595656699035e-06]
+100% 1/1 [00:05<00:00,  5.42s/it, best loss: 8.001165952009615e-06]
+[+] Child 3 training started (GPU: 0)
+
+[+] Training step: 0/5000	Elapsed time: 0.09min	Learning rate: 9.999283e-05	Device name: Tesla P100-PCIE-16GB
+  Acc@1 : 0.000%
+  Acc@5 : 0.000%
+  Loss : 7.353156566619873
+
+[+] Training step: 500/5000	Elapsed time: 6.07min	Learning rate: 9.647145853624023e-05	Device name: Tesla P100-PCIE-16GB
+  Acc@1 : 100.000%
+  Acc@5 : 100.000%
+  Loss : 0.0005485564470291138
+
+[+] Training step: 1000/5000	Elapsed time: 12.12min	Learning rate: 9.307409653381692e-05	Device name: Tesla P100-PCIE-16GB
+  Acc@1 : 100.000%
+  Acc@5 : 100.000%
+  Loss : 0.00013493746519088745
+
+[+] Training step: 1500/5000	Elapsed time: 18.06min	Learning rate: 8.979637684582141e-05	Device name: Tesla P100-PCIE-16GB
+  Acc@1 : 100.000%
+  Acc@5 : 100.000%
+  Loss : 5.833059549331665e-05
+
+[+] Training step: 2000/5000	Elapsed time: 24.09min	Learning rate: 8.663408611983758e-05	Device name: Tesla P100-PCIE-16GB
+  Acc@1 : 100.000%
+  Acc@5 : 100.000%
+  Loss : 4.260987043380737e-05
+
+[+] Training step: 2500/5000	Elapsed time: 30.12min	Learning rate: 8.358315938187757e-05	Device name: Tesla P100-PCIE-16GB
+  Acc@1 : 100.000%
+  Acc@5 : 100.000%
+  Loss : 2.2239983081817627e-05
+
+[+] Training step: 3000/5000	Elapsed time: 36.35min	Learning rate: 8.063967481105164e-05	Device name: Tesla P100-PCIE-16GB
+  Acc@1 : 100.000%
+  Acc@5 : 100.000%
+  Loss : 1.6763806343078613e-05
+
+[+] Training step: 3500/5000	Elapsed time: 42.52min	Learning rate: 7.779984869825457e-05	Device name: Tesla P100-PCIE-16GB
+  Acc@1 : 100.000%
+  Acc@5 : 100.000%
+  Loss : 1.4556081623595674e-05
+
+[+] Training step: 4000/5000	Elapsed time: 48.95min	Learning rate: 7.506003058238717e-05	Device name: Tesla P100-PCIE-16GB
+  Acc@1 : 100.000%
+  Acc@5 : 100.000%
+  Loss : 1.1704862117767334e-05
+
+[+] Training step: 4500/5000	Elapsed time: 55.31min	Learning rate: 7.24166985578635e-05	Device name: Tesla P100-PCIE-16GB
+  Acc@1 : 100.000%
+  Acc@5 : 100.000%
+  Loss : 1.0773539543151855e-05
+100% 1/1 [00:05<00:00,  5.62s/it, best loss: 6.91817967890529e-06]
+100% 1/1 [00:05<00:00,  5.63s/it, best loss: 6.578736247320194e-06]
+RandomChoice(
+    Compose(
+    Pad(padding=4, fill=0, padding_mode=constant)
+    RandomCrop(size=(32, 32), padding=None)
+    RandomHorizontalFlip(p=0.5)
+    Solarize(prob=0.91, magnitude=0.02)
+    Equalize(prob=0.79, magnitude=0.88)
+    ToTensor()
+)
+    Compose(
+    Pad(padding=4, fill=0, padding_mode=constant)
+    RandomCrop(size=(32, 32), padding=None)
+    RandomHorizontalFlip(p=0.5)
+    TranslateXY(prob=0.63, magnitude=0.78)
+    Cutout(prob=0.31, magnitude=0.73)
+    ToTensor()
+)
+    Compose(
+    Pad(padding=4, fill=0, padding_mode=constant)
+    RandomCrop(size=(32, 32), padding=None)
+    RandomHorizontalFlip(p=0.5)
+    TranslateXY(prob=0.26, magnitude=0.37)
+    AutoContrast(prob=0.78, magnitude=0.81)
+    ToTensor()
+)
+    Compose(
+    Pad(padding=4, fill=0, padding_mode=constant)
+    RandomCrop(size=(32, 32), padding=None)
+    RandomHorizontalFlip(p=0.5)
+    Posterize(prob=0.71, magnitude=0.48)
+    AutoContrast(prob=0.25, magnitude=0.62)
+    ToTensor()
+)
+    Compose(
+    Pad(padding=4, fill=0, padding_mode=constant)
+    RandomCrop(size=(32, 32), padding=None)
+    RandomHorizontalFlip(p=0.5)
+    Cutout(prob=0.78, magnitude=0.30)
+    TranslateXY(prob=0.72, magnitude=0.50)
+    ToTensor()
+)
+    Compose(
+    Pad(padding=4, fill=0, padding_mode=constant)
+    RandomCrop(size=(32, 32), padding=None)
+    RandomHorizontalFlip(p=0.5)
+    AutoContrast(prob=0.53, magnitude=0.95)
+    Contrast(prob=0.08, magnitude=0.43)
+    ToTensor()
+)
+    Compose(
+    Pad(padding=4, fill=0, padding_mode=constant)
+    RandomCrop(size=(32, 32), padding=None)
+    RandomHorizontalFlip(p=0.5)
+    Cutout(prob=0.38, magnitude=0.47)
+    Cutout(prob=0.98, magnitude=0.07)
+    ToTensor()
+)
+    Compose(
+    Pad(padding=4, fill=0, padding_mode=constant)
+    RandomCrop(size=(32, 32), padding=None)
+    RandomHorizontalFlip(p=0.5)
+    Brightness(prob=0.13, magnitude=0.42)
+    Brightness(prob=0.88, magnitude=0.09)
+    ToTensor()
+)
+)
+
+[+] Start training
+
+[+] Use 1 GPUs
+
+[+] Using GPU: Tesla P100-PCIE-16GB 
+/usr/local/lib/python3.6/dist-packages/torch/nn/functional.py:2457: UserWarning: nn.functional.upsample is deprecated. Use nn.functional.interpolate instead.
+  warnings.warn("nn.functional.upsample is deprecated. Use nn.functional.interpolate instead.")
+/usr/local/lib/python3.6/dist-packages/torch/nn/functional.py:2539: UserWarning: Default upsampling behavior when mode=bilinear is changed to align_corners=False since 0.4.0. Please specify align_corners=True if the old behavior is desired. See the documentation of nn.Upsample for details.
+  "See the documentation of nn.Upsample for details.".format(mode))
+
+[+] Training step: 0/5000	Training epoch: 0/31	Elapsed time: 0.54min	Learning rate: 9.999283e-05
+  Acc@1 : 0.000%
+  Acc@5 : 0.000%
+  Loss : 7.482730388641357
+  FW Time : 27.781ms
+  BW Time : 11.731ms
+
+[+] Valid results
+  Acc@1 : 100.000%
+  Acc@5 : 100.000%
+  Loss : 0.0
+
+[+] Model saved
+
+[+] Training step: 500/5000	Training epoch: 0/31	Elapsed time: 6.91min	Learning rate: 9.647145853624023e-05
+  Acc@1 : 100.000%
+  Acc@5 : 100.000%
+  Loss : 7.505388930439949e-05
+  FW Time : 30.267ms
+  BW Time : 30.651ms
+
+[+] Valid results
+  Acc@1 : 100.000%
+  Acc@5 : 100.000%
+  Loss : 0.0
+
+[+] Model saved
+
+[+] Training step: 1000/5000	Training epoch: 0/31	Elapsed time: 10.06min	Learning rate: 9.307409653381692e-05
+  Acc@1 : 100.000%
+  Acc@5 : 100.000%
+  Loss : 0.00013676658272743225
+  FW Time : 40.501ms
+  BW Time : 34.972ms
+
+[+] Valid results
+  Acc@1 : 100.000%
+  Acc@5 : 100.000%
+  Loss : 0.0
+
+[+] Model saved
+
+[+] Training step: 1500/5000	Training epoch: 0/31	Elapsed time: 13.08min	Learning rate: 8.979637684582141e-05
+  Acc@1 : 100.000%
+  Acc@5 : 100.000%
+  Loss : 4.213303327560425e-06
+  FW Time : 35.585ms
+  BW Time : 30.589ms
+
+[+] Valid results
+  Acc@1 : 100.000%
+  Acc@5 : 100.000%
+  Loss : 0.0
+
+[+] Model saved
+
+[+] Training step: 2000/5000	Training epoch: 0/31	Elapsed time: 16.21min	Learning rate: 8.663408611983758e-05
+  Acc@1 : 100.000%
+  Acc@5 : 100.000%
+  Loss : 5.885958671569824e-07
+  FW Time : 29.106ms
+  BW Time : 28.138ms
+
+[+] Valid results
+  Acc@1 : 100.000%
+  Acc@5 : 100.000%
+  Loss : 0.0
+
+[+] Model saved
+
+[+] Training step: 2500/5000	Training epoch: 0/31	Elapsed time: 19.30min	Learning rate: 8.358315938187757e-05
+  Acc@1 : 100.000%
+  Acc@5 : 100.000%
+  Loss : 2.130866050720215e-06
+  FW Time : 30.433ms
+  BW Time : 32.247ms
+
+[+] Valid results
+  Acc@1 : 100.000%
+  Acc@5 : 100.000%
+  Loss : 0.0
+
+[+] Model saved
+
+[+] Training step: 3000/5000	Training epoch: 0/31	Elapsed time: 22.28min	Learning rate: 8.063967481105164e-05
+  Acc@1 : 100.000%
+  Acc@5 : 100.000%
+  Loss : 0.0
+  FW Time : 16.860ms
+  BW Time : 13.823ms
+
+[+] Valid results
+  Acc@1 : 100.000%
+  Acc@5 : 100.000%
+  Loss : 0.0
+
+[+] Model saved
+
+[+] Training step: 3500/5000	Training epoch: 0/31	Elapsed time: 25.33min	Learning rate: 7.779984869825457e-05
+  Acc@1 : 100.000%
+  Acc@5 : 100.000%
+  Loss : 0.0
+  FW Time : 28.149ms
+  BW Time : 25.686ms
+
+[+] Valid results
+  Acc@1 : 100.000%
+  Acc@5 : 100.000%
+  Loss : 0.0
+
+[+] Model saved
+
+[+] Training step: 4000/5000	Training epoch: 0/31	Elapsed time: 28.45min	Learning rate: 7.506003058238717e-05
+  Acc@1 : 100.000%
+  Acc@5 : 100.000%
+  Loss : 0.0
+  FW Time : 38.845ms
+  BW Time : 32.935ms
+
+[+] Valid results
+  Acc@1 : 100.000%
+  Acc@5 : 100.000%
+  Loss : 0.0
+
+[+] Model saved
+
+[+] Training step: 4500/5000	Training epoch: 0/31	Elapsed time: 31.40min	Learning rate: 7.24166985578635e-05
+  Acc@1 : 100.000%
+  Acc@5 : 100.000%
+  Loss : 0.0
+  FW Time : 34.974ms
+  BW Time : 27.569ms
+
+[+] Valid results
+  Acc@1 : 100.000%
+  Acc@5 : 100.000%
+  Loss : 0.0
+
+[+] Model saved
\ No newline at end of file

+max step = 3000
+step = 500
+lr = 0.0001
+batch_size = 64
+
+
+DEFALUT_CANDIDATES = [
+    ShearXY,
+    TranslateXY,
+    # Rotate,
+    # AutoContrast, #߰
+    # Invert,
+    Equalize,
+    Solarize,
+    Posterize,
+    Contrast,
+    # Color,
+    Brightness,
+    Sharpness,
+    Cutout,
+#     SamplePairing,
+]
\ No newline at end of file