max step = 5000
step = 500
lr = 0.0001
batch_size = 128

DEFALUT_CANDIDATES = [
    ShearXY,
    TranslateXY,
    # Rotate,
    AutoContrast,
    # Invert,
    Equalize,
    Solarize,
    Posterize,
    Contrast,
    # Color,
    Brightness,
    Sharpness,
    Cutout,
#     SamplePairing,
]

[+] Parse arguments
Args(augment_path=None, batch_size=64, dataset='BraTS', fast_auto_augment=True, learning_rate=0.0001, max_step=5000, network='resnet50', num_workers=4, optimizer='adam', print_step=500, scheduler='exp', seed=None, start_step=0, use_cuda=True, val_step=500)

[+] Create log dir
2020-04-13 06:56:00.492033: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.1

[+] Create network

[+] Load dataset
[+] Child 0 training started (GPU: 0)

[+] Training step: 0/5000	Elapsed time: 0.55min	Learning rate: 9.999283e-05	Device name: Tesla P100-PCIE-16GB
  Acc@1 : 0.000%
  Acc@5 : 0.000%
  Loss : 7.331308841705322

[+] Training step: 500/5000	Elapsed time: 7.61min	Learning rate: 9.647145853624023e-05	Device name: Tesla P100-PCIE-16GB
  Acc@1 : 100.000%
  Acc@5 : 100.000%
  Loss : 0.0003173351287841797

[+] Training step: 1000/5000	Elapsed time: 13.69min	Learning rate: 9.307409653381692e-05	Device name: Tesla P100-PCIE-16GB
  Acc@1 : 100.000%
  Acc@5 : 100.000%
  Loss : 0.00012065470218658447

[+] Training step: 1500/5000	Elapsed time: 19.60min	Learning rate: 8.979637684582141e-05	Device name: Tesla P100-PCIE-16GB
  Acc@1 : 100.000%
  Acc@5 : 100.000%
  Loss : 8.422136306762695e-05

[+] Training step: 2000/5000	Elapsed time: 25.56min	Learning rate: 8.663408611983758e-05	Device name: Tesla P100-PCIE-16GB
  Acc@1 : 100.000%
  Acc@5 : 100.000%
  Loss : 5.1371753215789795e-05

[+] Training step: 2500/5000	Elapsed time: 31.51min	Learning rate: 8.358315938187757e-05	Device name: Tesla P100-PCIE-16GB
  Acc@1 : 100.000%
  Acc@5 : 100.000%
  Loss : 2.7231872081756592e-05

[+] Training step: 3000/5000	Elapsed time: 37.30min	Learning rate: 8.063967481105164e-05	Device name: Tesla P100-PCIE-16GB
  Acc@1 : 100.000%
  Acc@5 : 100.000%
  Loss : 1.662224531173706e-05

[+] Training step: 3500/5000	Elapsed time: 42.91min	Learning rate: 7.779984869825457e-05	Device name: Tesla P100-PCIE-16GB
  Acc@1 : 100.000%
  Acc@5 : 100.000%
  Loss : 1.1795445061579812e-05

[+] Training step: 4000/5000	Elapsed time: 48.72min	Learning rate: 7.506003058238717e-05	Device name: Tesla P100-PCIE-16GB
  Acc@1 : 100.000%
  Acc@5 : 100.000%
  Loss : 6.079673767089844e-06

[+] Training step: 4500/5000	Elapsed time: 54.77min	Learning rate: 7.24166985578635e-05	Device name: Tesla P100-PCIE-16GB
  Acc@1 : 100.000%
  Acc@5 : 100.000%
  Loss : 1.3053417205810547e-05
100% 1/1 [00:31<00:00, 31.34s/it, best loss: 2.456923766658292e-06]
100% 1/1 [00:05<00:00,  5.32s/it, best loss: 6.659556220256491e-06]
[+] Child 1 training started (GPU: 0)

[+] Training step: 0/5000	Elapsed time: 0.08min	Learning rate: 9.999283e-05	Device name: Tesla P100-PCIE-16GB
  Acc@1 : 0.000%
  Acc@5 : 0.000%
  Loss : 7.408554553985596

[+] Training step: 500/5000	Elapsed time: 6.04min	Learning rate: 9.647145853624023e-05	Device name: Tesla P100-PCIE-16GB
  Acc@1 : 100.000%
  Acc@5 : 100.000%
  Loss : 0.0002929195761680603

[+] Training step: 1000/5000	Elapsed time: 12.01min	Learning rate: 9.307409653381692e-05	Device name: Tesla P100-PCIE-16GB
  Acc@1 : 100.000%
  Acc@5 : 100.000%
  Loss : 0.00017809122800827026

[+] Training step: 1500/5000	Elapsed time: 17.93min	Learning rate: 8.979637684582141e-05	Device name: Tesla P100-PCIE-16GB
  Acc@1 : 100.000%
  Acc@5 : 100.000%
  Loss : 9.942799806594849e-05

[+] Training step: 2000/5000	Elapsed time: 23.73min	Learning rate: 8.663408611983758e-05	Device name: Tesla P100-PCIE-16GB
  Acc@1 : 100.000%
  Acc@5 : 100.000%
  Loss : 4.045665264129639e-05

[+] Training step: 2500/5000	Elapsed time: 29.59min	Learning rate: 8.358315938187757e-05	Device name: Tesla P100-PCIE-16GB
  Acc@1 : 100.000%
  Acc@5 : 100.000%
  Loss : 2.6069581508636475e-05

[+] Training step: 3000/5000	Elapsed time: 35.49min	Learning rate: 8.063967481105164e-05	Device name: Tesla P100-PCIE-16GB
  Acc@1 : 100.000%
  Acc@5 : 100.000%
  Loss : 2.197176218032837e-05

[+] Training step: 3500/5000	Elapsed time: 41.35min	Learning rate: 7.779984869825457e-05	Device name: Tesla P100-PCIE-16GB
  Acc@1 : 100.000%
  Acc@5 : 100.000%
  Loss : 7.2278476181963924e-06

[+] Training step: 4000/5000	Elapsed time: 47.20min	Learning rate: 7.506003058238717e-05	Device name: Tesla P100-PCIE-16GB
  Acc@1 : 100.000%
  Acc@5 : 100.000%
  Loss : 8.530914783477783e-06

[+] Training step: 4500/5000	Elapsed time: 52.98min	Learning rate: 7.24166985578635e-05	Device name: Tesla P100-PCIE-16GB
  Acc@1 : 100.000%
  Acc@5 : 100.000%
  Loss : 5.826354026794434e-06
100% 1/1 [00:05<00:00,  5.11s/it, best loss: 8.13047790870769e-06]
100% 1/1 [00:05<00:00,  5.03s/it, best loss: 6.885851689730771e-06]
[+] Child 2 training started (GPU: 0)

[+] Training step: 0/5000	Elapsed time: 0.08min	Learning rate: 9.999283e-05	Device name: Tesla P100-PCIE-16GB
  Acc@1 : 0.000%
  Acc@5 : 0.000%
  Loss : 7.380028247833252

[+] Training step: 500/5000	Elapsed time: 5.62min	Learning rate: 9.647145853624023e-05	Device name: Tesla P100-PCIE-16GB
  Acc@1 : 100.000%
  Acc@5 : 100.000%
  Loss : 0.0003273114562034607

[+] Training step: 1000/5000	Elapsed time: 11.24min	Learning rate: 9.307409653381692e-05	Device name: Tesla P100-PCIE-16GB
  Acc@1 : 100.000%
  Acc@5 : 100.000%
  Loss : 0.00010266900062561035

[+] Training step: 1500/5000	Elapsed time: 16.86min	Learning rate: 8.979637684582141e-05	Device name: Tesla P100-PCIE-16GB
  Acc@1 : 100.000%
  Acc@5 : 100.000%
  Loss : 6.0267746448516846e-05

[+] Training step: 2000/5000	Elapsed time: 22.77min	Learning rate: 8.663408611983758e-05	Device name: Tesla P100-PCIE-16GB
  Acc@1 : 100.000%
  Acc@5 : 100.000%
  Loss : 3.4943222999572754e-05

[+] Training step: 2500/5000	Elapsed time: 28.61min	Learning rate: 8.358315938187757e-05	Device name: Tesla P100-PCIE-16GB
  Acc@1 : 100.000%
  Acc@5 : 100.000%
  Loss : 3.550201654434204e-05

[+] Training step: 3000/5000	Elapsed time: 34.66min	Learning rate: 8.063967481105164e-05	Device name: Tesla P100-PCIE-16GB
  Acc@1 : 100.000%
  Acc@5 : 100.000%
  Loss : 2.2232532501220703e-05

[+] Training step: 3500/5000	Elapsed time: 40.65min	Learning rate: 7.779984869825457e-05	Device name: Tesla P100-PCIE-16GB
  Acc@1 : 100.000%
  Acc@5 : 100.000%
  Loss : 2.436888826196082e-05

[+] Training step: 4000/5000	Elapsed time: 46.78min	Learning rate: 7.506003058238717e-05	Device name: Tesla P100-PCIE-16GB
  Acc@1 : 100.000%
  Acc@5 : 100.000%
  Loss : 8.17328691482544e-06

[+] Training step: 4500/5000	Elapsed time: 52.87min	Learning rate: 7.24166985578635e-05	Device name: Tesla P100-PCIE-16GB
  Acc@1 : 100.000%
  Acc@5 : 100.000%
  Loss : 8.38935375213623e-06
100% 1/1 [00:05<00:00,  5.43s/it, best loss: 9.423595656699035e-06]
100% 1/1 [00:05<00:00,  5.42s/it, best loss: 8.001165952009615e-06]
[+] Child 3 training started (GPU: 0)

[+] Training step: 0/5000	Elapsed time: 0.09min	Learning rate: 9.999283e-05	Device name: Tesla P100-PCIE-16GB
  Acc@1 : 0.000%
  Acc@5 : 0.000%
  Loss : 7.353156566619873

[+] Training step: 500/5000	Elapsed time: 6.07min	Learning rate: 9.647145853624023e-05	Device name: Tesla P100-PCIE-16GB
  Acc@1 : 100.000%
  Acc@5 : 100.000%
  Loss : 0.0005485564470291138

[+] Training step: 1000/5000	Elapsed time: 12.12min	Learning rate: 9.307409653381692e-05	Device name: Tesla P100-PCIE-16GB
  Acc@1 : 100.000%
  Acc@5 : 100.000%
  Loss : 0.00013493746519088745

[+] Training step: 1500/5000	Elapsed time: 18.06min	Learning rate: 8.979637684582141e-05	Device name: Tesla P100-PCIE-16GB
  Acc@1 : 100.000%
  Acc@5 : 100.000%
  Loss : 5.833059549331665e-05

[+] Training step: 2000/5000	Elapsed time: 24.09min	Learning rate: 8.663408611983758e-05	Device name: Tesla P100-PCIE-16GB
  Acc@1 : 100.000%
  Acc@5 : 100.000%
  Loss : 4.260987043380737e-05

[+] Training step: 2500/5000	Elapsed time: 30.12min	Learning rate: 8.358315938187757e-05	Device name: Tesla P100-PCIE-16GB
  Acc@1 : 100.000%
  Acc@5 : 100.000%
  Loss : 2.2239983081817627e-05

[+] Training step: 3000/5000	Elapsed time: 36.35min	Learning rate: 8.063967481105164e-05	Device name: Tesla P100-PCIE-16GB
  Acc@1 : 100.000%
  Acc@5 : 100.000%
  Loss : 1.6763806343078613e-05

[+] Training step: 3500/5000	Elapsed time: 42.52min	Learning rate: 7.779984869825457e-05	Device name: Tesla P100-PCIE-16GB
  Acc@1 : 100.000%
  Acc@5 : 100.000%
  Loss : 1.4556081623595674e-05

[+] Training step: 4000/5000	Elapsed time: 48.95min	Learning rate: 7.506003058238717e-05	Device name: Tesla P100-PCIE-16GB
  Acc@1 : 100.000%
  Acc@5 : 100.000%
  Loss : 1.1704862117767334e-05

[+] Training step: 4500/5000	Elapsed time: 55.31min	Learning rate: 7.24166985578635e-05	Device name: Tesla P100-PCIE-16GB
  Acc@1 : 100.000%
  Acc@5 : 100.000%
  Loss : 1.0773539543151855e-05
100% 1/1 [00:05<00:00,  5.62s/it, best loss: 6.91817967890529e-06]
100% 1/1 [00:05<00:00,  5.63s/it, best loss: 6.578736247320194e-06]
RandomChoice(
    Compose(
    Pad(padding=4, fill=0, padding_mode=constant)
    RandomCrop(size=(32, 32), padding=None)
    RandomHorizontalFlip(p=0.5)
    Solarize(prob=0.91, magnitude=0.02)
    Equalize(prob=0.79, magnitude=0.88)
    ToTensor()
)
    Compose(
    Pad(padding=4, fill=0, padding_mode=constant)
    RandomCrop(size=(32, 32), padding=None)
    RandomHorizontalFlip(p=0.5)
    TranslateXY(prob=0.63, magnitude=0.78)
    Cutout(prob=0.31, magnitude=0.73)
    ToTensor()
)
    Compose(
    Pad(padding=4, fill=0, padding_mode=constant)
    RandomCrop(size=(32, 32), padding=None)
    RandomHorizontalFlip(p=0.5)
    TranslateXY(prob=0.26, magnitude=0.37)
    AutoContrast(prob=0.78, magnitude=0.81)
    ToTensor()
)
    Compose(
    Pad(padding=4, fill=0, padding_mode=constant)
    RandomCrop(size=(32, 32), padding=None)
    RandomHorizontalFlip(p=0.5)
    Posterize(prob=0.71, magnitude=0.48)
    AutoContrast(prob=0.25, magnitude=0.62)
    ToTensor()
)
    Compose(
    Pad(padding=4, fill=0, padding_mode=constant)
    RandomCrop(size=(32, 32), padding=None)
    RandomHorizontalFlip(p=0.5)
    Cutout(prob=0.78, magnitude=0.30)
    TranslateXY(prob=0.72, magnitude=0.50)
    ToTensor()
)
    Compose(
    Pad(padding=4, fill=0, padding_mode=constant)
    RandomCrop(size=(32, 32), padding=None)
    RandomHorizontalFlip(p=0.5)
    AutoContrast(prob=0.53, magnitude=0.95)
    Contrast(prob=0.08, magnitude=0.43)
    ToTensor()
)
    Compose(
    Pad(padding=4, fill=0, padding_mode=constant)
    RandomCrop(size=(32, 32), padding=None)
    RandomHorizontalFlip(p=0.5)
    Cutout(prob=0.38, magnitude=0.47)
    Cutout(prob=0.98, magnitude=0.07)
    ToTensor()
)
    Compose(
    Pad(padding=4, fill=0, padding_mode=constant)
    RandomCrop(size=(32, 32), padding=None)
    RandomHorizontalFlip(p=0.5)
    Brightness(prob=0.13, magnitude=0.42)
    Brightness(prob=0.88, magnitude=0.09)
    ToTensor()
)
)

[+] Start training

[+] Use 1 GPUs

[+] Using GPU: Tesla P100-PCIE-16GB 
/usr/local/lib/python3.6/dist-packages/torch/nn/functional.py:2457: UserWarning: nn.functional.upsample is deprecated. Use nn.functional.interpolate instead.
  warnings.warn("nn.functional.upsample is deprecated. Use nn.functional.interpolate instead.")
/usr/local/lib/python3.6/dist-packages/torch/nn/functional.py:2539: UserWarning: Default upsampling behavior when mode=bilinear is changed to align_corners=False since 0.4.0. Please specify align_corners=True if the old behavior is desired. See the documentation of nn.Upsample for details.
  "See the documentation of nn.Upsample for details.".format(mode))

[+] Training step: 0/5000	Training epoch: 0/31	Elapsed time: 0.54min	Learning rate: 9.999283e-05
  Acc@1 : 0.000%
  Acc@5 : 0.000%
  Loss : 7.482730388641357
  FW Time : 27.781ms
  BW Time : 11.731ms

[+] Valid results
  Acc@1 : 100.000%
  Acc@5 : 100.000%
  Loss : 0.0

[+] Model saved

[+] Training step: 500/5000	Training epoch: 0/31	Elapsed time: 6.91min	Learning rate: 9.647145853624023e-05
  Acc@1 : 100.000%
  Acc@5 : 100.000%
  Loss : 7.505388930439949e-05
  FW Time : 30.267ms
  BW Time : 30.651ms

[+] Valid results
  Acc@1 : 100.000%
  Acc@5 : 100.000%
  Loss : 0.0

[+] Model saved

[+] Training step: 1000/5000	Training epoch: 0/31	Elapsed time: 10.06min	Learning rate: 9.307409653381692e-05
  Acc@1 : 100.000%
  Acc@5 : 100.000%
  Loss : 0.00013676658272743225
  FW Time : 40.501ms
  BW Time : 34.972ms

[+] Valid results
  Acc@1 : 100.000%
  Acc@5 : 100.000%
  Loss : 0.0

[+] Model saved

[+] Training step: 1500/5000	Training epoch: 0/31	Elapsed time: 13.08min	Learning rate: 8.979637684582141e-05
  Acc@1 : 100.000%
  Acc@5 : 100.000%
  Loss : 4.213303327560425e-06
  FW Time : 35.585ms
  BW Time : 30.589ms

[+] Valid results
  Acc@1 : 100.000%
  Acc@5 : 100.000%
  Loss : 0.0

[+] Model saved

[+] Training step: 2000/5000	Training epoch: 0/31	Elapsed time: 16.21min	Learning rate: 8.663408611983758e-05
  Acc@1 : 100.000%
  Acc@5 : 100.000%
  Loss : 5.885958671569824e-07
  FW Time : 29.106ms
  BW Time : 28.138ms

[+] Valid results
  Acc@1 : 100.000%
  Acc@5 : 100.000%
  Loss : 0.0

[+] Model saved

[+] Training step: 2500/5000	Training epoch: 0/31	Elapsed time: 19.30min	Learning rate: 8.358315938187757e-05
  Acc@1 : 100.000%
  Acc@5 : 100.000%
  Loss : 2.130866050720215e-06
  FW Time : 30.433ms
  BW Time : 32.247ms

[+] Valid results
  Acc@1 : 100.000%
  Acc@5 : 100.000%
  Loss : 0.0

[+] Model saved

[+] Training step: 3000/5000	Training epoch: 0/31	Elapsed time: 22.28min	Learning rate: 8.063967481105164e-05
  Acc@1 : 100.000%
  Acc@5 : 100.000%
  Loss : 0.0
  FW Time : 16.860ms
  BW Time : 13.823ms

[+] Valid results
  Acc@1 : 100.000%
  Acc@5 : 100.000%
  Loss : 0.0

[+] Model saved

[+] Training step: 3500/5000	Training epoch: 0/31	Elapsed time: 25.33min	Learning rate: 7.779984869825457e-05
  Acc@1 : 100.000%
  Acc@5 : 100.000%
  Loss : 0.0
  FW Time : 28.149ms
  BW Time : 25.686ms

[+] Valid results
  Acc@1 : 100.000%
  Acc@5 : 100.000%
  Loss : 0.0

[+] Model saved

[+] Training step: 4000/5000	Training epoch: 0/31	Elapsed time: 28.45min	Learning rate: 7.506003058238717e-05
  Acc@1 : 100.000%
  Acc@5 : 100.000%
  Loss : 0.0
  FW Time : 38.845ms
  BW Time : 32.935ms

[+] Valid results
  Acc@1 : 100.000%
  Acc@5 : 100.000%
  Loss : 0.0

[+] Model saved

[+] Training step: 4500/5000	Training epoch: 0/31	Elapsed time: 31.40min	Learning rate: 7.24166985578635e-05
  Acc@1 : 100.000%
  Acc@5 : 100.000%
  Loss : 0.0
  FW Time : 34.974ms
  BW Time : 27.569ms

[+] Valid results
  Acc@1 : 100.000%
  Acc@5 : 100.000%
  Loss : 0.0

[+] Model saved