max step = 5000
step = 500
lr = 0.0001
batch_size = 128


DEFALUT_CANDIDATES = [
    ShearXY,
    TranslateXY,
    Rotate,
    AutoContrast,
    Invert,
    Equalize,
    Solarize,
    Posterize,
    Contrast,
    Color,
    Brightness,
    Sharpness,
    Cutout
]




[+] Parse arguments
Args(augment_path=None, batch_size=64, dataset='BraTS', fast_auto_augment=True, learning_rate=0.0001, max_step=5000, network='resnet50', num_workers=4, optimizer='adam', print_step=500, scheduler='exp', seed=None, start_step=0, use_cuda=True, val_step=500)

[+] Create log dir
2020-04-06 10:55:55.010279: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.1

[+] Create network

[+] Load dataset
[+] Child 0 training started (GPU: 0)

[+] Training step: 0/5000	Elapsed time: 0.11min	Learning rate: 9.999283e-05	Device name: Tesla P4
  Acc@1 : 0.000%
  Acc@5 : 1.562%
  Loss : 6.452464580535889

[+] Training step: 500/5000	Elapsed time: 4.94min	Learning rate: 9.647145853624023e-05	Device name: Tesla P4
  Acc@1 : 100.000%
  Acc@5 : 100.000%
  Loss : 0.0003164634108543396

[+] Training step: 1000/5000	Elapsed time: 9.78min	Learning rate: 9.307409653381692e-05	Device name: Tesla P4
  Acc@1 : 100.000%
  Acc@5 : 100.000%
  Loss : 0.00015767663717269897

[+] Training step: 1500/5000	Elapsed time: 14.55min	Learning rate: 8.979637684582141e-05	Device name: Tesla P4
  Acc@1 : 100.000%
  Acc@5 : 100.000%
  Loss : 7.193535566329956e-05

[+] Training step: 2000/5000	Elapsed time: 19.42min	Learning rate: 8.663408611983758e-05	Device name: Tesla P4
  Acc@1 : 100.000%
  Acc@5 : 100.000%
  Loss : 3.548711538314819e-05

[+] Training step: 2500/5000	Elapsed time: 24.18min	Learning rate: 8.358315938187757e-05	Device name: Tesla P4
  Acc@1 : 100.000%
  Acc@5 : 100.000%
  Loss : 2.549588680267334e-05

[+] Training step: 3000/5000	Elapsed time: 29.06min	Learning rate: 8.063967481105164e-05	Device name: Tesla P4
  Acc@1 : 100.000%
  Acc@5 : 100.000%
  Loss : 2.321600914001465e-05

[+] Training step: 3500/5000	Elapsed time: 33.86min	Learning rate: 7.779984869825457e-05	Device name: Tesla P4
  Acc@1 : 100.000%
  Acc@5 : 100.000%
  Loss : 2.5473142159171402e-05

[+] Training step: 4000/5000	Elapsed time: 38.75min	Learning rate: 7.506003058238717e-05	Device name: Tesla P4
  Acc@1 : 100.000%
  Acc@5 : 100.000%
  Loss : 8.58306884765625e-06

[+] Training step: 4500/5000	Elapsed time: 43.64min	Learning rate: 7.24166985578635e-05	Device name: Tesla P4
  Acc@1 : 100.000%
  Acc@5 : 100.000%
  Loss : 6.780028343200684e-06
100% 1/1 [00:06<00:00,  6.08s/it, best loss: 5.96450536249904e-06]
100% 1/1 [00:04<00:00,  4.25s/it, best loss: 8.08198547019856e-06]
[+] Child 1 training started (GPU: 0)

[+] Training step: 0/5000	Elapsed time: 0.06min	Learning rate: 9.999283e-05	Device name: Tesla P4
  Acc@1 : 0.000%
  Acc@5 : 1.562%
  Loss : 6.547057628631592

[+] Training step: 500/5000	Elapsed time: 4.90min	Learning rate: 9.647145853624023e-05	Device name: Tesla P4
  Acc@1 : 100.000%
  Acc@5 : 100.000%
  Loss : 0.000509507954120636

[+] Training step: 1000/5000	Elapsed time: 9.79min	Learning rate: 9.307409653381692e-05	Device name: Tesla P4
  Acc@1 : 100.000%
  Acc@5 : 100.000%
  Loss : 0.0001100301742553711

[+] Training step: 1500/5000	Elapsed time: 14.60min	Learning rate: 8.979637684582141e-05	Device name: Tesla P4
  Acc@1 : 100.000%
  Acc@5 : 100.000%
  Loss : 7.142126560211182e-05

[+] Training step: 2000/5000	Elapsed time: 19.46min	Learning rate: 8.663408611983758e-05	Device name: Tesla P4
  Acc@1 : 100.000%
  Acc@5 : 100.000%
  Loss : 5.066394805908203e-05

[+] Training step: 2500/5000	Elapsed time: 24.26min	Learning rate: 8.358315938187757e-05	Device name: Tesla P4
  Acc@1 : 100.000%
  Acc@5 : 100.000%
  Loss : 2.6144087314605713e-05

[+] Training step: 3000/5000	Elapsed time: 29.14min	Learning rate: 8.063967481105164e-05	Device name: Tesla P4
  Acc@1 : 100.000%
  Acc@5 : 100.000%
  Loss : 1.6264617443084717e-05

[+] Training step: 3500/5000	Elapsed time: 33.96min	Learning rate: 7.779984869825457e-05	Device name: Tesla P4
  Acc@1 : 100.000%
  Acc@5 : 100.000%
  Loss : 2.374147152295336e-05

[+] Training step: 4000/5000	Elapsed time: 38.83min	Learning rate: 7.506003058238717e-05	Device name: Tesla P4
  Acc@1 : 100.000%
  Acc@5 : 100.000%
  Loss : 1.138448715209961e-05

[+] Training step: 4500/5000	Elapsed time: 43.67min	Learning rate: 7.24166985578635e-05	Device name: Tesla P4
  Acc@1 : 100.000%
  Acc@5 : 100.000%
  Loss : 6.288290023803711e-06
100% 1/1 [00:04<00:00,  4.13s/it, best loss: 4.1864686863846146e-06]
100% 1/1 [00:04<00:00,  4.07s/it, best loss: 8.486084880132694e-06]
[+] Child 2 training started (GPU: 0)

[+] Training step: 0/5000	Elapsed time: 0.07min	Learning rate: 9.999283e-05	Device name: Tesla P4
  Acc@1 : 1.562%
  Acc@5 : 1.562%
  Loss : 6.490279674530029

[+] Training step: 500/5000	Elapsed time: 4.82min	Learning rate: 9.647145853624023e-05	Device name: Tesla P4
  Acc@1 : 100.000%
  Acc@5 : 100.000%
  Loss : 0.0005068182945251465

[+] Training step: 1000/5000	Elapsed time: 9.63min	Learning rate: 9.307409653381692e-05	Device name: Tesla P4
  Acc@1 : 100.000%
  Acc@5 : 100.000%
  Loss : 0.00012455880641937256

[+] Training step: 1500/5000	Elapsed time: 14.36min	Learning rate: 8.979637684582141e-05	Device name: Tesla P4
  Acc@1 : 100.000%
  Acc@5 : 100.000%
  Loss : 6.882846355438232e-05

[+] Training step: 2000/5000	Elapsed time: 19.16min	Learning rate: 8.663408611983758e-05	Device name: Tesla P4
  Acc@1 : 100.000%
  Acc@5 : 100.000%
  Loss : 2.9802322387695312e-05

[+] Training step: 2500/5000	Elapsed time: 23.87min	Learning rate: 8.358315938187757e-05	Device name: Tesla P4
  Acc@1 : 100.000%
  Acc@5 : 100.000%
  Loss : 1.917034387588501e-05

[+] Training step: 3000/5000	Elapsed time: 28.66min	Learning rate: 8.063967481105164e-05	Device name: Tesla P4
  Acc@1 : 100.000%
  Acc@5 : 100.000%
  Loss : 1.6361474990844727e-05

[+] Training step: 3500/5000	Elapsed time: 33.37min	Learning rate: 7.779984869825457e-05	Device name: Tesla P4
  Acc@1 : 100.000%
  Acc@5 : 100.000%
  Loss : 1.0741384357970674e-05

[+] Training step: 4000/5000	Elapsed time: 38.21min	Learning rate: 7.506003058238717e-05	Device name: Tesla P4
  Acc@1 : 100.000%
  Acc@5 : 100.000%
  Loss : 7.048249244689941e-06

[+] Training step: 4500/5000	Elapsed time: 43.05min	Learning rate: 7.24166985578635e-05	Device name: Tesla P4
  Acc@1 : 100.000%
  Acc@5 : 100.000%
  Loss : 7.450580596923828e-06
100% 1/1 [00:04<00:00,  4.30s/it, best loss: 3.3297781101282453e-06]
100% 1/1 [00:04<00:00,  4.30s/it, best loss: 7.823362466297112e-06]
[+] Child 3 training started (GPU: 0)

[+] Training step: 0/5000	Elapsed time: 0.06min	Learning rate: 9.999283e-05	Device name: Tesla P4
  Acc@1 : 0.000%
  Acc@5 : 0.000%
  Loss : 6.425505638122559

[+] Training step: 500/5000	Elapsed time: 4.89min	Learning rate: 9.647145853624023e-05	Device name: Tesla P4
  Acc@1 : 100.000%
  Acc@5 : 100.000%
  Loss : 0.0004970133304595947

[+] Training step: 1000/5000	Elapsed time: 9.81min	Learning rate: 9.307409653381692e-05	Device name: Tesla P4
  Acc@1 : 100.000%
  Acc@5 : 100.000%
  Loss : 0.00017832964658737183

[+] Training step: 1500/5000	Elapsed time: 14.64min	Learning rate: 8.979637684582141e-05	Device name: Tesla P4
  Acc@1 : 100.000%
  Acc@5 : 100.000%
  Loss : 6.403028964996338e-05

[+] Training step: 2000/5000	Elapsed time: 19.53min	Learning rate: 8.663408611983758e-05	Device name: Tesla P4
  Acc@1 : 100.000%
  Acc@5 : 100.000%
  Loss : 7.484853267669678e-05

[+] Training step: 2500/5000	Elapsed time: 24.35min	Learning rate: 8.358315938187757e-05	Device name: Tesla P4
  Acc@1 : 100.000%
  Acc@5 : 100.000%
  Loss : 3.053247928619385e-05

[+] Training step: 3000/5000	Elapsed time: 29.26min	Learning rate: 8.063967481105164e-05	Device name: Tesla P4
  Acc@1 : 100.000%
  Acc@5 : 100.000%
  Loss : 1.741945743560791e-05

[+] Training step: 3500/5000	Elapsed time: 34.08min	Learning rate: 7.779984869825457e-05	Device name: Tesla P4
  Acc@1 : 100.000%
  Acc@5 : 100.000%
  Loss : 2.042870801233221e-05

[+] Training step: 4000/5000	Elapsed time: 38.94min	Learning rate: 7.506003058238717e-05	Device name: Tesla P4
  Acc@1 : 100.000%
  Acc@5 : 100.000%
  Loss : 7.957220077514648e-06

[+] Training step: 4500/5000	Elapsed time: 43.80min	Learning rate: 7.24166985578635e-05	Device name: Tesla P4
  Acc@1 : 100.000%
  Acc@5 : 100.000%
  Loss : 6.332993507385254e-06
100% 1/1 [00:04<00:00,  4.14s/it, best loss: 6.594900241907453e-06]
100% 1/1 [00:04<00:00,  4.14s/it, best loss: 2.6993832307198318e-06]
RandomChoice(
    Compose(
    Pad(padding=4, fill=0, padding_mode=constant)
    RandomCrop(size=(32, 32), padding=None)
    RandomHorizontalFlip(p=0.5)
    ShearXY(prob=0.67, magnitude=0.71)
    Posterize(prob=0.38, magnitude=0.90)
    ToTensor()
)
    Compose(
    Pad(padding=4, fill=0, padding_mode=constant)
    RandomCrop(size=(32, 32), padding=None)
    RandomHorizontalFlip(p=0.5)
    Color(prob=0.90, magnitude=0.30)
    Rotate(prob=0.41, magnitude=0.51)
    ToTensor()
)
    Compose(
    Pad(padding=4, fill=0, padding_mode=constant)
    RandomCrop(size=(32, 32), padding=None)
    RandomHorizontalFlip(p=0.5)
    Sharpness(prob=0.84, magnitude=0.21)
    Contrast(prob=0.88, magnitude=0.16)
    ToTensor()
)
    Compose(
    Pad(padding=4, fill=0, padding_mode=constant)
    RandomCrop(size=(32, 32), padding=None)
    RandomHorizontalFlip(p=0.5)
    ShearXY(prob=0.24, magnitude=0.39)
    TranslateXY(prob=0.47, magnitude=0.55)
    ToTensor()
)
    Compose(
    Pad(padding=4, fill=0, padding_mode=constant)
    RandomCrop(size=(32, 32), padding=None)
    RandomHorizontalFlip(p=0.5)
    Sharpness(prob=0.38, magnitude=0.48)
    Equalize(prob=0.72, magnitude=0.25)
    ToTensor()
)
    Compose(
    Pad(padding=4, fill=0, padding_mode=constant)
    RandomCrop(size=(32, 32), padding=None)
    RandomHorizontalFlip(p=0.5)
    Rotate(prob=0.84, magnitude=0.36)
    AutoContrast(prob=0.12, magnitude=0.05)
    ToTensor()
)
    Compose(
    Pad(padding=4, fill=0, padding_mode=constant)
    RandomCrop(size=(32, 32), padding=None)
    RandomHorizontalFlip(p=0.5)
    Contrast(prob=0.26, magnitude=0.12)
    ShearXY(prob=0.32, magnitude=0.13)
    ToTensor()
)
    Compose(
    Pad(padding=4, fill=0, padding_mode=constant)
    RandomCrop(size=(32, 32), padding=None)
    RandomHorizontalFlip(p=0.5)
    Solarize(prob=0.05, magnitude=0.03)
    Sharpness(prob=0.97, magnitude=0.83)
    ToTensor()
)
)

[+] Start training

[+] Use 1 GPUs

[+] Using GPU: Tesla P4 
/usr/local/lib/python3.6/dist-packages/torch/nn/functional.py:2457: UserWarning: nn.functional.upsample is deprecated. Use nn.functional.interpolate instead.
  warnings.warn("nn.functional.upsample is deprecated. Use nn.functional.interpolate instead.")
/usr/local/lib/python3.6/dist-packages/torch/nn/functional.py:2539: UserWarning: Default upsampling behavior when mode=bilinear is changed to align_corners=False since 0.4.0. Please specify align_corners=True if the old behavior is desired. See the documentation of nn.Upsample for details.
  "See the documentation of nn.Upsample for details.".format(mode))

[+] Training step: 0/5000	Training epoch: 0/31	Elapsed time: 0.62min	Learning rate: 9.999283e-05
  Acc@1 : 0.000%
  Acc@5 : 0.000%
  Loss : 6.816585063934326
  FW Time : 33.351ms
  BW Time : 10.075ms

[+] Valid results
  Acc@1 : 100.000%
  Acc@5 : 100.000%
  Loss : 5.762917498941533e-05

[+] Model saved

[+] Training step: 500/5000	Training epoch: 0/31	Elapsed time: 6.52min	Learning rate: 9.647145853624023e-05
  Acc@1 : 100.000%
  Acc@5 : 100.000%
  Loss : 0.00011359900236129761
  FW Time : 19.469ms
  BW Time : 14.039ms

[+] Valid results
  Acc@1 : 100.000%
  Acc@5 : 100.000%
  Loss : 0.0

[+] Model saved

[+] Training step: 1000/5000	Training epoch: 0/31	Elapsed time: 8.88min	Learning rate: 9.307409653381692e-05
  Acc@1 : 100.000%
  Acc@5 : 100.000%
  Loss : 8.489564061164856e-05
  FW Time : 23.165ms
  BW Time : 13.944ms

[+] Valid results
  Acc@1 : 100.000%
  Acc@5 : 100.000%
  Loss : 0.0

[+] Model saved

[+] Training step: 1500/5000	Training epoch: 0/31	Elapsed time: 11.14min	Learning rate: 8.979637684582141e-05
  Acc@1 : 100.000%
  Acc@5 : 100.000%
  Loss : 6.42240047454834e-06
  FW Time : 14.039ms
  BW Time : 8.235ms

[+] Valid results
  Acc@1 : 100.000%
  Acc@5 : 100.000%
  Loss : 0.0

[+] Model saved

[+] Training step: 2000/5000	Training epoch: 0/31	Elapsed time: 13.49min	Learning rate: 8.663408611983758e-05
  Acc@1 : 100.000%
  Acc@5 : 100.000%
  Loss : 2.175569534301758e-06
  FW Time : 20.693ms
  BW Time : 14.328ms

[+] Valid results
  Acc@1 : 100.000%
  Acc@5 : 100.000%
  Loss : 0.0

[+] Model saved

[+] Training step: 2500/5000	Training epoch: 0/31	Elapsed time: 15.84min	Learning rate: 8.358315938187757e-05
  Acc@1 : 100.000%
  Acc@5 : 100.000%
  Loss : 0.0
  FW Time : 21.502ms
  BW Time : 11.221ms

[+] Valid results
  Acc@1 : 100.000%
  Acc@5 : 100.000%
  Loss : 0.0

[+] Model saved

[+] Training step: 3000/5000	Training epoch: 0/31	Elapsed time: 18.10min	Learning rate: 8.063967481105164e-05
  Acc@1 : 100.000%
  Acc@5 : 100.000%
  Loss : 0.0
  FW Time : 24.350ms
  BW Time : 13.500ms

[+] Valid results
  Acc@1 : 100.000%
  Acc@5 : 100.000%
  Loss : 0.0

[+] Model saved

[+] Training step: 3500/5000	Training epoch: 0/31	Elapsed time: 20.44min	Learning rate: 7.779984869825457e-05
  Acc@1 : 100.000%
  Acc@5 : 100.000%
  Loss : 0.0
  FW Time : 24.146ms
  BW Time : 12.905ms

[+] Valid results
  Acc@1 : 100.000%
  Acc@5 : 100.000%
  Loss : 0.0

[+] Model saved

[+] Training step: 4000/5000	Training epoch: 0/31	Elapsed time: 22.77min	Learning rate: 7.506003058238717e-05
  Acc@1 : 100.000%
  Acc@5 : 100.000%
  Loss : 0.0
  FW Time : 64.314ms
  BW Time : 17.536ms

[+] Valid results
  Acc@1 : 100.000%
  Acc@5 : 100.000%
  Loss : 0.0

[+] Model saved

[+] Training step: 4500/5000	Training epoch: 0/31	Elapsed time: 25.01min	Learning rate: 7.24166985578635e-05
  Acc@1 : 100.000%
  Acc@5 : 100.000%
  Loss : 0.0
  FW Time : 17.300ms
  BW Time : 12.326ms

[+] Valid results
  Acc@1 : 100.000%
  Acc@5 : 100.000%
  Loss : 0.0

[+] Model saved