2020-1-capstone-design2 / 2016104167

Go to a project
Toggle navigation Toggle navigation pinning
조현아
cf89600f 1 parent bba51889

rm a few aug policies

Inline Side-by-side
Showing 18 changed files with 955 additions and 5 deletions
...@@ -17,14 +17,14 @@ from utils import * ...@@ -17,14 +17,14 @@ from utils import *
17 DEFALUT_CANDIDATES = [ 17 DEFALUT_CANDIDATES = [
18 ShearXY, 18 ShearXY,
19 TranslateXY, 19 TranslateXY,
20 - Rotate, 20 + # Rotate,
21 - AutoContrast, 21 + # AutoContrast,
22 - Invert, 22 + # Invert,
23 Equalize, 23 Equalize,
24 Solarize, 24 Solarize,
25 Posterize, 25 Posterize,
26 Contrast, 26 Contrast,
27 - Color, 27 + # Color,
28 Brightness, 28 Brightness,
29 Sharpness, 29 Sharpness,
30 Cutout, 30 Cutout,
...@@ -87,7 +87,7 @@ def validate_child(args, model, dataset, subset_indx, transform, device=None): ...@@ -87,7 +87,7 @@ def validate_child(args, model, dataset, subset_indx, transform, device=None):
87 87
88 dataset.transform = transform 88 dataset.transform = transform
89 subset = Subset(dataset, subset_indx) 89 subset = Subset(dataset, subset_indx)
90 - data_loader = get_dataloader(args, subset, pin_memory=False) 90 + data_loader = get_dataloader(args, subset, pin_memory=False) ##
91 91
92 return validate(args, model, criterion, data_loader, 0, None, device) 92 return validate(args, model, criterion, data_loader, 0, None, device)
93 93
......
1 +{"use_cuda": true, "network": "resnet50", "dataset": "BraTS", "optimizer": "adam", "fast_auto_augment": true, "learning_rate": 0.01, "seed": null, "num_workers": 4, "print_step": 500, "val_step": 500, "scheduler": "exp", "batch_size": 128, "start_step": 0, "max_step": 10000, "augment_path": null}
...\ No newline at end of file ...\ No newline at end of file
1 +
2 +04:28
...\ No newline at end of file ...\ No newline at end of file
1 +{"use_cuda": true, "network": "resnet50", "dataset": "BraTS", "optimizer": "adam", "fast_auto_augment": true, "learning_rate": 0.0001, "seed": null, "num_workers": 4, "print_step": 500, "val_step": 500, "scheduler": "exp", "batch_size": 128, "start_step": 0, "max_step": 10000, "augment_path": null}
...\ No newline at end of file ...\ No newline at end of file
1 +{"use_cuda": true, "network": "resnet50", "dataset": "BraTS", "optimizer": "adam", "fast_auto_augment": true, "print_step": 300, "batch_size": 4, "val_step": 300, "max_step": 300, "learning_rate": 0.0001, "seed": null, "num_workers": 4, "scheduler": "exp", "start_step": 0, "augment_path": null}
...\ No newline at end of file ...\ No newline at end of file
1 +max step = 5000
2 +step = 500
3 +lr = 0.0001
4 +batch_size = 128
5 +
6 +
7 +DEFALUT_CANDIDATES = [
8 + ShearXY,
9 + TranslateXY,
10 + Rotate,
11 + AutoContrast,
12 + Invert,
13 + Equalize,
14 + Solarize,
15 + Posterize,
16 + Contrast,
17 + Color,
18 + Brightness,
19 + Sharpness,
20 + Cutout
21 +]
22 +
23 +
24 +
25 +
26 +[+] Parse arguments
27 +Args(augment_path=None, batch_size=64, dataset='BraTS', fast_auto_augment=True, learning_rate=0.0001, max_step=5000, network='resnet50', num_workers=4, optimizer='adam', print_step=500, scheduler='exp', seed=None, start_step=0, use_cuda=True, val_step=500)
28 +
29 +[+] Create log dir
30 +2020-04-06 10:55:55.010279: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.1
31 +
32 +[+] Create network
33 +
34 +[+] Load dataset
35 +[+] Child 0 training started (GPU: 0)
36 +
37 +[+] Training step: 0/5000 Elapsed time: 0.11min Learning rate: 9.999283e-05 Device name: Tesla P4
38 + Acc@1 : 0.000%
39 + Acc@5 : 1.562%
40 + Loss : 6.452464580535889
41 +
42 +[+] Training step: 500/5000 Elapsed time: 4.94min Learning rate: 9.647145853624023e-05 Device name: Tesla P4
43 + Acc@1 : 100.000%
44 + Acc@5 : 100.000%
45 + Loss : 0.0003164634108543396
46 +
47 +[+] Training step: 1000/5000 Elapsed time: 9.78min Learning rate: 9.307409653381692e-05 Device name: Tesla P4
48 + Acc@1 : 100.000%
49 + Acc@5 : 100.000%
50 + Loss : 0.00015767663717269897
51 +
52 +[+] Training step: 1500/5000 Elapsed time: 14.55min Learning rate: 8.979637684582141e-05 Device name: Tesla P4
53 + Acc@1 : 100.000%
54 + Acc@5 : 100.000%
55 + Loss : 7.193535566329956e-05
56 +
57 +[+] Training step: 2000/5000 Elapsed time: 19.42min Learning rate: 8.663408611983758e-05 Device name: Tesla P4
58 + Acc@1 : 100.000%
59 + Acc@5 : 100.000%
60 + Loss : 3.548711538314819e-05
61 +
62 +[+] Training step: 2500/5000 Elapsed time: 24.18min Learning rate: 8.358315938187757e-05 Device name: Tesla P4
63 + Acc@1 : 100.000%
64 + Acc@5 : 100.000%
65 + Loss : 2.549588680267334e-05
66 +
67 +[+] Training step: 3000/5000 Elapsed time: 29.06min Learning rate: 8.063967481105164e-05 Device name: Tesla P4
68 + Acc@1 : 100.000%
69 + Acc@5 : 100.000%
70 + Loss : 2.321600914001465e-05
71 +
72 +[+] Training step: 3500/5000 Elapsed time: 33.86min Learning rate: 7.779984869825457e-05 Device name: Tesla P4
73 + Acc@1 : 100.000%
74 + Acc@5 : 100.000%
75 + Loss : 2.5473142159171402e-05
76 +
77 +[+] Training step: 4000/5000 Elapsed time: 38.75min Learning rate: 7.506003058238717e-05 Device name: Tesla P4
78 + Acc@1 : 100.000%
79 + Acc@5 : 100.000%
80 + Loss : 8.58306884765625e-06
81 +
82 +[+] Training step: 4500/5000 Elapsed time: 43.64min Learning rate: 7.24166985578635e-05 Device name: Tesla P4
83 + Acc@1 : 100.000%
84 + Acc@5 : 100.000%
85 + Loss : 6.780028343200684e-06
86 +100% 1/1 [00:06<00:00, 6.08s/it, best loss: 5.96450536249904e-06]
87 +100% 1/1 [00:04<00:00, 4.25s/it, best loss: 8.08198547019856e-06]
88 +[+] Child 1 training started (GPU: 0)
89 +
90 +[+] Training step: 0/5000 Elapsed time: 0.06min Learning rate: 9.999283e-05 Device name: Tesla P4
91 + Acc@1 : 0.000%
92 + Acc@5 : 1.562%
93 + Loss : 6.547057628631592
94 +
95 +[+] Training step: 500/5000 Elapsed time: 4.90min Learning rate: 9.647145853624023e-05 Device name: Tesla P4
96 + Acc@1 : 100.000%
97 + Acc@5 : 100.000%
98 + Loss : 0.000509507954120636
99 +
100 +[+] Training step: 1000/5000 Elapsed time: 9.79min Learning rate: 9.307409653381692e-05 Device name: Tesla P4
101 + Acc@1 : 100.000%
102 + Acc@5 : 100.000%
103 + Loss : 0.0001100301742553711
104 +
105 +[+] Training step: 1500/5000 Elapsed time: 14.60min Learning rate: 8.979637684582141e-05 Device name: Tesla P4
106 + Acc@1 : 100.000%
107 + Acc@5 : 100.000%
108 + Loss : 7.142126560211182e-05
109 +
110 +[+] Training step: 2000/5000 Elapsed time: 19.46min Learning rate: 8.663408611983758e-05 Device name: Tesla P4
111 + Acc@1 : 100.000%
112 + Acc@5 : 100.000%
113 + Loss : 5.066394805908203e-05
114 +
115 +[+] Training step: 2500/5000 Elapsed time: 24.26min Learning rate: 8.358315938187757e-05 Device name: Tesla P4
116 + Acc@1 : 100.000%
117 + Acc@5 : 100.000%
118 + Loss : 2.6144087314605713e-05
119 +
120 +[+] Training step: 3000/5000 Elapsed time: 29.14min Learning rate: 8.063967481105164e-05 Device name: Tesla P4
121 + Acc@1 : 100.000%
122 + Acc@5 : 100.000%
123 + Loss : 1.6264617443084717e-05
124 +
125 +[+] Training step: 3500/5000 Elapsed time: 33.96min Learning rate: 7.779984869825457e-05 Device name: Tesla P4
126 + Acc@1 : 100.000%
127 + Acc@5 : 100.000%
128 + Loss : 2.374147152295336e-05
129 +
130 +[+] Training step: 4000/5000 Elapsed time: 38.83min Learning rate: 7.506003058238717e-05 Device name: Tesla P4
131 + Acc@1 : 100.000%
132 + Acc@5 : 100.000%
133 + Loss : 1.138448715209961e-05
134 +
135 +[+] Training step: 4500/5000 Elapsed time: 43.67min Learning rate: 7.24166985578635e-05 Device name: Tesla P4
136 + Acc@1 : 100.000%
137 + Acc@5 : 100.000%
138 + Loss : 6.288290023803711e-06
139 +100% 1/1 [00:04<00:00, 4.13s/it, best loss: 4.1864686863846146e-06]
140 +100% 1/1 [00:04<00:00, 4.07s/it, best loss: 8.486084880132694e-06]
141 +[+] Child 2 training started (GPU: 0)
142 +
143 +[+] Training step: 0/5000 Elapsed time: 0.07min Learning rate: 9.999283e-05 Device name: Tesla P4
144 + Acc@1 : 1.562%
145 + Acc@5 : 1.562%
146 + Loss : 6.490279674530029
147 +
148 +[+] Training step: 500/5000 Elapsed time: 4.82min Learning rate: 9.647145853624023e-05 Device name: Tesla P4
149 + Acc@1 : 100.000%
150 + Acc@5 : 100.000%
151 + Loss : 0.0005068182945251465
152 +
153 +[+] Training step: 1000/5000 Elapsed time: 9.63min Learning rate: 9.307409653381692e-05 Device name: Tesla P4
154 + Acc@1 : 100.000%
155 + Acc@5 : 100.000%
156 + Loss : 0.00012455880641937256
157 +
158 +[+] Training step: 1500/5000 Elapsed time: 14.36min Learning rate: 8.979637684582141e-05 Device name: Tesla P4
159 + Acc@1 : 100.000%
160 + Acc@5 : 100.000%
161 + Loss : 6.882846355438232e-05
162 +
163 +[+] Training step: 2000/5000 Elapsed time: 19.16min Learning rate: 8.663408611983758e-05 Device name: Tesla P4
164 + Acc@1 : 100.000%
165 + Acc@5 : 100.000%
166 + Loss : 2.9802322387695312e-05
167 +
168 +[+] Training step: 2500/5000 Elapsed time: 23.87min Learning rate: 8.358315938187757e-05 Device name: Tesla P4
169 + Acc@1 : 100.000%
170 + Acc@5 : 100.000%
171 + Loss : 1.917034387588501e-05
172 +
173 +[+] Training step: 3000/5000 Elapsed time: 28.66min Learning rate: 8.063967481105164e-05 Device name: Tesla P4
174 + Acc@1 : 100.000%
175 + Acc@5 : 100.000%
176 + Loss : 1.6361474990844727e-05
177 +
178 +[+] Training step: 3500/5000 Elapsed time: 33.37min Learning rate: 7.779984869825457e-05 Device name: Tesla P4
179 + Acc@1 : 100.000%
180 + Acc@5 : 100.000%
181 + Loss : 1.0741384357970674e-05
182 +
183 +[+] Training step: 4000/5000 Elapsed time: 38.21min Learning rate: 7.506003058238717e-05 Device name: Tesla P4
184 + Acc@1 : 100.000%
185 + Acc@5 : 100.000%
186 + Loss : 7.048249244689941e-06
187 +
188 +[+] Training step: 4500/5000 Elapsed time: 43.05min Learning rate: 7.24166985578635e-05 Device name: Tesla P4
189 + Acc@1 : 100.000%
190 + Acc@5 : 100.000%
191 + Loss : 7.450580596923828e-06
192 +100% 1/1 [00:04<00:00, 4.30s/it, best loss: 3.3297781101282453e-06]
193 +100% 1/1 [00:04<00:00, 4.30s/it, best loss: 7.823362466297112e-06]
194 +[+] Child 3 training started (GPU: 0)
195 +
196 +[+] Training step: 0/5000 Elapsed time: 0.06min Learning rate: 9.999283e-05 Device name: Tesla P4
197 + Acc@1 : 0.000%
198 + Acc@5 : 0.000%
199 + Loss : 6.425505638122559
200 +
201 +[+] Training step: 500/5000 Elapsed time: 4.89min Learning rate: 9.647145853624023e-05 Device name: Tesla P4
202 + Acc@1 : 100.000%
203 + Acc@5 : 100.000%
204 + Loss : 0.0004970133304595947
205 +
206 +[+] Training step: 1000/5000 Elapsed time: 9.81min Learning rate: 9.307409653381692e-05 Device name: Tesla P4
207 + Acc@1 : 100.000%
208 + Acc@5 : 100.000%
209 + Loss : 0.00017832964658737183
210 +
211 +[+] Training step: 1500/5000 Elapsed time: 14.64min Learning rate: 8.979637684582141e-05 Device name: Tesla P4
212 + Acc@1 : 100.000%
213 + Acc@5 : 100.000%
214 + Loss : 6.403028964996338e-05
215 +
216 +[+] Training step: 2000/5000 Elapsed time: 19.53min Learning rate: 8.663408611983758e-05 Device name: Tesla P4
217 + Acc@1 : 100.000%
218 + Acc@5 : 100.000%
219 + Loss : 7.484853267669678e-05
220 +
221 +[+] Training step: 2500/5000 Elapsed time: 24.35min Learning rate: 8.358315938187757e-05 Device name: Tesla P4
222 + Acc@1 : 100.000%
223 + Acc@5 : 100.000%
224 + Loss : 3.053247928619385e-05
225 +
226 +[+] Training step: 3000/5000 Elapsed time: 29.26min Learning rate: 8.063967481105164e-05 Device name: Tesla P4
227 + Acc@1 : 100.000%
228 + Acc@5 : 100.000%
229 + Loss : 1.741945743560791e-05
230 +
231 +[+] Training step: 3500/5000 Elapsed time: 34.08min Learning rate: 7.779984869825457e-05 Device name: Tesla P4
232 + Acc@1 : 100.000%
233 + Acc@5 : 100.000%
234 + Loss : 2.042870801233221e-05
235 +
236 +[+] Training step: 4000/5000 Elapsed time: 38.94min Learning rate: 7.506003058238717e-05 Device name: Tesla P4
237 + Acc@1 : 100.000%
238 + Acc@5 : 100.000%
239 + Loss : 7.957220077514648e-06
240 +
241 +[+] Training step: 4500/5000 Elapsed time: 43.80min Learning rate: 7.24166985578635e-05 Device name: Tesla P4
242 + Acc@1 : 100.000%
243 + Acc@5 : 100.000%
244 + Loss : 6.332993507385254e-06
245 +100% 1/1 [00:04<00:00, 4.14s/it, best loss: 6.594900241907453e-06]
246 +100% 1/1 [00:04<00:00, 4.14s/it, best loss: 2.6993832307198318e-06]
247 +RandomChoice(
248 + Compose(
249 + Pad(padding=4, fill=0, padding_mode=constant)
250 + RandomCrop(size=(32, 32), padding=None)
251 + RandomHorizontalFlip(p=0.5)
252 + ShearXY(prob=0.67, magnitude=0.71)
253 + Posterize(prob=0.38, magnitude=0.90)
254 + ToTensor()
255 +)
256 + Compose(
257 + Pad(padding=4, fill=0, padding_mode=constant)
258 + RandomCrop(size=(32, 32), padding=None)
259 + RandomHorizontalFlip(p=0.5)
260 + Color(prob=0.90, magnitude=0.30)
261 + Rotate(prob=0.41, magnitude=0.51)
262 + ToTensor()
263 +)
264 + Compose(
265 + Pad(padding=4, fill=0, padding_mode=constant)
266 + RandomCrop(size=(32, 32), padding=None)
267 + RandomHorizontalFlip(p=0.5)
268 + Sharpness(prob=0.84, magnitude=0.21)
269 + Contrast(prob=0.88, magnitude=0.16)
270 + ToTensor()
271 +)
272 + Compose(
273 + Pad(padding=4, fill=0, padding_mode=constant)
274 + RandomCrop(size=(32, 32), padding=None)
275 + RandomHorizontalFlip(p=0.5)
276 + ShearXY(prob=0.24, magnitude=0.39)
277 + TranslateXY(prob=0.47, magnitude=0.55)
278 + ToTensor()
279 +)
280 + Compose(
281 + Pad(padding=4, fill=0, padding_mode=constant)
282 + RandomCrop(size=(32, 32), padding=None)
283 + RandomHorizontalFlip(p=0.5)
284 + Sharpness(prob=0.38, magnitude=0.48)
285 + Equalize(prob=0.72, magnitude=0.25)
286 + ToTensor()
287 +)
288 + Compose(
289 + Pad(padding=4, fill=0, padding_mode=constant)
290 + RandomCrop(size=(32, 32), padding=None)
291 + RandomHorizontalFlip(p=0.5)
292 + Rotate(prob=0.84, magnitude=0.36)
293 + AutoContrast(prob=0.12, magnitude=0.05)
294 + ToTensor()
295 +)
296 + Compose(
297 + Pad(padding=4, fill=0, padding_mode=constant)
298 + RandomCrop(size=(32, 32), padding=None)
299 + RandomHorizontalFlip(p=0.5)
300 + Contrast(prob=0.26, magnitude=0.12)
301 + ShearXY(prob=0.32, magnitude=0.13)
302 + ToTensor()
303 +)
304 + Compose(
305 + Pad(padding=4, fill=0, padding_mode=constant)
306 + RandomCrop(size=(32, 32), padding=None)
307 + RandomHorizontalFlip(p=0.5)
308 + Solarize(prob=0.05, magnitude=0.03)
309 + Sharpness(prob=0.97, magnitude=0.83)
310 + ToTensor()
311 +)
312 +)
313 +
314 +[+] Start training
315 +
316 +[+] Use 1 GPUs
317 +
318 +[+] Using GPU: Tesla P4
319 +/usr/local/lib/python3.6/dist-packages/torch/nn/functional.py:2457: UserWarning: nn.functional.upsample is deprecated. Use nn.functional.interpolate instead.
320 + warnings.warn("nn.functional.upsample is deprecated. Use nn.functional.interpolate instead.")
321 +/usr/local/lib/python3.6/dist-packages/torch/nn/functional.py:2539: UserWarning: Default upsampling behavior when mode=bilinear is changed to align_corners=False since 0.4.0. Please specify align_corners=True if the old behavior is desired. See the documentation of nn.Upsample for details.
322 + "See the documentation of nn.Upsample for details.".format(mode))
323 +
324 +[+] Training step: 0/5000 Training epoch: 0/31 Elapsed time: 0.62min Learning rate: 9.999283e-05
325 + Acc@1 : 0.000%
326 + Acc@5 : 0.000%
327 + Loss : 6.816585063934326
328 + FW Time : 33.351ms
329 + BW Time : 10.075ms
330 +
331 +[+] Valid results
332 + Acc@1 : 100.000%
333 + Acc@5 : 100.000%
334 + Loss : 5.762917498941533e-05
335 +
336 +[+] Model saved
337 +
338 +[+] Training step: 500/5000 Training epoch: 0/31 Elapsed time: 6.52min Learning rate: 9.647145853624023e-05
339 + Acc@1 : 100.000%
340 + Acc@5 : 100.000%
341 + Loss : 0.00011359900236129761
342 + FW Time : 19.469ms
343 + BW Time : 14.039ms
344 +
345 +[+] Valid results
346 + Acc@1 : 100.000%
347 + Acc@5 : 100.000%
348 + Loss : 0.0
349 +
350 +[+] Model saved
351 +
352 +[+] Training step: 1000/5000 Training epoch: 0/31 Elapsed time: 8.88min Learning rate: 9.307409653381692e-05
353 + Acc@1 : 100.000%
354 + Acc@5 : 100.000%
355 + Loss : 8.489564061164856e-05
356 + FW Time : 23.165ms
357 + BW Time : 13.944ms
358 +
359 +[+] Valid results
360 + Acc@1 : 100.000%
361 + Acc@5 : 100.000%
362 + Loss : 0.0
363 +
364 +[+] Model saved
365 +
366 +[+] Training step: 1500/5000 Training epoch: 0/31 Elapsed time: 11.14min Learning rate: 8.979637684582141e-05
367 + Acc@1 : 100.000%
368 + Acc@5 : 100.000%
369 + Loss : 6.42240047454834e-06
370 + FW Time : 14.039ms
371 + BW Time : 8.235ms
372 +
373 +[+] Valid results
374 + Acc@1 : 100.000%
375 + Acc@5 : 100.000%
376 + Loss : 0.0
377 +
378 +[+] Model saved
379 +
380 +[+] Training step: 2000/5000 Training epoch: 0/31 Elapsed time: 13.49min Learning rate: 8.663408611983758e-05
381 + Acc@1 : 100.000%
382 + Acc@5 : 100.000%
383 + Loss : 2.175569534301758e-06
384 + FW Time : 20.693ms
385 + BW Time : 14.328ms
386 +
387 +[+] Valid results
388 + Acc@1 : 100.000%
389 + Acc@5 : 100.000%
390 + Loss : 0.0
391 +
392 +[+] Model saved
393 +
394 +[+] Training step: 2500/5000 Training epoch: 0/31 Elapsed time: 15.84min Learning rate: 8.358315938187757e-05
395 + Acc@1 : 100.000%
396 + Acc@5 : 100.000%
397 + Loss : 0.0
398 + FW Time : 21.502ms
399 + BW Time : 11.221ms
400 +
401 +[+] Valid results
402 + Acc@1 : 100.000%
403 + Acc@5 : 100.000%
404 + Loss : 0.0
405 +
406 +[+] Model saved
407 +
408 +[+] Training step: 3000/5000 Training epoch: 0/31 Elapsed time: 18.10min Learning rate: 8.063967481105164e-05
409 + Acc@1 : 100.000%
410 + Acc@5 : 100.000%
411 + Loss : 0.0
412 + FW Time : 24.350ms
413 + BW Time : 13.500ms
414 +
415 +[+] Valid results
416 + Acc@1 : 100.000%
417 + Acc@5 : 100.000%
418 + Loss : 0.0
419 +
420 +[+] Model saved
421 +
422 +[+] Training step: 3500/5000 Training epoch: 0/31 Elapsed time: 20.44min Learning rate: 7.779984869825457e-05
423 + Acc@1 : 100.000%
424 + Acc@5 : 100.000%
425 + Loss : 0.0
426 + FW Time : 24.146ms
427 + BW Time : 12.905ms
428 +
429 +[+] Valid results
430 + Acc@1 : 100.000%
431 + Acc@5 : 100.000%
432 + Loss : 0.0
433 +
434 +[+] Model saved
435 +
436 +[+] Training step: 4000/5000 Training epoch: 0/31 Elapsed time: 22.77min Learning rate: 7.506003058238717e-05
437 + Acc@1 : 100.000%
438 + Acc@5 : 100.000%
439 + Loss : 0.0
440 + FW Time : 64.314ms
441 + BW Time : 17.536ms
442 +
443 +[+] Valid results
444 + Acc@1 : 100.000%
445 + Acc@5 : 100.000%
446 + Loss : 0.0
447 +
448 +[+] Model saved
449 +
450 +[+] Training step: 4500/5000 Training epoch: 0/31 Elapsed time: 25.01min Learning rate: 7.24166985578635e-05
451 + Acc@1 : 100.000%
452 + Acc@5 : 100.000%
453 + Loss : 0.0
454 + FW Time : 17.300ms
455 + BW Time : 12.326ms
456 +
457 +[+] Valid results
458 + Acc@1 : 100.000%
459 + Acc@5 : 100.000%
460 + Loss : 0.0
461 +
462 +[+] Model saved
...\ No newline at end of file ...\ No newline at end of file
1 +max step = 5000
2 +step = 500
3 +lr = 0.0001
4 +batch_size = 128
5 +
6 +DEFALUT_CANDIDATES = [
7 + ShearXY,
8 + TranslateXY,
9 + # Rotate,
10 + AutoContrast,
11 + # Invert,
12 + Equalize,
13 + Solarize,
14 + Posterize,
15 + Contrast,
16 + # Color,
17 + Brightness,
18 + Sharpness,
19 + Cutout,
20 +# SamplePairing,
21 +]
22 +
23 +[+] Parse arguments
24 +Args(augment_path=None, batch_size=64, dataset='BraTS', fast_auto_augment=True, learning_rate=0.0001, max_step=5000, network='resnet50', num_workers=4, optimizer='adam', print_step=500, scheduler='exp', seed=None, start_step=0, use_cuda=True, val_step=500)
25 +
26 +[+] Create log dir
27 +2020-04-13 06:56:00.492033: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.1
28 +
29 +[+] Create network
30 +
31 +[+] Load dataset
32 +[+] Child 0 training started (GPU: 0)
33 +
34 +[+] Training step: 0/5000 Elapsed time: 0.55min Learning rate: 9.999283e-05 Device name: Tesla P100-PCIE-16GB
35 + Acc@1 : 0.000%
36 + Acc@5 : 0.000%
37 + Loss : 7.331308841705322
38 +
39 +[+] Training step: 500/5000 Elapsed time: 7.61min Learning rate: 9.647145853624023e-05 Device name: Tesla P100-PCIE-16GB
40 + Acc@1 : 100.000%
41 + Acc@5 : 100.000%
42 + Loss : 0.0003173351287841797
43 +
44 +[+] Training step: 1000/5000 Elapsed time: 13.69min Learning rate: 9.307409653381692e-05 Device name: Tesla P100-PCIE-16GB
45 + Acc@1 : 100.000%
46 + Acc@5 : 100.000%
47 + Loss : 0.00012065470218658447
48 +
49 +[+] Training step: 1500/5000 Elapsed time: 19.60min Learning rate: 8.979637684582141e-05 Device name: Tesla P100-PCIE-16GB
50 + Acc@1 : 100.000%
51 + Acc@5 : 100.000%
52 + Loss : 8.422136306762695e-05
53 +
54 +[+] Training step: 2000/5000 Elapsed time: 25.56min Learning rate: 8.663408611983758e-05 Device name: Tesla P100-PCIE-16GB
55 + Acc@1 : 100.000%
56 + Acc@5 : 100.000%
57 + Loss : 5.1371753215789795e-05
58 +
59 +[+] Training step: 2500/5000 Elapsed time: 31.51min Learning rate: 8.358315938187757e-05 Device name: Tesla P100-PCIE-16GB
60 + Acc@1 : 100.000%
61 + Acc@5 : 100.000%
62 + Loss : 2.7231872081756592e-05
63 +
64 +[+] Training step: 3000/5000 Elapsed time: 37.30min Learning rate: 8.063967481105164e-05 Device name: Tesla P100-PCIE-16GB
65 + Acc@1 : 100.000%
66 + Acc@5 : 100.000%
67 + Loss : 1.662224531173706e-05
68 +
69 +[+] Training step: 3500/5000 Elapsed time: 42.91min Learning rate: 7.779984869825457e-05 Device name: Tesla P100-PCIE-16GB
70 + Acc@1 : 100.000%
71 + Acc@5 : 100.000%
72 + Loss : 1.1795445061579812e-05
73 +
74 +[+] Training step: 4000/5000 Elapsed time: 48.72min Learning rate: 7.506003058238717e-05 Device name: Tesla P100-PCIE-16GB
75 + Acc@1 : 100.000%
76 + Acc@5 : 100.000%
77 + Loss : 6.079673767089844e-06
78 +
79 +[+] Training step: 4500/5000 Elapsed time: 54.77min Learning rate: 7.24166985578635e-05 Device name: Tesla P100-PCIE-16GB
80 + Acc@1 : 100.000%
81 + Acc@5 : 100.000%
82 + Loss : 1.3053417205810547e-05
83 +100% 1/1 [00:31<00:00, 31.34s/it, best loss: 2.456923766658292e-06]
84 +100% 1/1 [00:05<00:00, 5.32s/it, best loss: 6.659556220256491e-06]
85 +[+] Child 1 training started (GPU: 0)
86 +
87 +[+] Training step: 0/5000 Elapsed time: 0.08min Learning rate: 9.999283e-05 Device name: Tesla P100-PCIE-16GB
88 + Acc@1 : 0.000%
89 + Acc@5 : 0.000%
90 + Loss : 7.408554553985596
91 +
92 +[+] Training step: 500/5000 Elapsed time: 6.04min Learning rate: 9.647145853624023e-05 Device name: Tesla P100-PCIE-16GB
93 + Acc@1 : 100.000%
94 + Acc@5 : 100.000%
95 + Loss : 0.0002929195761680603
96 +
97 +[+] Training step: 1000/5000 Elapsed time: 12.01min Learning rate: 9.307409653381692e-05 Device name: Tesla P100-PCIE-16GB
98 + Acc@1 : 100.000%
99 + Acc@5 : 100.000%
100 + Loss : 0.00017809122800827026
101 +
102 +[+] Training step: 1500/5000 Elapsed time: 17.93min Learning rate: 8.979637684582141e-05 Device name: Tesla P100-PCIE-16GB
103 + Acc@1 : 100.000%
104 + Acc@5 : 100.000%
105 + Loss : 9.942799806594849e-05
106 +
107 +[+] Training step: 2000/5000 Elapsed time: 23.73min Learning rate: 8.663408611983758e-05 Device name: Tesla P100-PCIE-16GB
108 + Acc@1 : 100.000%
109 + Acc@5 : 100.000%
110 + Loss : 4.045665264129639e-05
111 +
112 +[+] Training step: 2500/5000 Elapsed time: 29.59min Learning rate: 8.358315938187757e-05 Device name: Tesla P100-PCIE-16GB
113 + Acc@1 : 100.000%
114 + Acc@5 : 100.000%
115 + Loss : 2.6069581508636475e-05
116 +
117 +[+] Training step: 3000/5000 Elapsed time: 35.49min Learning rate: 8.063967481105164e-05 Device name: Tesla P100-PCIE-16GB
118 + Acc@1 : 100.000%
119 + Acc@5 : 100.000%
120 + Loss : 2.197176218032837e-05
121 +
122 +[+] Training step: 3500/5000 Elapsed time: 41.35min Learning rate: 7.779984869825457e-05 Device name: Tesla P100-PCIE-16GB
123 + Acc@1 : 100.000%
124 + Acc@5 : 100.000%
125 + Loss : 7.2278476181963924e-06
126 +
127 +[+] Training step: 4000/5000 Elapsed time: 47.20min Learning rate: 7.506003058238717e-05 Device name: Tesla P100-PCIE-16GB
128 + Acc@1 : 100.000%
129 + Acc@5 : 100.000%
130 + Loss : 8.530914783477783e-06
131 +
132 +[+] Training step: 4500/5000 Elapsed time: 52.98min Learning rate: 7.24166985578635e-05 Device name: Tesla P100-PCIE-16GB
133 + Acc@1 : 100.000%
134 + Acc@5 : 100.000%
135 + Loss : 5.826354026794434e-06
136 +100% 1/1 [00:05<00:00, 5.11s/it, best loss: 8.13047790870769e-06]
137 +100% 1/1 [00:05<00:00, 5.03s/it, best loss: 6.885851689730771e-06]
138 +[+] Child 2 training started (GPU: 0)
139 +
140 +[+] Training step: 0/5000 Elapsed time: 0.08min Learning rate: 9.999283e-05 Device name: Tesla P100-PCIE-16GB
141 + Acc@1 : 0.000%
142 + Acc@5 : 0.000%
143 + Loss : 7.380028247833252
144 +
145 +[+] Training step: 500/5000 Elapsed time: 5.62min Learning rate: 9.647145853624023e-05 Device name: Tesla P100-PCIE-16GB
146 + Acc@1 : 100.000%
147 + Acc@5 : 100.000%
148 + Loss : 0.0003273114562034607
149 +
150 +[+] Training step: 1000/5000 Elapsed time: 11.24min Learning rate: 9.307409653381692e-05 Device name: Tesla P100-PCIE-16GB
151 + Acc@1 : 100.000%
152 + Acc@5 : 100.000%
153 + Loss : 0.00010266900062561035
154 +
155 +[+] Training step: 1500/5000 Elapsed time: 16.86min Learning rate: 8.979637684582141e-05 Device name: Tesla P100-PCIE-16GB
156 + Acc@1 : 100.000%
157 + Acc@5 : 100.000%
158 + Loss : 6.0267746448516846e-05
159 +
160 +[+] Training step: 2000/5000 Elapsed time: 22.77min Learning rate: 8.663408611983758e-05 Device name: Tesla P100-PCIE-16GB
161 + Acc@1 : 100.000%
162 + Acc@5 : 100.000%
163 + Loss : 3.4943222999572754e-05
164 +
165 +[+] Training step: 2500/5000 Elapsed time: 28.61min Learning rate: 8.358315938187757e-05 Device name: Tesla P100-PCIE-16GB
166 + Acc@1 : 100.000%
167 + Acc@5 : 100.000%
168 + Loss : 3.550201654434204e-05
169 +
170 +[+] Training step: 3000/5000 Elapsed time: 34.66min Learning rate: 8.063967481105164e-05 Device name: Tesla P100-PCIE-16GB
171 + Acc@1 : 100.000%
172 + Acc@5 : 100.000%
173 + Loss : 2.2232532501220703e-05
174 +
175 +[+] Training step: 3500/5000 Elapsed time: 40.65min Learning rate: 7.779984869825457e-05 Device name: Tesla P100-PCIE-16GB
176 + Acc@1 : 100.000%
177 + Acc@5 : 100.000%
178 + Loss : 2.436888826196082e-05
179 +
180 +[+] Training step: 4000/5000 Elapsed time: 46.78min Learning rate: 7.506003058238717e-05 Device name: Tesla P100-PCIE-16GB
181 + Acc@1 : 100.000%
182 + Acc@5 : 100.000%
183 + Loss : 8.17328691482544e-06
184 +
185 +[+] Training step: 4500/5000 Elapsed time: 52.87min Learning rate: 7.24166985578635e-05 Device name: Tesla P100-PCIE-16GB
186 + Acc@1 : 100.000%
187 + Acc@5 : 100.000%
188 + Loss : 8.38935375213623e-06
189 +100% 1/1 [00:05<00:00, 5.43s/it, best loss: 9.423595656699035e-06]
190 +100% 1/1 [00:05<00:00, 5.42s/it, best loss: 8.001165952009615e-06]
191 +[+] Child 3 training started (GPU: 0)
192 +
193 +[+] Training step: 0/5000 Elapsed time: 0.09min Learning rate: 9.999283e-05 Device name: Tesla P100-PCIE-16GB
194 + Acc@1 : 0.000%
195 + Acc@5 : 0.000%
196 + Loss : 7.353156566619873
197 +
198 +[+] Training step: 500/5000 Elapsed time: 6.07min Learning rate: 9.647145853624023e-05 Device name: Tesla P100-PCIE-16GB
199 + Acc@1 : 100.000%
200 + Acc@5 : 100.000%
201 + Loss : 0.0005485564470291138
202 +
203 +[+] Training step: 1000/5000 Elapsed time: 12.12min Learning rate: 9.307409653381692e-05 Device name: Tesla P100-PCIE-16GB
204 + Acc@1 : 100.000%
205 + Acc@5 : 100.000%
206 + Loss : 0.00013493746519088745
207 +
208 +[+] Training step: 1500/5000 Elapsed time: 18.06min Learning rate: 8.979637684582141e-05 Device name: Tesla P100-PCIE-16GB
209 + Acc@1 : 100.000%
210 + Acc@5 : 100.000%
211 + Loss : 5.833059549331665e-05
212 +
213 +[+] Training step: 2000/5000 Elapsed time: 24.09min Learning rate: 8.663408611983758e-05 Device name: Tesla P100-PCIE-16GB
214 + Acc@1 : 100.000%
215 + Acc@5 : 100.000%
216 + Loss : 4.260987043380737e-05
217 +
218 +[+] Training step: 2500/5000 Elapsed time: 30.12min Learning rate: 8.358315938187757e-05 Device name: Tesla P100-PCIE-16GB
219 + Acc@1 : 100.000%
220 + Acc@5 : 100.000%
221 + Loss : 2.2239983081817627e-05
222 +
223 +[+] Training step: 3000/5000 Elapsed time: 36.35min Learning rate: 8.063967481105164e-05 Device name: Tesla P100-PCIE-16GB
224 + Acc@1 : 100.000%
225 + Acc@5 : 100.000%
226 + Loss : 1.6763806343078613e-05
227 +
228 +[+] Training step: 3500/5000 Elapsed time: 42.52min Learning rate: 7.779984869825457e-05 Device name: Tesla P100-PCIE-16GB
229 + Acc@1 : 100.000%
230 + Acc@5 : 100.000%
231 + Loss : 1.4556081623595674e-05
232 +
233 +[+] Training step: 4000/5000 Elapsed time: 48.95min Learning rate: 7.506003058238717e-05 Device name: Tesla P100-PCIE-16GB
234 + Acc@1 : 100.000%
235 + Acc@5 : 100.000%
236 + Loss : 1.1704862117767334e-05
237 +
238 +[+] Training step: 4500/5000 Elapsed time: 55.31min Learning rate: 7.24166985578635e-05 Device name: Tesla P100-PCIE-16GB
239 + Acc@1 : 100.000%
240 + Acc@5 : 100.000%
241 + Loss : 1.0773539543151855e-05
242 +100% 1/1 [00:05<00:00, 5.62s/it, best loss: 6.91817967890529e-06]
243 +100% 1/1 [00:05<00:00, 5.63s/it, best loss: 6.578736247320194e-06]
244 +RandomChoice(
245 + Compose(
246 + Pad(padding=4, fill=0, padding_mode=constant)
247 + RandomCrop(size=(32, 32), padding=None)
248 + RandomHorizontalFlip(p=0.5)
249 + Solarize(prob=0.91, magnitude=0.02)
250 + Equalize(prob=0.79, magnitude=0.88)
251 + ToTensor()
252 +)
253 + Compose(
254 + Pad(padding=4, fill=0, padding_mode=constant)
255 + RandomCrop(size=(32, 32), padding=None)
256 + RandomHorizontalFlip(p=0.5)
257 + TranslateXY(prob=0.63, magnitude=0.78)
258 + Cutout(prob=0.31, magnitude=0.73)
259 + ToTensor()
260 +)
261 + Compose(
262 + Pad(padding=4, fill=0, padding_mode=constant)
263 + RandomCrop(size=(32, 32), padding=None)
264 + RandomHorizontalFlip(p=0.5)
265 + TranslateXY(prob=0.26, magnitude=0.37)
266 + AutoContrast(prob=0.78, magnitude=0.81)
267 + ToTensor()
268 +)
269 + Compose(
270 + Pad(padding=4, fill=0, padding_mode=constant)
271 + RandomCrop(size=(32, 32), padding=None)
272 + RandomHorizontalFlip(p=0.5)
273 + Posterize(prob=0.71, magnitude=0.48)
274 + AutoContrast(prob=0.25, magnitude=0.62)
275 + ToTensor()
276 +)
277 + Compose(
278 + Pad(padding=4, fill=0, padding_mode=constant)
279 + RandomCrop(size=(32, 32), padding=None)
280 + RandomHorizontalFlip(p=0.5)
281 + Cutout(prob=0.78, magnitude=0.30)
282 + TranslateXY(prob=0.72, magnitude=0.50)
283 + ToTensor()
284 +)
285 + Compose(
286 + Pad(padding=4, fill=0, padding_mode=constant)
287 + RandomCrop(size=(32, 32), padding=None)
288 + RandomHorizontalFlip(p=0.5)
289 + AutoContrast(prob=0.53, magnitude=0.95)
290 + Contrast(prob=0.08, magnitude=0.43)
291 + ToTensor()
292 +)
293 + Compose(
294 + Pad(padding=4, fill=0, padding_mode=constant)
295 + RandomCrop(size=(32, 32), padding=None)
296 + RandomHorizontalFlip(p=0.5)
297 + Cutout(prob=0.38, magnitude=0.47)
298 + Cutout(prob=0.98, magnitude=0.07)
299 + ToTensor()
300 +)
301 + Compose(
302 + Pad(padding=4, fill=0, padding_mode=constant)
303 + RandomCrop(size=(32, 32), padding=None)
304 + RandomHorizontalFlip(p=0.5)
305 + Brightness(prob=0.13, magnitude=0.42)
306 + Brightness(prob=0.88, magnitude=0.09)
307 + ToTensor()
308 +)
309 +)
310 +
311 +[+] Start training
312 +
313 +[+] Use 1 GPUs
314 +
315 +[+] Using GPU: Tesla P100-PCIE-16GB
316 +/usr/local/lib/python3.6/dist-packages/torch/nn/functional.py:2457: UserWarning: nn.functional.upsample is deprecated. Use nn.functional.interpolate instead.
317 + warnings.warn("nn.functional.upsample is deprecated. Use nn.functional.interpolate instead.")
318 +/usr/local/lib/python3.6/dist-packages/torch/nn/functional.py:2539: UserWarning: Default upsampling behavior when mode=bilinear is changed to align_corners=False since 0.4.0. Please specify align_corners=True if the old behavior is desired. See the documentation of nn.Upsample for details.
319 + "See the documentation of nn.Upsample for details.".format(mode))
320 +
321 +[+] Training step: 0/5000 Training epoch: 0/31 Elapsed time: 0.54min Learning rate: 9.999283e-05
322 + Acc@1 : 0.000%
323 + Acc@5 : 0.000%
324 + Loss : 7.482730388641357
325 + FW Time : 27.781ms
326 + BW Time : 11.731ms
327 +
328 +[+] Valid results
329 + Acc@1 : 100.000%
330 + Acc@5 : 100.000%
331 + Loss : 0.0
332 +
333 +[+] Model saved
334 +
335 +[+] Training step: 500/5000 Training epoch: 0/31 Elapsed time: 6.91min Learning rate: 9.647145853624023e-05
336 + Acc@1 : 100.000%
337 + Acc@5 : 100.000%
338 + Loss : 7.505388930439949e-05
339 + FW Time : 30.267ms
340 + BW Time : 30.651ms
341 +
342 +[+] Valid results
343 + Acc@1 : 100.000%
344 + Acc@5 : 100.000%
345 + Loss : 0.0
346 +
347 +[+] Model saved
348 +
349 +[+] Training step: 1000/5000 Training epoch: 0/31 Elapsed time: 10.06min Learning rate: 9.307409653381692e-05
350 + Acc@1 : 100.000%
351 + Acc@5 : 100.000%
352 + Loss : 0.00013676658272743225
353 + FW Time : 40.501ms
354 + BW Time : 34.972ms
355 +
356 +[+] Valid results
357 + Acc@1 : 100.000%
358 + Acc@5 : 100.000%
359 + Loss : 0.0
360 +
361 +[+] Model saved
362 +
363 +[+] Training step: 1500/5000 Training epoch: 0/31 Elapsed time: 13.08min Learning rate: 8.979637684582141e-05
364 + Acc@1 : 100.000%
365 + Acc@5 : 100.000%
366 + Loss : 4.213303327560425e-06
367 + FW Time : 35.585ms
368 + BW Time : 30.589ms
369 +
370 +[+] Valid results
371 + Acc@1 : 100.000%
372 + Acc@5 : 100.000%
373 + Loss : 0.0
374 +
375 +[+] Model saved
376 +
377 +[+] Training step: 2000/5000 Training epoch: 0/31 Elapsed time: 16.21min Learning rate: 8.663408611983758e-05
378 + Acc@1 : 100.000%
379 + Acc@5 : 100.000%
380 + Loss : 5.885958671569824e-07
381 + FW Time : 29.106ms
382 + BW Time : 28.138ms
383 +
384 +[+] Valid results
385 + Acc@1 : 100.000%
386 + Acc@5 : 100.000%
387 + Loss : 0.0
388 +
389 +[+] Model saved
390 +
391 +[+] Training step: 2500/5000 Training epoch: 0/31 Elapsed time: 19.30min Learning rate: 8.358315938187757e-05
392 + Acc@1 : 100.000%
393 + Acc@5 : 100.000%
394 + Loss : 2.130866050720215e-06
395 + FW Time : 30.433ms
396 + BW Time : 32.247ms
397 +
398 +[+] Valid results
399 + Acc@1 : 100.000%
400 + Acc@5 : 100.000%
401 + Loss : 0.0
402 +
403 +[+] Model saved
404 +
405 +[+] Training step: 3000/5000 Training epoch: 0/31 Elapsed time: 22.28min Learning rate: 8.063967481105164e-05
406 + Acc@1 : 100.000%
407 + Acc@5 : 100.000%
408 + Loss : 0.0
409 + FW Time : 16.860ms
410 + BW Time : 13.823ms
411 +
412 +[+] Valid results
413 + Acc@1 : 100.000%
414 + Acc@5 : 100.000%
415 + Loss : 0.0
416 +
417 +[+] Model saved
418 +
419 +[+] Training step: 3500/5000 Training epoch: 0/31 Elapsed time: 25.33min Learning rate: 7.779984869825457e-05
420 + Acc@1 : 100.000%
421 + Acc@5 : 100.000%
422 + Loss : 0.0
423 + FW Time : 28.149ms
424 + BW Time : 25.686ms
425 +
426 +[+] Valid results
427 + Acc@1 : 100.000%
428 + Acc@5 : 100.000%
429 + Loss : 0.0
430 +
431 +[+] Model saved
432 +
433 +[+] Training step: 4000/5000 Training epoch: 0/31 Elapsed time: 28.45min Learning rate: 7.506003058238717e-05
434 + Acc@1 : 100.000%
435 + Acc@5 : 100.000%
436 + Loss : 0.0
437 + FW Time : 38.845ms
438 + BW Time : 32.935ms
439 +
440 +[+] Valid results
441 + Acc@1 : 100.000%
442 + Acc@5 : 100.000%
443 + Loss : 0.0
444 +
445 +[+] Model saved
446 +
447 +[+] Training step: 4500/5000 Training epoch: 0/31 Elapsed time: 31.40min Learning rate: 7.24166985578635e-05
448 + Acc@1 : 100.000%
449 + Acc@5 : 100.000%
450 + Loss : 0.0
451 + FW Time : 34.974ms
452 + BW Time : 27.569ms
453 +
454 +[+] Valid results
455 + Acc@1 : 100.000%
456 + Acc@5 : 100.000%
457 + Loss : 0.0
458 +
459 +[+] Model saved
...\ No newline at end of file ...\ No newline at end of file
1 +max step = 3000
2 +step = 500
3 +lr = 0.0001
4 +batch_size = 64
5 +
6 +
7 +DEFALUT_CANDIDATES = [
8 + ShearXY,
9 + TranslateXY,
10 + # Rotate,
11 + # AutoContrast, #߰
12 + # Invert,
13 + Equalize,
14 + Solarize,
15 + Posterize,
16 + Contrast,
17 + # Color,
18 + Brightness,
19 + Sharpness,
20 + Cutout,
21 +# SamplePairing,
22 +]
...\ No newline at end of file ...\ No newline at end of file