2020-1-capstone-design2 / 2016104167

Go to a project
Toggle navigation Toggle navigation pinning
조현아
032d33ee 0 parents

flair2seg & filter normal

Expand all
Inline Side-by-side
Showing 64 changed files with 3692 additions and 0 deletions
<?xml version="1.0" encoding="UTF-8"?>
<module type="PYTHON_MODULE" version="4">
<component name="NewModuleRootManager">
<content url="file://$MODULE_DIR$" />
<orderEntry type="inheritedJdk" />
<orderEntry type="sourceFolder" forTests="false" />
</component>
<component name="TestRunnerService">
<option name="PROJECT_TEST_RUNNER" value="Unittests" />
</component>
</module>
\ No newline at end of file
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="Encoding" addBOMForNewFiles="with NO BOM" />
</project>
\ No newline at end of file
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="JavaScriptSettings">
<option name="languageLevel" value="ES6" />
</component>
<component name="ProjectRootManager" version="2" project-jdk-name="Python 3.7 (code) (3)" project-jdk-type="Python SDK" />
</project>
\ No newline at end of file
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="ProjectModuleManager">
<modules>
<module fileurl="file://$PROJECT_DIR$/.idea/code.iml" filepath="$PROJECT_DIR$/.idea/code.iml" />
</modules>
</component>
</project>
\ No newline at end of file
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="ChangeListManager">
<list default="true" id="38bbd81d-cf79-4a3f-8979-7c3ceb27bc32" name="Default Changelist" comment="" />
<option name="EXCLUDED_CONVERTED_TO_IGNORED" value="true" />
<option name="SHOW_DIALOG" value="false" />
<option name="HIGHLIGHT_CONFLICTS" value="true" />
<option name="HIGHLIGHT_NON_ACTIVE_CHANGELIST" value="false" />
<option name="LAST_RESOLUTION" value="IGNORE" />
</component>
<component name="ProjectFrameBounds">
<option name="x" value="-9" />
<option name="width" value="1825" />
<option name="height" value="1039" />
</component>
<component name="ProjectView">
<navigator proportions="" version="1">
<foldersAlwaysOnTop value="true" />
</navigator>
<panes>
<pane id="Scope" />
<pane id="ProjectPane" />
</panes>
</component>
<component name="PropertiesComponent">
<property name="WebServerToolWindowFactoryState" value="false" />
<property name="nodejs_interpreter_path.stuck_in_default_project" value="undefined stuck path" />
<property name="nodejs_npm_path_reset_for_default_project" value="true" />
<property name="settings.editor.selected.configurable" value="com.jetbrains.python.configuration.PyActiveSdkModuleConfigurable" />
</component>
<component name="RunDashboard">
<option name="ruleStates">
<list>
<RuleState>
<option name="name" value="ConfigurationTypeDashboardGroupingRule" />
</RuleState>
<RuleState>
<option name="name" value="StatusDashboardGroupingRule" />
</RuleState>
</list>
</option>
</component>
<component name="SvnConfiguration">
<configuration />
</component>
<component name="TaskManager">
<task active="true" id="Default" summary="Default task">
<changelist id="38bbd81d-cf79-4a3f-8979-7c3ceb27bc32" name="Default Changelist" comment="" />
<created>1584871834688</created>
<option name="number" value="Default" />
<option name="presentableId" value="Default" />
<updated>1584871834688</updated>
<workItem from="1584871840396" duration="72000" />
</task>
<servers />
</component>
<component name="TimeTrackingManager">
<option name="totallyTimeSpent" value="72000" />
</component>
<component name="ToolWindowManager">
<frame x="-7" y="0" width="1460" height="831" extended-state="0" />
<layout>
<window_info id="Favorites" side_tool="true" />
<window_info active="true" content_ui="combo" id="Project" order="0" visible="true" weight="0.25" />
<window_info id="Structure" order="1" side_tool="true" weight="0.25" />
<window_info anchor="bottom" id="Docker" show_stripe_button="false" />
<window_info anchor="bottom" id="Database Changes" />
<window_info anchor="bottom" id="Version Control" />
<window_info anchor="bottom" id="Python Console" />
<window_info anchor="bottom" id="Terminal" />
<window_info anchor="bottom" id="Event Log" side_tool="true" />
<window_info anchor="bottom" id="Message" order="0" />
<window_info anchor="bottom" id="Find" order="1" />
<window_info anchor="bottom" id="Run" order="2" />
<window_info anchor="bottom" id="Debug" order="3" weight="0.4" />
<window_info anchor="bottom" id="Cvs" order="4" weight="0.25" />
<window_info anchor="bottom" id="Inspection" order="5" weight="0.4" />
<window_info anchor="bottom" id="TODO" order="6" />
<window_info anchor="right" id="SciView" />
<window_info anchor="right" id="Database" />
<window_info anchor="right" id="Commander" internal_type="SLIDING" order="0" type="SLIDING" weight="0.4" />
<window_info anchor="right" id="Ant Build" order="1" weight="0.25" />
<window_info anchor="right" content_ui="combo" id="Hierarchy" order="2" weight="0.25" />
</layout>
</component>
<component name="TypeScriptGeneratedFilesManager">
<option name="version" value="1" />
</component>
</project>
\ No newline at end of file
{
"cells": [
{
"cell_type": "code",
"execution_count": 1,
"metadata": {},
"outputs": [],
"source": [
"import pandas as pd"
]
},
{
"cell_type": "code",
"execution_count": 3,
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>id</th>\n",
" <th>Label</th>\n",
" <th>Subject</th>\n",
" <th>Date</th>\n",
" <th>Gender</th>\n",
" <th>Age</th>\n",
" <th>mmse</th>\n",
" <th>ageAtEntry</th>\n",
" <th>cdr</th>\n",
" <th>commun</th>\n",
" <th>...</th>\n",
" <th>memory</th>\n",
" <th>orient</th>\n",
" <th>perscare</th>\n",
" <th>apoe</th>\n",
" <th>sumbox</th>\n",
" <th>acsparnt</th>\n",
" <th>height</th>\n",
" <th>weight</th>\n",
" <th>primStudy</th>\n",
" <th>acsStudy</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>/@WEBAPP/images/r.gif</td>\n",
" <td>OAS30001_ClinicalData_d3025</td>\n",
" <td>OAS30001</td>\n",
" <td>NaN</td>\n",
" <td>female</td>\n",
" <td>NaN</td>\n",
" <td>30.0</td>\n",
" <td>65.149895</td>\n",
" <td>0.0</td>\n",
" <td>0.0</td>\n",
" <td>...</td>\n",
" <td>0.0</td>\n",
" <td>0.0</td>\n",
" <td>0.0</td>\n",
" <td>23.0</td>\n",
" <td>0.0</td>\n",
" <td>NaN</td>\n",
" <td>64.0</td>\n",
" <td>180.0</td>\n",
" <td>NaN</td>\n",
" <td>NaN</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1</th>\n",
" <td>/@WEBAPP/images/r.gif</td>\n",
" <td>OAS30001_ClinicalData_d3977</td>\n",
" <td>OAS30001</td>\n",
" <td>NaN</td>\n",
" <td>female</td>\n",
" <td>NaN</td>\n",
" <td>29.0</td>\n",
" <td>65.149895</td>\n",
" <td>0.0</td>\n",
" <td>0.0</td>\n",
" <td>...</td>\n",
" <td>0.0</td>\n",
" <td>0.0</td>\n",
" <td>0.0</td>\n",
" <td>23.0</td>\n",
" <td>0.0</td>\n",
" <td>NaN</td>\n",
" <td>NaN</td>\n",
" <td>NaN</td>\n",
" <td>NaN</td>\n",
" <td>NaN</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2</th>\n",
" <td>/@WEBAPP/images/r.gif</td>\n",
" <td>OAS30001_ClinicalData_d3332</td>\n",
" <td>OAS30001</td>\n",
" <td>NaN</td>\n",
" <td>female</td>\n",
" <td>NaN</td>\n",
" <td>30.0</td>\n",
" <td>65.149895</td>\n",
" <td>0.0</td>\n",
" <td>0.0</td>\n",
" <td>...</td>\n",
" <td>0.0</td>\n",
" <td>0.0</td>\n",
" <td>0.0</td>\n",
" <td>23.0</td>\n",
" <td>0.0</td>\n",
" <td>NaN</td>\n",
" <td>63.0</td>\n",
" <td>185.0</td>\n",
" <td>NaN</td>\n",
" <td>NaN</td>\n",
" </tr>\n",
" <tr>\n",
" <th>3</th>\n",
" <td>/@WEBAPP/images/r.gif</td>\n",
" <td>OAS30001_ClinicalData_d0000</td>\n",
" <td>OAS30001</td>\n",
" <td>NaN</td>\n",
" <td>female</td>\n",
" <td>NaN</td>\n",
" <td>28.0</td>\n",
" <td>65.149895</td>\n",
" <td>0.0</td>\n",
" <td>0.0</td>\n",
" <td>...</td>\n",
" <td>0.0</td>\n",
" <td>0.0</td>\n",
" <td>0.0</td>\n",
" <td>23.0</td>\n",
" <td>0.0</td>\n",
" <td>NaN</td>\n",
" <td>NaN</td>\n",
" <td>NaN</td>\n",
" <td>NaN</td>\n",
" <td>NaN</td>\n",
" </tr>\n",
" <tr>\n",
" <th>4</th>\n",
" <td>/@WEBAPP/images/r.gif</td>\n",
" <td>OAS30001_ClinicalData_d1456</td>\n",
" <td>OAS30001</td>\n",
" <td>NaN</td>\n",
" <td>female</td>\n",
" <td>NaN</td>\n",
" <td>30.0</td>\n",
" <td>65.149895</td>\n",
" <td>0.0</td>\n",
" <td>0.0</td>\n",
" <td>...</td>\n",
" <td>0.0</td>\n",
" <td>0.0</td>\n",
" <td>0.0</td>\n",
" <td>23.0</td>\n",
" <td>0.0</td>\n",
" <td>NaN</td>\n",
" <td>63.0</td>\n",
" <td>173.0</td>\n",
" <td>NaN</td>\n",
" <td>NaN</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"<p>5 rows × 27 columns</p>\n",
"</div>"
],
"text/plain": [
" id Label Subject Date Gender \\\n",
"0 /@WEBAPP/images/r.gif OAS30001_ClinicalData_d3025 OAS30001 NaN female \n",
"1 /@WEBAPP/images/r.gif OAS30001_ClinicalData_d3977 OAS30001 NaN female \n",
"2 /@WEBAPP/images/r.gif OAS30001_ClinicalData_d3332 OAS30001 NaN female \n",
"3 /@WEBAPP/images/r.gif OAS30001_ClinicalData_d0000 OAS30001 NaN female \n",
"4 /@WEBAPP/images/r.gif OAS30001_ClinicalData_d1456 OAS30001 NaN female \n",
"\n",
" Age mmse ageAtEntry cdr commun ... memory orient perscare apoe \\\n",
"0 NaN 30.0 65.149895 0.0 0.0 ... 0.0 0.0 0.0 23.0 \n",
"1 NaN 29.0 65.149895 0.0 0.0 ... 0.0 0.0 0.0 23.0 \n",
"2 NaN 30.0 65.149895 0.0 0.0 ... 0.0 0.0 0.0 23.0 \n",
"3 NaN 28.0 65.149895 0.0 0.0 ... 0.0 0.0 0.0 23.0 \n",
"4 NaN 30.0 65.149895 0.0 0.0 ... 0.0 0.0 0.0 23.0 \n",
"\n",
" sumbox acsparnt height weight primStudy acsStudy \n",
"0 0.0 NaN 64.0 180.0 NaN NaN \n",
"1 0.0 NaN NaN NaN NaN NaN \n",
"2 0.0 NaN 63.0 185.0 NaN NaN \n",
"3 0.0 NaN NaN NaN NaN NaN \n",
"4 0.0 NaN 63.0 173.0 NaN NaN \n",
"\n",
"[5 rows x 27 columns]"
]
},
"execution_count": 3,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"all_data = pd.read_csv(\"..\\data\\ADRC clinical data_all.csv\")\n",
"\n",
"all_data.head()"
]
},
{
"cell_type": "code",
"execution_count": 18,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Subject\n",
"OAS30001 0.0\n",
"OAS30002 0.0\n",
"OAS30003 0.0\n",
"OAS30004 0.0\n",
"OAS30005 0.0\n",
" ... \n",
"OAS31168 0.0\n",
"OAS31169 3.0\n",
"OAS31170 2.0\n",
"OAS31171 2.0\n",
"OAS31172 0.0\n",
"Name: cdr, Length: 1098, dtype: float64\n"
]
}
],
"source": [
"ad = all_data.groupby(['Subject'])['cdr'].max()\n",
"print(ad)"
]
},
{
"cell_type": "code",
"execution_count": 21,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"'OAS30001'"
]
},
"execution_count": 21,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"ad.index[0]"
]
},
{
"cell_type": "code",
"execution_count": 22,
"metadata": {},
"outputs": [
{
"ename": "SyntaxError",
"evalue": "unexpected EOF while parsing (<ipython-input-22-b8a078b72aca>, line 5)",
"output_type": "error",
"traceback": [
"\u001b[1;36m File \u001b[1;32m\"<ipython-input-22-b8a078b72aca>\"\u001b[1;36m, line \u001b[1;32m5\u001b[0m\n\u001b[1;33m #print(filtered)\u001b[0m\n\u001b[1;37m ^\u001b[0m\n\u001b[1;31mSyntaxError\u001b[0m\u001b[1;31m:\u001b[0m unexpected EOF while parsing\n"
]
}
],
"source": [
"filtered = []\n",
"for i, val in enumerate(ad):\n",
" if ad[i] == 0:\n",
" filtered.append(ad.index[i])\n",
"#print(filtered)"
]
},
{
"cell_type": "code",
"execution_count": 23,
"metadata": {},
"outputs": [],
"source": [
"df_filtered = pd.DataFrame(filtered)\n",
"df_filtered.to_csv('..\\data\\ADRC clinical data_normal.csv')"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "ML",
"language": "python",
"name": "ml"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.7.4"
}
},
"nbformat": 4,
"nbformat_minor": 2
}
This diff could not be displayed because it is too large.
# Byte-compiled / optimized / DLL files
__pycache__/
*.py[cod]
*$py.class
# C extensions
*.so
# Distribution / packaging
.Python
build/
develop-eggs/
dist/
downloads/
eggs/
.eggs/
lib/
lib64/
parts/
sdist/
var/
wheels/
*.egg-info/
.installed.cfg
*.egg
MANIFEST
# PyInstaller
# Usually these files are written by a python script from a template
# before PyInstaller builds the exe, so as to inject date/other infos into it.
*.manifest
*.spec
# Installer logs
pip-log.txt
pip-delete-this-directory.txt
# Unit test / coverage reports
htmlcov/
.tox/
.coverage
.coverage.*
.cache
nosetests.xml
coverage.xml
*.cover
.hypothesis/
.pytest_cache/
# Translations
*.mo
*.pot
# Django stuff:
*.log
local_settings.py
db.sqlite3
# Flask stuff:
instance/
.webassets-cache
# Scrapy stuff:
.scrapy
# Sphinx documentation
docs/_build/
# PyBuilder
target/
# Jupyter Notebook
.ipynb_checkpoints
# pyenv
.python-version
# celery beat schedule file
celerybeat-schedule
# SageMath parsed files
*.sage.py
# Environments
.env
.venv
env/
venv/
ENV/
env.bak/
venv.bak/
# Spyder project settings
.spyderproject
.spyproject
# Rope project settings
.ropeproject
# mkdocs documentation
/site
# mypy
.mypy_cache/
This diff could not be displayed because it is too large.
"""
Reference :
- https://github.com/hysts/pytorch_image_classification/blob/master/augmentations/mixup.py
- https://github.com/tensorflow/tpu/blob/master/models/official/efficientnet/imagenet_input.py#L120
"""
import numpy as np
import torch
from FastAutoAugment.metrics import CrossEntropyLabelSmooth
def mixup(data, targets, alpha):
indices = torch.randperm(data.size(0))
shuffled_data = data[indices]
shuffled_targets = targets[indices]
lam = np.random.beta(alpha, alpha)
lam = max(lam, 1. - lam)
assert 0.0 <= lam <= 1.0, lam
data = data * lam + shuffled_data * (1 - lam)
return data, targets, shuffled_targets, lam
class CrossEntropyMixUpLabelSmooth(torch.nn.Module):
def __init__(self, num_classes, epsilon, reduction='mean'):
super(CrossEntropyMixUpLabelSmooth, self).__init__()
self.ce = CrossEntropyLabelSmooth(num_classes, epsilon, reduction=reduction)
def forward(self, input, target1, target2, lam): # pylint: disable=redefined-builtin
return lam * self.ce(input, target1) + (1 - lam) * self.ce(input, target2)
# code in this file is adpated from rpmcruz/autoaugment
# https://github.com/rpmcruz/autoaugment/blob/master/transformations.py
import random
import PIL, PIL.ImageOps, PIL.ImageEnhance, PIL.ImageDraw
import numpy as np
import torch
from torchvision.transforms.transforms import Compose
random_mirror = True
def ShearX(img, v): # [-0.3, 0.3]
assert -0.3 <= v <= 0.3
if random_mirror and random.random() > 0.5:
v = -v
return img.transform(img.size, PIL.Image.AFFINE, (1, v, 0, 0, 1, 0))
def ShearY(img, v): # [-0.3, 0.3]
assert -0.3 <= v <= 0.3
if random_mirror and random.random() > 0.5:
v = -v
return img.transform(img.size, PIL.Image.AFFINE, (1, 0, 0, v, 1, 0))
def TranslateX(img, v): # [-150, 150] => percentage: [-0.45, 0.45]
assert -0.45 <= v <= 0.45
if random_mirror and random.random() > 0.5:
v = -v
v = v * img.size[0]
return img.transform(img.size, PIL.Image.AFFINE, (1, 0, v, 0, 1, 0))
def TranslateY(img, v): # [-150, 150] => percentage: [-0.45, 0.45]
assert -0.45 <= v <= 0.45
if random_mirror and random.random() > 0.5:
v = -v
v = v * img.size[1]
return img.transform(img.size, PIL.Image.AFFINE, (1, 0, 0, 0, 1, v))
def TranslateXAbs(img, v): # [-150, 150] => percentage: [-0.45, 0.45]
assert 0 <= v <= 10
if random.random() > 0.5:
v = -v
return img.transform(img.size, PIL.Image.AFFINE, (1, 0, v, 0, 1, 0))
def TranslateYAbs(img, v): # [-150, 150] => percentage: [-0.45, 0.45]
assert 0 <= v <= 10
if random.random() > 0.5:
v = -v
return img.transform(img.size, PIL.Image.AFFINE, (1, 0, 0, 0, 1, v))
def Rotate(img, v): # [-30, 30]
assert -30 <= v <= 30
if random_mirror and random.random() > 0.5:
v = -v
return img.rotate(v)
def AutoContrast(img, _):
return PIL.ImageOps.autocontrast(img)
def Invert(img, _):
return PIL.ImageOps.invert(img)
def Equalize(img, _):
return PIL.ImageOps.equalize(img)
def Flip(img, _): # not from the paper
return PIL.ImageOps.mirror(img)
def Solarize(img, v): # [0, 256]
assert 0 <= v <= 256
return PIL.ImageOps.solarize(img, v)
def Posterize(img, v): # [4, 8]
assert 4 <= v <= 8
v = int(v)
return PIL.ImageOps.posterize(img, v)
def Posterize2(img, v): # [0, 4]
assert 0 <= v <= 4
v = int(v)
return PIL.ImageOps.posterize(img, v)
def Contrast(img, v): # [0.1,1.9]
assert 0.1 <= v <= 1.9
return PIL.ImageEnhance.Contrast(img).enhance(v)
def Color(img, v): # [0.1,1.9]
assert 0.1 <= v <= 1.9
return PIL.ImageEnhance.Color(img).enhance(v)
def Brightness(img, v): # [0.1,1.9]
assert 0.1 <= v <= 1.9
return PIL.ImageEnhance.Brightness(img).enhance(v)
def Sharpness(img, v): # [0.1,1.9]
assert 0.1 <= v <= 1.9
return PIL.ImageEnhance.Sharpness(img).enhance(v)
def Cutout(img, v): # [0, 60] => percentage: [0, 0.2]
assert 0.0 <= v <= 0.2
if v <= 0.:
return img
v = v * img.size[0]
return CutoutAbs(img, v)
def CutoutAbs(img, v): # [0, 60] => percentage: [0, 0.2]
# assert 0 <= v <= 20
if v < 0:
return img
w, h = img.size
x0 = np.random.uniform(w)
y0 = np.random.uniform(h)
x0 = int(max(0, x0 - v / 2.))
y0 = int(max(0, y0 - v / 2.))
x1 = min(w, x0 + v)
y1 = min(h, y0 + v)
xy = (x0, y0, x1, y1)
color = (125, 123, 114)
# color = (0, 0, 0)
img = img.copy()
PIL.ImageDraw.Draw(img).rectangle(xy, color)
return img
def SamplePairing(imgs): # [0, 0.4]
def f(img1, v):
i = np.random.choice(len(imgs))
img2 = PIL.Image.fromarray(imgs[i])
return PIL.Image.blend(img1, img2, v)
return f
def augment_list(for_autoaug=True): # 16 oeprations and their ranges
l = [
(ShearX, -0.3, 0.3), # 0
(ShearY, -0.3, 0.3), # 1
(TranslateX, -0.45, 0.45), # 2
(TranslateY, -0.45, 0.45), # 3
(Rotate, -30, 30), # 4
(AutoContrast, 0, 1), # 5
(Invert, 0, 1), # 6
(Equalize, 0, 1), # 7
(Solarize, 0, 256), # 8
(Posterize, 4, 8), # 9
(Contrast, 0.1, 1.9), # 10
(Color, 0.1, 1.9), # 11
(Brightness, 0.1, 1.9), # 12
(Sharpness, 0.1, 1.9), # 13
(Cutout, 0, 0.2), # 14
# (SamplePairing(imgs), 0, 0.4), # 15
]
if for_autoaug:
l += [
(CutoutAbs, 0, 20), # compatible with auto-augment
(Posterize2, 0, 4), # 9
(TranslateXAbs, 0, 10), # 9
(TranslateYAbs, 0, 10), # 9
]
return l
augment_dict = {fn.__name__: (fn, v1, v2) for fn, v1, v2 in augment_list()}
def get_augment(name):
return augment_dict[name]
def apply_augment(img, name, level):
augment_fn, low, high = get_augment(name)
return augment_fn(img.copy(), level * (high - low) + low)
class Lighting(object):
"""Lighting noise(AlexNet - style PCA - based noise)"""
def __init__(self, alphastd, eigval, eigvec):
self.alphastd = alphastd
self.eigval = torch.Tensor(eigval)
self.eigvec = torch.Tensor(eigvec)
def __call__(self, img):
if self.alphastd == 0:
return img
alpha = img.new().resize_(3).normal_(0, self.alphastd)
rgb = self.eigvec.type_as(img).clone() \
.mul(alpha.view(1, 3).expand(3, 3)) \
.mul(self.eigval.view(1, 3).expand(3, 3)) \
.sum(1).squeeze()
return img.add(rgb.view(3, 1, 1).expand_as(img))
import copy
import logging
import warnings
formatter = logging.Formatter('[%(asctime)s] [%(name)s] [%(levelname)s] %(message)s')
warnings.filterwarnings("ignore", "(Possibly )?corrupt EXIF data", UserWarning)
warnings.filterwarnings("ignore", "DeprecationWarning: 'saved_variables' is deprecated", UserWarning)
def get_logger(name, level=logging.DEBUG):
logger = logging.getLogger(name)
logger.handlers.clear()
logger.setLevel(level)
ch = logging.StreamHandler()
ch.setLevel(level)
ch.setFormatter(formatter)
logger.addHandler(ch)
return logger
def add_filehandler(logger, filepath, level=logging.DEBUG):
fh = logging.FileHandler(filepath)
fh.setLevel(level)
fh.setFormatter(formatter)
logger.addHandler(fh)
class EMA:
def __init__(self, mu):
self.mu = mu
self.shadow = {}
def state_dict(self):
return copy.deepcopy(self.shadow)
def __len__(self):
return len(self.shadow)
def __call__(self, module, step=None):
if step is None:
mu = self.mu
else:
# see : https://www.tensorflow.org/versions/r1.15/api_docs/python/tf/train/ExponentialMovingAverage?hl=PL
mu = min(self.mu, (1. + step) / (10 + step))
for name, x in module.state_dict().items():
if name in self.shadow:
new_average = (1.0 - mu) * x + mu * self.shadow[name]
self.shadow[name] = new_average.clone()
else:
self.shadow[name] = x.clone()
This diff is collapsed. Click to expand it.
from __future__ import print_function
import os
import shutil
import torch
ARCHIVE_DICT = {
'train': {
'url': 'http://www.image-net.org/challenges/LSVRC/2012/nnoupb/ILSVRC2012_img_train.tar',
'md5': '1d675b47d978889d74fa0da5fadfb00e',
},
'val': {
'url': 'http://www.image-net.org/challenges/LSVRC/2012/nnoupb/ILSVRC2012_img_val.tar',
'md5': '29b22e2961454d5413ddabcf34fc5622',
},
'devkit': {
'url': 'http://www.image-net.org/challenges/LSVRC/2012/nnoupb/ILSVRC2012_devkit_t12.tar.gz',
'md5': 'fa75699e90414af021442c21a62c3abf',
}
}
import torchvision
from torchvision.datasets.utils import check_integrity, download_url
# copy ILSVRC/ImageSets/CLS-LOC/train_cls.txt to ./root/
# to skip os walk (it's too slow) using ILSVRC/ImageSets/CLS-LOC/train_cls.txt file
class ImageNet(torchvision.datasets.ImageFolder):
"""`ImageNet <http://image-net.org/>`_ 2012 Classification Dataset.
Args:
root (string): Root directory of the ImageNet Dataset.
split (string, optional): The dataset split, supports ``train``, or ``val``.
download (bool, optional): If true, downloads the dataset from the internet and
puts it in root directory. If dataset is already downloaded, it is not
downloaded again.
transform (callable, optional): A function/transform that takes in an PIL image
and returns a transformed version. E.g, ``transforms.RandomCrop``
target_transform (callable, optional): A function/transform that takes in the
target and transforms it.
loader (callable, optional): A function to load an image given its path.
Attributes:
classes (list): List of the class names.
class_to_idx (dict): Dict with items (class_name, class_index).
wnids (list): List of the WordNet IDs.
wnid_to_idx (dict): Dict with items (wordnet_id, class_index).
imgs (list): List of (image path, class_index) tuples
targets (list): The class_index value for each image in the dataset
"""
def __init__(self, root, split='train', download=False, **kwargs):
root = self.root = os.path.expanduser(root)
self.split = self._verify_split(split)
if download:
self.download()
wnid_to_classes = self._load_meta_file()[0]
# to skip os walk (it's too slow) using ILSVRC/ImageSets/CLS-LOC/train_cls.txt file
listfile = os.path.join(root, 'train_cls.txt')
if split == 'train' and os.path.exists(listfile):
torchvision.datasets.VisionDataset.__init__(self, root, **kwargs)
with open(listfile, 'r') as f:
datalist = [
line.strip().split(' ')[0]
for line in f.readlines()
if line.strip()
]
classes = list(set([line.split('/')[0] for line in datalist]))
classes.sort()
class_to_idx = {classes[i]: i for i in range(len(classes))}
samples = [
(os.path.join(self.split_folder, line + '.JPEG'), class_to_idx[line.split('/')[0]])
for line in datalist
]
self.loader = torchvision.datasets.folder.default_loader
self.extensions = torchvision.datasets.folder.IMG_EXTENSIONS
self.classes = classes
self.class_to_idx = class_to_idx
self.samples = samples
self.targets = [s[1] for s in samples]
self.imgs = self.samples
else:
super(ImageNet, self).__init__(self.split_folder, **kwargs)
self.root = root
idcs = [idx for _, idx in self.imgs]
self.wnids = self.classes
self.wnid_to_idx = {wnid: idx for idx, wnid in zip(idcs, self.wnids)}
self.classes = [wnid_to_classes[wnid] for wnid in self.wnids]
self.class_to_idx = {cls: idx
for clss, idx in zip(self.classes, idcs)
for cls in clss}
def download(self):
if not check_integrity(self.meta_file):
tmpdir = os.path.join(self.root, 'tmp')
archive_dict = ARCHIVE_DICT['devkit']
download_and_extract_tar(archive_dict['url'], self.root,
extract_root=tmpdir,
md5=archive_dict['md5'])
devkit_folder = _splitexts(os.path.basename(archive_dict['url']))[0]
meta = parse_devkit(os.path.join(tmpdir, devkit_folder))
self._save_meta_file(*meta)
shutil.rmtree(tmpdir)
if not os.path.isdir(self.split_folder):
archive_dict = ARCHIVE_DICT[self.split]
download_and_extract_tar(archive_dict['url'], self.root,
extract_root=self.split_folder,
md5=archive_dict['md5'])
if self.split == 'train':
prepare_train_folder(self.split_folder)
elif self.split == 'val':
val_wnids = self._load_meta_file()[1]
prepare_val_folder(self.split_folder, val_wnids)
else:
msg = ("You set download=True, but a folder '{}' already exist in "
"the root directory. If you want to re-download or re-extract the "
"archive, delete the folder.")
print(msg.format(self.split))
@property
def meta_file(self):
return os.path.join(self.root, 'meta.bin')
def _load_meta_file(self):
if check_integrity(self.meta_file):
return torch.load(self.meta_file)
raise RuntimeError("Meta file not found or corrupted.",
"You can use download=True to create it.")
def _save_meta_file(self, wnid_to_class, val_wnids):
torch.save((wnid_to_class, val_wnids), self.meta_file)
def _verify_split(self, split):
if split not in self.valid_splits:
msg = "Unknown split {} .".format(split)
msg += "Valid splits are {{}}.".format(", ".join(self.valid_splits))
raise ValueError(msg)
return split
@property
def valid_splits(self):
return 'train', 'val'
@property
def split_folder(self):
return os.path.join(self.root, self.split)
def extra_repr(self):
return "Split: {split}".format(**self.__dict__)
def extract_tar(src, dest=None, gzip=None, delete=False):
import tarfile
if dest is None:
dest = os.path.dirname(src)
if gzip is None:
gzip = src.lower().endswith('.gz')
mode = 'r:gz' if gzip else 'r'
with tarfile.open(src, mode) as tarfh:
tarfh.extractall(path=dest)
if delete:
os.remove(src)
def download_and_extract_tar(url, download_root, extract_root=None, filename=None,
md5=None, **kwargs):
download_root = os.path.expanduser(download_root)
if extract_root is None:
extract_root = download_root
if filename is None:
filename = os.path.basename(url)
if not check_integrity(os.path.join(download_root, filename), md5):
download_url(url, download_root, filename=filename, md5=md5)
extract_tar(os.path.join(download_root, filename), extract_root, **kwargs)
def parse_devkit(root):
idx_to_wnid, wnid_to_classes = parse_meta(root)
val_idcs = parse_val_groundtruth(root)
val_wnids = [idx_to_wnid[idx] for idx in val_idcs]
return wnid_to_classes, val_wnids
def parse_meta(devkit_root, path='data', filename='meta.mat'):
import scipy.io as sio
metafile = os.path.join(devkit_root, path, filename)
meta = sio.loadmat(metafile, squeeze_me=True)['synsets']
nums_children = list(zip(*meta))[4]
meta = [meta[idx] for idx, num_children in enumerate(nums_children)
if num_children == 0]
idcs, wnids, classes = list(zip(*meta))[:3]
classes = [tuple(clss.split(', ')) for clss in classes]
idx_to_wnid = {idx: wnid for idx, wnid in zip(idcs, wnids)}
wnid_to_classes = {wnid: clss for wnid, clss in zip(wnids, classes)}
return idx_to_wnid, wnid_to_classes
def parse_val_groundtruth(devkit_root, path='data',
filename='ILSVRC2012_validation_ground_truth.txt'):
with open(os.path.join(devkit_root, path, filename), 'r') as txtfh:
val_idcs = txtfh.readlines()
return [int(val_idx) for val_idx in val_idcs]
def prepare_train_folder(folder):
for archive in [os.path.join(folder, archive) for archive in os.listdir(folder)]:
extract_tar(archive, os.path.splitext(archive)[0], delete=True)
def prepare_val_folder(folder, wnids):
img_files = sorted([os.path.join(folder, file) for file in os.listdir(folder)])
for wnid in set(wnids):
os.mkdir(os.path.join(folder, wnid))
for wnid, img_file in zip(wnids, img_files):
shutil.move(img_file, os.path.join(folder, wnid, os.path.basename(img_file)))
def _splitexts(root):
exts = []
ext = '.'
while ext:
root, ext = os.path.splitext(root)
exts.append(ext)
return root, ''.join(reversed(exts))
import torch
from theconf import Config as C
def adjust_learning_rate_resnet(optimizer):
"""
Sets the learning rate to the initial LR decayed by 10 on every predefined epochs
Ref: AutoAugment
"""
if C.get()['epoch'] == 90:
return torch.optim.lr_scheduler.MultiStepLR(optimizer, [30, 60, 80])
elif C.get()['epoch'] == 270: # autoaugment
return torch.optim.lr_scheduler.MultiStepLR(optimizer, [90, 180, 240])
else:
raise ValueError('invalid epoch=%d for resnet scheduler' % C.get()['epoch'])
import copy
import torch
import numpy as np
from collections import defaultdict
from torch import nn
def accuracy(output, target, topk=(1,)):
"""Computes the precision@k for the specified values of k"""
maxk = max(topk)
batch_size = target.size(0)
_, pred = output.topk(maxk, 1, True, True)
pred = pred.t()
correct = pred.eq(target.view(1, -1).expand_as(pred))
res = []
for k in topk:
correct_k = correct[:k].view(-1).float().sum(0)
res.append(correct_k.mul_(1. / batch_size))
return res
class CrossEntropyLabelSmooth(torch.nn.Module):
def __init__(self, num_classes, epsilon, reduction='mean'):
super(CrossEntropyLabelSmooth, self).__init__()
self.num_classes = num_classes
self.epsilon = epsilon
self.reduction = reduction
self.logsoftmax = torch.nn.LogSoftmax(dim=1)
def forward(self, input, target): # pylint: disable=redefined-builtin
log_probs = self.logsoftmax(input)
targets = torch.zeros_like(log_probs).scatter_(1, target.unsqueeze(1), 1)
if self.epsilon > 0.0:
targets = (1 - self.epsilon) * targets + self.epsilon / self.num_classes
targets = targets.detach()
loss = (-targets * log_probs)
if self.reduction in ['avg', 'mean']:
loss = torch.mean(torch.sum(loss, dim=1))
elif self.reduction == 'sum':
loss = loss.sum()
return loss
class Accumulator:
def __init__(self):
self.metrics = defaultdict(lambda: 0.)
def add(self, key, value):
self.metrics[key] += value
def add_dict(self, dict):
for key, value in dict.items():
self.add(key, value)
def __getitem__(self, item):
return self.metrics[item]
def __setitem__(self, key, value):
self.metrics[key] = value
def get_dict(self):
return copy.deepcopy(dict(self.metrics))
def items(self):
return self.metrics.items()
def __str__(self):
return str(dict(self.metrics))
def __truediv__(self, other):
newone = Accumulator()
for key, value in self.items():
if isinstance(other, str):
if other != key:
newone[key] = value / self[other]
else:
newone[key] = value
else:
newone[key] = value / other
return newone
class SummaryWriterDummy:
def __init__(self, log_dir):
pass
def add_scalar(self, *args, **kwargs):
pass
import torch
from torch import nn
from torch.nn import DataParallel
from torch.nn.parallel import DistributedDataParallel
import torch.backends.cudnn as cudnn
# from torchvision import models
import numpy as np
from FastAutoAugment.networks.resnet import ResNet
from FastAutoAugment.networks.pyramidnet import PyramidNet
from FastAutoAugment.networks.shakeshake.shake_resnet import ShakeResNet
from FastAutoAugment.networks.wideresnet import WideResNet
from FastAutoAugment.networks.shakeshake.shake_resnext import ShakeResNeXt
from FastAutoAugment.networks.efficientnet_pytorch import EfficientNet, RoutingFn
from FastAutoAugment.tf_port.tpu_bn import TpuBatchNormalization
def get_model(conf, num_class=10, local_rank=-1):
name = conf['type']
if name == 'resnet50':
model = ResNet(dataset='imagenet', depth=50, num_classes=num_class, bottleneck=True)
elif name == 'resnet200':
model = ResNet(dataset='imagenet', depth=200, num_classes=num_class, bottleneck=True)
elif name == 'wresnet40_2':
model = WideResNet(40, 2, dropout_rate=0.0, num_classes=num_class)
elif name == 'wresnet28_10':
model = WideResNet(28, 10, dropout_rate=0.0, num_classes=num_class)
elif name == 'shakeshake26_2x32d':
model = ShakeResNet(26, 32, num_class)
elif name == 'shakeshake26_2x64d':
model = ShakeResNet(26, 64, num_class)
elif name == 'shakeshake26_2x96d':
model = ShakeResNet(26, 96, num_class)
elif name == 'shakeshake26_2x112d':
model = ShakeResNet(26, 112, num_class)
elif name == 'shakeshake26_2x96d_next':
model = ShakeResNeXt(26, 96, 4, num_class)
elif name == 'pyramid':
model = PyramidNet('cifar10', depth=conf['depth'], alpha=conf['alpha'], num_classes=num_class, bottleneck=conf['bottleneck'])
elif 'efficientnet' in name:
model = EfficientNet.from_name(name, condconv_num_expert=conf['condconv_num_expert'], norm_layer=None) # TpuBatchNormalization
if local_rank >= 0:
model = nn.SyncBatchNorm.convert_sync_batchnorm(model)
def kernel_initializer(module):
def get_fan_in_out(module):
num_input_fmaps = module.weight.size(1)
num_output_fmaps = module.weight.size(0)
receptive_field_size = 1
if module.weight.dim() > 2:
receptive_field_size = module.weight[0][0].numel()
fan_in = num_input_fmaps * receptive_field_size
fan_out = num_output_fmaps * receptive_field_size
return fan_in, fan_out
if isinstance(module, torch.nn.Conv2d):
# https://github.com/tensorflow/tpu/blob/master/models/official/efficientnet/efficientnet_model.py#L58
fan_in, fan_out = get_fan_in_out(module)
torch.nn.init.normal_(module.weight, mean=0.0, std=np.sqrt(2.0 / fan_out))
if module.bias is not None:
torch.nn.init.constant_(module.bias, val=0.)
elif isinstance(module, RoutingFn):
torch.nn.init.xavier_uniform_(module.weight)
torch.nn.init.constant_(module.bias, val=0.)
elif isinstance(module, torch.nn.Linear):
# https://github.com/tensorflow/tpu/blob/master/models/official/efficientnet/efficientnet_model.py#L82
fan_in, fan_out = get_fan_in_out(module)
delta = 1.0 / np.sqrt(fan_out)
torch.nn.init.uniform_(module.weight, a=-delta, b=delta)
if module.bias is not None:
torch.nn.init.constant_(module.bias, val=0.)
model.apply(kernel_initializer)
else:
raise NameError('no model named, %s' % name)
if local_rank >= 0:
device = torch.device('cuda', local_rank)
model = model.to(device)
model = DistributedDataParallel(model, device_ids=[local_rank], output_device=local_rank)
else:
model = model.cuda()
# model = DataParallel(model)
cudnn.benchmark = True
return model
def num_class(dataset):
return {
'cifar10': 10,
'reduced_cifar10': 10,
'cifar10.1': 10,
'cifar100': 100,
'svhn': 10,
'reduced_svhn': 10,
'imagenet': 1000,
'reduced_imagenet': 120,
}[dataset]
__version__ = "0.5.1"
from .model import EfficientNet, RoutingFn
from .utils import (
GlobalParams,
BlockArgs,
BlockDecoder,
efficientnet,
get_model_params,
)
\ No newline at end of file
import torch
import torch.nn as nn
import torch.nn.functional as F
from torch._six import container_abcs
from itertools import repeat
from functools import partial
from typing import Union, List, Tuple, Optional, Callable
import numpy as np
import math
def _ntuple(n):
def parse(x):
if isinstance(x, container_abcs.Iterable):
return x
return tuple(repeat(x, n))
return parse
_single = _ntuple(1)
_pair = _ntuple(2)
_triple = _ntuple(3)
_quadruple = _ntuple(4)
def _is_static_pad(kernel_size, stride=1, dilation=1, **_):
return stride == 1 and (dilation * (kernel_size - 1)) % 2 == 0
def _get_padding(kernel_size, stride=1, dilation=1, **_):
padding = ((stride - 1) + dilation * (kernel_size - 1)) // 2
return padding
def _calc_same_pad(i: int, k: int, s: int, d: int):
return max((math.ceil(i / s) - 1) * s + (k - 1) * d + 1 - i, 0)
def conv2d_same(
x, weight: torch.Tensor, bias: Optional[torch.Tensor] = None, stride: Tuple[int, int] = (1, 1),
padding: Tuple[int, int] = (0, 0), dilation: Tuple[int, int] = (1, 1), groups: int = 1):
ih, iw = x.size()[-2:]
kh, kw = weight.size()[-2:]
pad_h = _calc_same_pad(ih, kh, stride[0], dilation[0])
pad_w = _calc_same_pad(iw, kw, stride[1], dilation[1])
if pad_h > 0 or pad_w > 0:
x = F.pad(x, [pad_w // 2, pad_w - pad_w // 2, pad_h // 2, pad_h - pad_h // 2])
return F.conv2d(x, weight, bias, stride, (0, 0), dilation, groups)
def get_padding_value(padding, kernel_size, **kwargs):
dynamic = False
if isinstance(padding, str):
# for any string padding, the padding will be calculated for you, one of three ways
padding = padding.lower()
if padding == 'same':
# TF compatible 'SAME' padding, has a performance and GPU memory allocation impact
if _is_static_pad(kernel_size, **kwargs):
# static case, no extra overhead
padding = _get_padding(kernel_size, **kwargs)
else:
# dynamic padding
padding = 0
dynamic = True
elif padding == 'valid':
# 'VALID' padding, same as padding=0
padding = 0
else:
# Default to PyTorch style 'same'-ish symmetric padding
padding = _get_padding(kernel_size, **kwargs)
return padding, dynamic
def get_condconv_initializer(initializer, num_experts, expert_shape):
def condconv_initializer(weight):
"""CondConv initializer function."""
num_params = np.prod(expert_shape)
if (len(weight.shape) != 2 or weight.shape[0] != num_experts or weight.shape[1] != num_params):
raise (ValueError('CondConv variables must have shape [num_experts, num_params]'))
for i in range(num_experts):
initializer(weight[i].view(expert_shape))
return condconv_initializer
class CondConv2d(nn.Module):
""" Conditional Convolution
Inspired by: https://github.com/tensorflow/tpu/blob/master/models/official/efficientnet/condconv/condconv_layers.py
Grouped convolution hackery for parallel execution of the per-sample kernel filters inspired by this discussion:
https://github.com/pytorch/pytorch/issues/17983
"""
__constants__ = ['bias', 'in_channels', 'out_channels', 'dynamic_padding']
def __init__(self, in_channels, out_channels, kernel_size=3,
stride=1, padding='', dilation=1, groups=1, bias=False, num_experts=4):
super(CondConv2d, self).__init__()
assert num_experts > 1
if isinstance(stride, container_abcs.Iterable) and len(stride) == 1:
stride = stride[0]
# print('CondConv', num_experts)
self.in_channels = in_channels
self.out_channels = out_channels
self.kernel_size = _pair(kernel_size)
self.stride = _pair(stride)
padding_val, is_padding_dynamic = get_padding_value(padding, kernel_size, stride=stride, dilation=dilation)
self.dynamic_padding = is_padding_dynamic # if in forward to work with torchscript
self.padding = _pair(padding_val)
self.dilation = _pair(dilation)
self.groups = groups
self.num_experts = num_experts
self.weight_shape = (self.out_channels, self.in_channels // self.groups) + self.kernel_size
weight_num_param = 1
for wd in self.weight_shape:
weight_num_param *= wd
self.weight = torch.nn.Parameter(torch.Tensor(self.num_experts, weight_num_param))
if bias:
self.bias_shape = (self.out_channels,)
self.bias = torch.nn.Parameter(torch.Tensor(self.num_experts, self.out_channels))
else:
self.register_parameter('bias', None)
self.reset_parameters()
def reset_parameters(self):
num_input_fmaps = self.weight.size(1)
num_output_fmaps = self.weight.size(0)
receptive_field_size = 1
if self.weight.dim() > 2:
receptive_field_size = self.weight[0][0].numel()
fan_in = num_input_fmaps * receptive_field_size
fan_out = num_output_fmaps * receptive_field_size
init_weight = get_condconv_initializer(partial(nn.init.normal_, mean=0.0, std=np.sqrt(2.0 / fan_out)), self.num_experts, self.weight_shape)
init_weight(self.weight)
if self.bias is not None:
# fan_in = np.prod(self.weight_shape[1:])
# bound = 1 / math.sqrt(fan_in)
init_bias = get_condconv_initializer(partial(nn.init.constant_, val=0), self.num_experts, self.bias_shape)
init_bias(self.bias)
def forward(self, x, routing_weights):
x_orig = x
B, C, H, W = x.shape
weight = torch.matmul(routing_weights, self.weight) # (Expert x out x in x 3x3) --> (B x out x in x 3x3)
new_weight_shape = (B * self.out_channels, self.in_channels // self.groups) + self.kernel_size
weight = weight.view(new_weight_shape) # (B*out x in x 3 x 3)
bias = None
if self.bias is not None:
bias = torch.matmul(routing_weights, self.bias)
bias = bias.view(B * self.out_channels)
# move batch elements with channels so each batch element can be efficiently convolved with separate kernel
x = x.view(1, B * C, H, W)
if self.dynamic_padding:
out = conv2d_same(
x, weight, bias, stride=self.stride, padding=self.padding,
dilation=self.dilation, groups=self.groups * B)
else:
out = F.conv2d(
x, weight, bias, stride=self.stride, padding=self.padding,
dilation=self.dilation, groups=self.groups * B)
# out : (1 x B*out x ...)
out = out.permute([1, 0, 2, 3]).view(B, self.out_channels, out.shape[-2], out.shape[-1])
# out2 = self.forward_legacy(x_orig, routing_weights)
# lt = torch.lt(torch.abs(torch.add(out, -out2)), 1e-8)
# assert torch.all(lt), torch.abs(torch.add(out, -out2))[lt]
# print('checked')
return out
def forward_legacy(self, x, routing_weights):
# Literal port (from TF definition)
B, C, H, W = x.shape
weight = torch.matmul(routing_weights, self.weight) # (Expert x out x in x 3x3) --> (B x out x in x 3x3)
x = torch.split(x, 1, 0)
weight = torch.split(weight, 1, 0)
if self.bias is not None:
bias = torch.matmul(routing_weights, self.bias)
bias = torch.split(bias, 1, 0)
else:
bias = [None] * B
out = []
if self.dynamic_padding:
conv_fn = conv2d_same
else:
conv_fn = F.conv2d
for xi, wi, bi in zip(x, weight, bias):
wi = wi.view(*self.weight_shape)
if bi is not None:
bi = bi.view(*self.bias_shape)
out.append(conv_fn(
xi, wi, bi, stride=self.stride, padding=self.padding,
dilation=self.dilation, groups=self.groups))
out = torch.cat(out, 0)
return out
import torch
import torch.nn as nn
import math
from FastAutoAugment.networks.shakedrop import ShakeDrop
def conv3x3(in_planes, out_planes, stride=1):
"""
3x3 convolution with padding
"""
return nn.Conv2d(in_planes, out_planes, kernel_size=3, stride=stride, padding=1, bias=False)
class BasicBlock(nn.Module):
outchannel_ratio = 1
def __init__(self, inplanes, planes, stride=1, downsample=None, p_shakedrop=1.0):
super(BasicBlock, self).__init__()
self.bn1 = nn.BatchNorm2d(inplanes)
self.conv1 = conv3x3(inplanes, planes, stride)
self.bn2 = nn.BatchNorm2d(planes)
self.conv2 = conv3x3(planes, planes)
self.bn3 = nn.BatchNorm2d(planes)
self.relu = nn.ReLU(inplace=True)
self.downsample = downsample
self.stride = stride
self.shake_drop = ShakeDrop(p_shakedrop)
def forward(self, x):
out = self.bn1(x)
out = self.conv1(out)
out = self.bn2(out)
out = self.relu(out)
out = self.conv2(out)
out = self.bn3(out)
out = self.shake_drop(out)
if self.downsample is not None:
shortcut = self.downsample(x)
featuremap_size = shortcut.size()[2:4]
else:
shortcut = x
featuremap_size = out.size()[2:4]
batch_size = out.size()[0]
residual_channel = out.size()[1]
shortcut_channel = shortcut.size()[1]
if residual_channel != shortcut_channel:
padding = torch.autograd.Variable(
torch.cuda.FloatTensor(batch_size, residual_channel - shortcut_channel, featuremap_size[0],
featuremap_size[1]).fill_(0))
out += torch.cat((shortcut, padding), 1)
else:
out += shortcut
return out
class Bottleneck(nn.Module):
outchannel_ratio = 4
def __init__(self, inplanes, planes, stride=1, downsample=None, p_shakedrop=1.0):
super(Bottleneck, self).__init__()
self.bn1 = nn.BatchNorm2d(inplanes)
self.conv1 = nn.Conv2d(inplanes, planes, kernel_size=1, bias=False)
self.bn2 = nn.BatchNorm2d(planes)
self.conv2 = nn.Conv2d(planes, (planes * 1), kernel_size=3, stride=stride,
padding=1, bias=False)
self.bn3 = nn.BatchNorm2d((planes * 1))
self.conv3 = nn.Conv2d((planes * 1), planes * Bottleneck.outchannel_ratio, kernel_size=1, bias=False)
self.bn4 = nn.BatchNorm2d(planes * Bottleneck.outchannel_ratio)
self.relu = nn.ReLU(inplace=True)
self.downsample = downsample
self.stride = stride
self.shake_drop = ShakeDrop(p_shakedrop)
def forward(self, x):
out = self.bn1(x)
out = self.conv1(out)
out = self.bn2(out)
out = self.relu(out)
out = self.conv2(out)
out = self.bn3(out)
out = self.relu(out)
out = self.conv3(out)
out = self.bn4(out)
out = self.shake_drop(out)
if self.downsample is not None:
shortcut = self.downsample(x)
featuremap_size = shortcut.size()[2:4]
else:
shortcut = x
featuremap_size = out.size()[2:4]
batch_size = out.size()[0]
residual_channel = out.size()[1]
shortcut_channel = shortcut.size()[1]
if residual_channel != shortcut_channel:
padding = torch.autograd.Variable(
torch.cuda.FloatTensor(batch_size, residual_channel - shortcut_channel, featuremap_size[0],
featuremap_size[1]).fill_(0))
out += torch.cat((shortcut, padding), 1)
else:
out += shortcut
return out
class PyramidNet(nn.Module):
def __init__(self, dataset, depth, alpha, num_classes, bottleneck=True):
super(PyramidNet, self).__init__()
self.dataset = dataset
if self.dataset.startswith('cifar'):
self.inplanes = 16
if bottleneck:
n = int((depth - 2) / 9)
block = Bottleneck
else:
n = int((depth - 2) / 6)
block = BasicBlock
self.addrate = alpha / (3 * n * 1.0)
self.ps_shakedrop = [1. - (1.0 - (0.5 / (3 * n)) * (i + 1)) for i in range(3 * n)]
self.input_featuremap_dim = self.inplanes
self.conv1 = nn.Conv2d(3, self.input_featuremap_dim, kernel_size=3, stride=1, padding=1, bias=False)
self.bn1 = nn.BatchNorm2d(self.input_featuremap_dim)
self.featuremap_dim = self.input_featuremap_dim
self.layer1 = self.pyramidal_make_layer(block, n)
self.layer2 = self.pyramidal_make_layer(block, n, stride=2)
self.layer3 = self.pyramidal_make_layer(block, n, stride=2)
self.final_featuremap_dim = self.input_featuremap_dim
self.bn_final = nn.BatchNorm2d(self.final_featuremap_dim)
self.relu_final = nn.ReLU(inplace=True)
self.avgpool = nn.AvgPool2d(8)
self.fc = nn.Linear(self.final_featuremap_dim, num_classes)
elif dataset == 'imagenet':
blocks = {18: BasicBlock, 34: BasicBlock, 50: Bottleneck, 101: Bottleneck, 152: Bottleneck, 200: Bottleneck}
layers = {18: [2, 2, 2, 2], 34: [3, 4, 6, 3], 50: [3, 4, 6, 3], 101: [3, 4, 23, 3], 152: [3, 8, 36, 3],
200: [3, 24, 36, 3]}
if layers.get(depth) is None:
if bottleneck == True:
blocks[depth] = Bottleneck
temp_cfg = int((depth - 2) / 12)
else:
blocks[depth] = BasicBlock
temp_cfg = int((depth - 2) / 8)
layers[depth] = [temp_cfg, temp_cfg, temp_cfg, temp_cfg]
print('=> the layer configuration for each stage is set to', layers[depth])
self.inplanes = 64
self.addrate = alpha / (sum(layers[depth]) * 1.0)
self.input_featuremap_dim = self.inplanes
self.conv1 = nn.Conv2d(3, self.input_featuremap_dim, kernel_size=7, stride=2, padding=3, bias=False)
self.bn1 = nn.BatchNorm2d(self.input_featuremap_dim)
self.relu = nn.ReLU(inplace=True)
self.maxpool = nn.MaxPool2d(kernel_size=3, stride=2, padding=1)
self.featuremap_dim = self.input_featuremap_dim
self.layer1 = self.pyramidal_make_layer(blocks[depth], layers[depth][0])
self.layer2 = self.pyramidal_make_layer(blocks[depth], layers[depth][1], stride=2)
self.layer3 = self.pyramidal_make_layer(blocks[depth], layers[depth][2], stride=2)
self.layer4 = self.pyramidal_make_layer(blocks[depth], layers[depth][3], stride=2)
self.final_featuremap_dim = self.input_featuremap_dim
self.bn_final = nn.BatchNorm2d(self.final_featuremap_dim)
self.relu_final = nn.ReLU(inplace=True)
self.avgpool = nn.AvgPool2d(7)
self.fc = nn.Linear(self.final_featuremap_dim, num_classes)
for m in self.modules():
if isinstance(m, nn.Conv2d):
n = m.kernel_size[0] * m.kernel_size[1] * m.out_channels
m.weight.data.normal_(0, math.sqrt(2. / n))
elif isinstance(m, nn.BatchNorm2d):
m.weight.data.fill_(1)
m.bias.data.zero_()
assert len(self.ps_shakedrop) == 0, self.ps_shakedrop
def pyramidal_make_layer(self, block, block_depth, stride=1):
downsample = None
if stride != 1: # or self.inplanes != int(round(featuremap_dim_1st)) * block.outchannel_ratio:
downsample = nn.AvgPool2d((2, 2), stride=(2, 2), ceil_mode=True)
layers = []
self.featuremap_dim = self.featuremap_dim + self.addrate
layers.append(block(self.input_featuremap_dim, int(round(self.featuremap_dim)), stride, downsample, p_shakedrop=self.ps_shakedrop.pop(0)))
for i in range(1, block_depth):
temp_featuremap_dim = self.featuremap_dim + self.addrate
layers.append(
block(int(round(self.featuremap_dim)) * block.outchannel_ratio, int(round(temp_featuremap_dim)), 1, p_shakedrop=self.ps_shakedrop.pop(0)))
self.featuremap_dim = temp_featuremap_dim
self.input_featuremap_dim = int(round(self.featuremap_dim)) * block.outchannel_ratio
return nn.Sequential(*layers)
def forward(self, x):
if self.dataset == 'cifar10' or self.dataset == 'cifar100':
x = self.conv1(x)
x = self.bn1(x)
x = self.layer1(x)
x = self.layer2(x)
x = self.layer3(x)
x = self.bn_final(x)
x = self.relu_final(x)
x = self.avgpool(x)
x = x.view(x.size(0), -1)
x = self.fc(x)
elif self.dataset == 'imagenet':
x = self.conv1(x)
x = self.bn1(x)
x = self.relu(x)
x = self.maxpool(x)
x = self.layer1(x)
x = self.layer2(x)
x = self.layer3(x)
x = self.layer4(x)
x = self.bn_final(x)
x = self.relu_final(x)
x = self.avgpool(x)
x = x.view(x.size(0), -1)
x = self.fc(x)
return x
# Original code: https://github.com/pytorch/vision/blob/master/torchvision/models/resnet.py
import torch.nn as nn
import math
def conv3x3(in_planes, out_planes, stride=1):
"3x3 convolution with padding"
return nn.Conv2d(in_planes, out_planes, kernel_size=3, stride=stride,
padding=1, bias=False)
class BasicBlock(nn.Module):
expansion = 1
def __init__(self, inplanes, planes, stride=1, downsample=None):
super(BasicBlock, self).__init__()
self.conv1 = conv3x3(inplanes, planes, stride)
self.bn1 = nn.BatchNorm2d(planes)
self.conv2 = conv3x3(planes, planes)
self.bn2 = nn.BatchNorm2d(planes)
self.relu = nn.ReLU(inplace=True)
self.downsample = downsample
self.stride = stride
def forward(self, x):
residual = x
out = self.conv1(x)
out = self.bn1(out)
out = self.relu(out)
out = self.conv2(out)
out = self.bn2(out)
if self.downsample is not None:
residual = self.downsample(x)
out += residual
out = self.relu(out)
return out
class Bottleneck(nn.Module):
expansion = 4
def __init__(self, inplanes, planes, stride=1, downsample=None):
super(Bottleneck, self).__init__()
self.conv1 = nn.Conv2d(inplanes, planes, kernel_size=1, bias=False)
self.bn1 = nn.BatchNorm2d(planes)
self.conv2 = nn.Conv2d(planes, planes, kernel_size=3, stride=stride, padding=1, bias=False)
self.bn2 = nn.BatchNorm2d(planes)
self.conv3 = nn.Conv2d(planes, planes * Bottleneck.expansion, kernel_size=1, bias=False)
self.bn3 = nn.BatchNorm2d(planes * Bottleneck.expansion)
self.relu = nn.ReLU(inplace=True)
self.downsample = downsample
self.stride = stride
def forward(self, x):
residual = x
out = self.conv1(x)
out = self.bn1(out)
out = self.relu(out)
out = self.conv2(out)
out = self.bn2(out)
out = self.relu(out)
out = self.conv3(out)
out = self.bn3(out)
if self.downsample is not None:
residual = self.downsample(x)
out += residual
out = self.relu(out)
return out
class ResNet(nn.Module):
def __init__(self, dataset, depth, num_classes, bottleneck=False):
super(ResNet, self).__init__()
self.dataset = dataset
if self.dataset.startswith('cifar'):
self.inplanes = 16
print(bottleneck)
if bottleneck == True:
n = int((depth - 2) / 9)
block = Bottleneck
else:
n = int((depth - 2) / 6)
block = BasicBlock
self.conv1 = nn.Conv2d(3, self.inplanes, kernel_size=3, stride=1, padding=1, bias=False)
self.bn1 = nn.BatchNorm2d(self.inplanes)
self.relu = nn.ReLU(inplace=True)
self.layer1 = self._make_layer(block, 16, n)
self.layer2 = self._make_layer(block, 32, n, stride=2)
self.layer3 = self._make_layer(block, 64, n, stride=2)
# self.avgpool = nn.AvgPool2d(8)
self.avgpool = nn.AdaptiveAvgPool2d((1, 1))
self.fc = nn.Linear(64 * block.expansion, num_classes)
elif dataset == 'imagenet':
blocks ={18: BasicBlock, 34: BasicBlock, 50: Bottleneck, 101: Bottleneck, 152: Bottleneck, 200: Bottleneck}
layers ={18: [2, 2, 2, 2], 34: [3, 4, 6, 3], 50: [3, 4, 6, 3], 101: [3, 4, 23, 3], 152: [3, 8, 36, 3], 200: [3, 24, 36, 3]}
assert layers[depth], 'invalid detph for ResNet (depth should be one of 18, 34, 50, 101, 152, and 200)'
self.inplanes = 64
self.conv1 = nn.Conv2d(3, self.inplanes, kernel_size=7, stride=2, padding=3, bias=False)
self.bn1 = nn.BatchNorm2d(64)
self.relu = nn.ReLU(inplace=True)
self.maxpool = nn.MaxPool2d(kernel_size=3, stride=2, padding=1)
self.layer1 = self._make_layer(blocks[depth], 64, layers[depth][0])
self.layer2 = self._make_layer(blocks[depth], 128, layers[depth][1], stride=2)
self.layer3 = self._make_layer(blocks[depth], 256, layers[depth][2], stride=2)
self.layer4 = self._make_layer(blocks[depth], 512, layers[depth][3], stride=2)
# self.avgpool = nn.AvgPool2d(7)
self.avgpool = nn.AdaptiveAvgPool2d((1, 1))
self.fc = nn.Linear(512 * blocks[depth].expansion, num_classes)
for m in self.modules():
if isinstance(m, nn.Conv2d):
n = m.kernel_size[0] * m.kernel_size[1] * m.out_channels
m.weight.data.normal_(0, math.sqrt(2. / n))
elif isinstance(m, nn.BatchNorm2d):
m.weight.data.fill_(1)
m.bias.data.zero_()
def _make_layer(self, block, planes, blocks, stride=1):
downsample = None
if stride != 1 or self.inplanes != planes * block.expansion:
downsample = nn.Sequential(
nn.Conv2d(self.inplanes, planes * block.expansion,
kernel_size=1, stride=stride, bias=False),
nn.BatchNorm2d(planes * block.expansion),
)
layers = []
layers.append(block(self.inplanes, planes, stride, downsample))
self.inplanes = planes * block.expansion
for i in range(1, blocks):
layers.append(block(self.inplanes, planes))
return nn.Sequential(*layers)
def forward(self, x):
if self.dataset == 'cifar10' or self.dataset == 'cifar100':
x = self.conv1(x)
x = self.bn1(x)
x = self.relu(x)
x = self.layer1(x)
x = self.layer2(x)
x = self.layer3(x)
x = self.avgpool(x)
x = x.view(x.size(0), -1)
x = self.fc(x)
elif self.dataset == 'imagenet':
x = self.conv1(x)
x = self.bn1(x)
x = self.relu(x)
x = self.maxpool(x)
x = self.layer1(x)
x = self.layer2(x)
x = self.layer3(x)
x = self.layer4(x)
x = self.avgpool(x)
x = x.view(x.size(0), -1)
x = self.fc(x)
return x
# -*- coding: utf-8 -*-
import torch
import torch.nn as nn
import torch.nn.functional as F
from torch.autograd import Variable
class ShakeDropFunction(torch.autograd.Function):
@staticmethod
def forward(ctx, x, training=True, p_drop=0.5, alpha_range=[-1, 1]):
if training:
gate = torch.cuda.FloatTensor([0]).bernoulli_(1 - p_drop)
ctx.save_for_backward(gate)
if gate.item() == 0:
alpha = torch.cuda.FloatTensor(x.size(0)).uniform_(*alpha_range)
alpha = alpha.view(alpha.size(0), 1, 1, 1).expand_as(x)
return alpha * x
else:
return x
else:
return (1 - p_drop) * x
@staticmethod
def backward(ctx, grad_output):
gate = ctx.saved_tensors[0]
if gate.item() == 0:
beta = torch.cuda.FloatTensor(grad_output.size(0)).uniform_(0, 1)
beta = beta.view(beta.size(0), 1, 1, 1).expand_as(grad_output)
beta = Variable(beta)
return beta * grad_output, None, None, None
else:
return grad_output, None, None, None
class ShakeDrop(nn.Module):
def __init__(self, p_drop=0.5, alpha_range=[-1, 1]):
super(ShakeDrop, self).__init__()
self.p_drop = p_drop
self.alpha_range = alpha_range
def forward(self, x):
return ShakeDropFunction.apply(x, self.training, self.p_drop, self.alpha_range)
# -*- coding: utf-8 -*-
import math
import torch.nn as nn
import torch.nn.functional as F
from FastAutoAugment.networks.shakeshake.shakeshake import ShakeShake
from FastAutoAugment.networks.shakeshake.shakeshake import Shortcut
class ShakeBlock(nn.Module):
def __init__(self, in_ch, out_ch, stride=1):
super(ShakeBlock, self).__init__()
self.equal_io = in_ch == out_ch
self.shortcut = self.equal_io and None or Shortcut(in_ch, out_ch, stride=stride)
self.branch1 = self._make_branch(in_ch, out_ch, stride)
self.branch2 = self._make_branch(in_ch, out_ch, stride)
def forward(self, x):
h1 = self.branch1(x)
h2 = self.branch2(x)
h = ShakeShake.apply(h1, h2, self.training)
h0 = x if self.equal_io else self.shortcut(x)
return h + h0
def _make_branch(self, in_ch, out_ch, stride=1):
return nn.Sequential(
nn.ReLU(inplace=False),
nn.Conv2d(in_ch, out_ch, 3, padding=1, stride=stride, bias=False),
nn.BatchNorm2d(out_ch),
nn.ReLU(inplace=False),
nn.Conv2d(out_ch, out_ch, 3, padding=1, stride=1, bias=False),
nn.BatchNorm2d(out_ch))
class ShakeResNet(nn.Module):
def __init__(self, depth, w_base, label):
super(ShakeResNet, self).__init__()
n_units = (depth - 2) / 6
in_chs = [16, w_base, w_base * 2, w_base * 4]
self.in_chs = in_chs
self.c_in = nn.Conv2d(3, in_chs[0], 3, padding=1)
self.layer1 = self._make_layer(n_units, in_chs[0], in_chs[1])
self.layer2 = self._make_layer(n_units, in_chs[1], in_chs[2], 2)
self.layer3 = self._make_layer(n_units, in_chs[2], in_chs[3], 2)
self.fc_out = nn.Linear(in_chs[3], label)
# Initialize paramters
for m in self.modules():
if isinstance(m, nn.Conv2d):
n = m.kernel_size[0] * m.kernel_size[1] * m.out_channels
m.weight.data.normal_(0, math.sqrt(2. / n))
elif isinstance(m, nn.BatchNorm2d):
m.weight.data.fill_(1)
m.bias.data.zero_()
elif isinstance(m, nn.Linear):
m.bias.data.zero_()
def forward(self, x):
h = self.c_in(x)
h = self.layer1(h)
h = self.layer2(h)
h = self.layer3(h)
h = F.relu(h)
h = F.avg_pool2d(h, 8)
h = h.view(-1, self.in_chs[3])
h = self.fc_out(h)
return h
def _make_layer(self, n_units, in_ch, out_ch, stride=1):
layers = []
for i in range(int(n_units)):
layers.append(ShakeBlock(in_ch, out_ch, stride=stride))
in_ch, stride = out_ch, 1
return nn.Sequential(*layers)
# -*- coding: utf-8 -*-
import math
import torch.nn as nn
import torch.nn.functional as F
from FastAutoAugment.networks.shakeshake.shakeshake import ShakeShake
from FastAutoAugment.networks.shakeshake.shakeshake import Shortcut
class ShakeBottleNeck(nn.Module):
def __init__(self, in_ch, mid_ch, out_ch, cardinary, stride=1):
super(ShakeBottleNeck, self).__init__()
self.equal_io = in_ch == out_ch
self.shortcut = None if self.equal_io else Shortcut(in_ch, out_ch, stride=stride)
self.branch1 = self._make_branch(in_ch, mid_ch, out_ch, cardinary, stride)
self.branch2 = self._make_branch(in_ch, mid_ch, out_ch, cardinary, stride)
def forward(self, x):
h1 = self.branch1(x)
h2 = self.branch2(x)
h = ShakeShake.apply(h1, h2, self.training)
h0 = x if self.equal_io else self.shortcut(x)
return h + h0
def _make_branch(self, in_ch, mid_ch, out_ch, cardinary, stride=1):
return nn.Sequential(
nn.Conv2d(in_ch, mid_ch, 1, padding=0, bias=False),
nn.BatchNorm2d(mid_ch),
nn.ReLU(inplace=False),
nn.Conv2d(mid_ch, mid_ch, 3, padding=1, stride=stride, groups=cardinary, bias=False),
nn.BatchNorm2d(mid_ch),
nn.ReLU(inplace=False),
nn.Conv2d(mid_ch, out_ch, 1, padding=0, bias=False),
nn.BatchNorm2d(out_ch))
class ShakeResNeXt(nn.Module):
def __init__(self, depth, w_base, cardinary, label):
super(ShakeResNeXt, self).__init__()
n_units = (depth - 2) // 9
n_chs = [64, 128, 256, 1024]
self.n_chs = n_chs
self.in_ch = n_chs[0]
self.c_in = nn.Conv2d(3, n_chs[0], 3, padding=1)
self.layer1 = self._make_layer(n_units, n_chs[0], w_base, cardinary)
self.layer2 = self._make_layer(n_units, n_chs[1], w_base, cardinary, 2)
self.layer3 = self._make_layer(n_units, n_chs[2], w_base, cardinary, 2)
self.fc_out = nn.Linear(n_chs[3], label)
# Initialize paramters
for m in self.modules():
if isinstance(m, nn.Conv2d):
n = m.kernel_size[0] * m.kernel_size[1] * m.out_channels
m.weight.data.normal_(0, math.sqrt(2. / n))
elif isinstance(m, nn.BatchNorm2d):
m.weight.data.fill_(1)
m.bias.data.zero_()
elif isinstance(m, nn.Linear):
m.bias.data.zero_()
def forward(self, x):
h = self.c_in(x)
h = self.layer1(h)
h = self.layer2(h)
h = self.layer3(h)
h = F.relu(h)
h = F.avg_pool2d(h, 8)
h = h.view(-1, self.n_chs[3])
h = self.fc_out(h)
return h
def _make_layer(self, n_units, n_ch, w_base, cardinary, stride=1):
layers = []
mid_ch, out_ch = n_ch * (w_base // 64) * cardinary, n_ch * 4
for i in range(n_units):
layers.append(ShakeBottleNeck(self.in_ch, mid_ch, out_ch, cardinary, stride=stride))
self.in_ch, stride = out_ch, 1
return nn.Sequential(*layers)
# -*- coding: utf-8 -*-
import torch
import torch.nn as nn
import torch.nn.functional as F
from torch.autograd import Variable
class ShakeShake(torch.autograd.Function):
@staticmethod
def forward(ctx, x1, x2, training=True):
if training:
alpha = torch.cuda.FloatTensor(x1.size(0)).uniform_()
alpha = alpha.view(alpha.size(0), 1, 1, 1).expand_as(x1)
else:
alpha = 0.5
return alpha * x1 + (1 - alpha) * x2
@staticmethod
def backward(ctx, grad_output):
beta = torch.cuda.FloatTensor(grad_output.size(0)).uniform_()
beta = beta.view(beta.size(0), 1, 1, 1).expand_as(grad_output)
beta = Variable(beta)
return beta * grad_output, (1 - beta) * grad_output, None
class Shortcut(nn.Module):
def __init__(self, in_ch, out_ch, stride):
super(Shortcut, self).__init__()
self.stride = stride
self.conv1 = nn.Conv2d(in_ch, out_ch // 2, 1, stride=1, padding=0, bias=False)
self.conv2 = nn.Conv2d(in_ch, out_ch // 2, 1, stride=1, padding=0, bias=False)
self.bn = nn.BatchNorm2d(out_ch)
def forward(self, x):
h = F.relu(x)
h1 = F.avg_pool2d(h, 1, self.stride)
h1 = self.conv1(h1)
h2 = F.avg_pool2d(F.pad(h, (-1, 1, -1, 1)), 1, self.stride)
h2 = self.conv2(h2)
h = torch.cat((h1, h2), 1)
return self.bn(h)
import torch.nn as nn
import torch.nn.init as init
import torch.nn.functional as F
import numpy as np
def conv3x3(in_planes, out_planes, stride=1):
return nn.Conv2d(in_planes, out_planes, kernel_size=3, stride=stride, padding=1, bias=True)
def conv_init(m):
classname = m.__class__.__name__
if classname.find('Conv') != -1:
init.xavier_uniform_(m.weight, gain=np.sqrt(2))
init.constant_(m.bias, 0)
elif classname.find('BatchNorm') != -1:
init.constant_(m.weight, 1)
init.constant_(m.bias, 0)
class WideBasic(nn.Module):
def __init__(self, in_planes, planes, dropout_rate, stride=1):
super(WideBasic, self).__init__()
self.bn1 = nn.BatchNorm2d(in_planes, momentum=0.9)
self.conv1 = nn.Conv2d(in_planes, planes, kernel_size=3, padding=1, bias=True)
self.dropout = nn.Dropout(p=dropout_rate)
self.bn2 = nn.BatchNorm2d(planes, momentum=0.9)
self.conv2 = nn.Conv2d(planes, planes, kernel_size=3, stride=stride, padding=1, bias=True)
self.shortcut = nn.Sequential()
if stride != 1 or in_planes != planes:
self.shortcut = nn.Sequential(
nn.Conv2d(in_planes, planes, kernel_size=1, stride=stride, bias=True),
)
def forward(self, x):
out = self.dropout(self.conv1(F.relu(self.bn1(x))))
out = self.conv2(F.relu(self.bn2(out)))
out += self.shortcut(x)
return out
class WideResNet(nn.Module):
def __init__(self, depth, widen_factor, dropout_rate, num_classes):
super(WideResNet, self).__init__()
self.in_planes = 16
assert ((depth - 4) % 6 == 0), 'Wide-resnet depth should be 6n+4'
n = int((depth - 4) / 6)
k = widen_factor
nStages = [16, 16*k, 32*k, 64*k]
self.conv1 = conv3x3(3, nStages[0])
self.layer1 = self._wide_layer(WideBasic, nStages[1], n, dropout_rate, stride=1)
self.layer2 = self._wide_layer(WideBasic, nStages[2], n, dropout_rate, stride=2)
self.layer3 = self._wide_layer(WideBasic, nStages[3], n, dropout_rate, stride=2)
self.bn1 = nn.BatchNorm2d(nStages[3], momentum=0.9)
self.linear = nn.Linear(nStages[3], num_classes)
# self.apply(conv_init)
def _wide_layer(self, block, planes, num_blocks, dropout_rate, stride):
strides = [stride] + [1]*(num_blocks-1)
layers = []
for stride in strides:
layers.append(block(self.in_planes, planes, dropout_rate, stride))
self.in_planes = planes
return nn.Sequential(*layers)
def forward(self, x):
out = self.conv1(x)
out = self.layer1(out)
out = self.layer2(out)
out = self.layer3(out)
out = F.relu(self.bn1(out))
# out = F.avg_pool2d(out, 8)
out = F.adaptive_avg_pool2d(out, (1, 1))
out = out.view(out.size(0), -1)
out = self.linear(out)
return out
# Copyright 2019 Uber Technologies, Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
import os
import psutil
import re
import signal
import subprocess
import sys
import threading
import time
GRACEFUL_TERMINATION_TIME_S = 5
def terminate_executor_shell_and_children(pid):
print('terminate_executor_shell_and_children+', pid)
# If the shell already ends, no need to terminate its child.
try:
p = psutil.Process(pid)
except psutil.NoSuchProcess:
print('nosuchprocess')
return
# Terminate children gracefully.
for child in p.children():
try:
child.terminate()
except psutil.NoSuchProcess:
pass
# Wait for graceful termination.
time.sleep(GRACEFUL_TERMINATION_TIME_S)
# Send STOP to executor shell to stop progress.
p.send_signal(signal.SIGSTOP)
# Kill children recursively.
for child in p.children(recursive=True):
try:
child.kill()
except psutil.NoSuchProcess:
pass
# Kill shell itself.
p.kill()
print('terminate_executor_shell_and_children-', pid)
def forward_stream(src_fd, dst_stream, prefix, index):
with os.fdopen(src_fd, 'r') as src:
line_buffer = ''
while True:
text = os.read(src.fileno(), 1000)
if not isinstance(text, str):
text = text.decode('utf-8')
if not text:
break
for line in re.split('([\r\n])', text):
line_buffer += line
if line == '\r' or line == '\n':
if index is not None:
localtime = time.asctime(time.localtime(time.time()))
line_buffer = '{time}[{rank}]<{prefix}>:{line}'.format(
time=localtime,
rank=str(index),
prefix=prefix,
line=line_buffer
)
dst_stream.write(line_buffer)
dst_stream.flush()
line_buffer = ''
def execute(command, env=None, stdout=None, stderr=None, index=None, event=None):
# Make a pipe for the subprocess stdout/stderr.
(stdout_r, stdout_w) = os.pipe()
(stderr_r, stderr_w) = os.pipe()
# Make a pipe for notifying the child that parent has died.
(r, w) = os.pipe()
middleman_pid = os.fork()
if middleman_pid == 0:
# Close unused file descriptors to enforce PIPE behavior.
os.close(w)
os.setsid()
executor_shell = subprocess.Popen(command, shell=True, env=env,
stdout=stdout_w, stderr=stderr_w)
sigterm_received = threading.Event()
def set_sigterm_received(signum, frame):
sigterm_received.set()
signal.signal(signal.SIGINT, set_sigterm_received)
signal.signal(signal.SIGTERM, set_sigterm_received)
def kill_executor_children_if_parent_dies():
# This read blocks until the pipe is closed on the other side
# due to the process termination.
os.read(r, 1)
terminate_executor_shell_and_children(executor_shell.pid)
bg = threading.Thread(target=kill_executor_children_if_parent_dies)
bg.daemon = True
bg.start()
def kill_executor_children_if_sigterm_received():
sigterm_received.wait()
terminate_executor_shell_and_children(executor_shell.pid)
bg = threading.Thread(target=kill_executor_children_if_sigterm_received)
bg.daemon = True
bg.start()
exit_code = executor_shell.wait()
os._exit(exit_code)
# Close unused file descriptors to enforce PIPE behavior.
os.close(r)
os.close(stdout_w)
os.close(stderr_w)
# Redirect command stdout & stderr to provided streams or sys.stdout/sys.stderr.
# This is useful for Jupyter Notebook that uses custom sys.stdout/sys.stderr or
# for redirecting to a file on disk.
if stdout is None:
stdout = sys.stdout
if stderr is None:
stderr = sys.stderr
stdout_fwd = threading.Thread(target=forward_stream, args=(stdout_r, stdout, 'stdout', index))
stderr_fwd = threading.Thread(target=forward_stream, args=(stderr_r, stderr, 'stderr', index))
stdout_fwd.start()
stderr_fwd.start()
def kill_middleman_if_master_thread_terminate():
event.wait()
try:
os.kill(middleman_pid, signal.SIGTERM)
except:
# The process has already been killed elsewhere
pass
# TODO: Currently this requires explicitly declaration of the event and signal handler to set
# the event (gloo_run.py:_launch_jobs()). Need to figure out a generalized way to hide this behind
# interfaces.
if event is not None:
bg_thread = threading.Thread(target=kill_middleman_if_master_thread_terminate)
bg_thread.daemon = True
bg_thread.start()
try:
res, status = os.waitpid(middleman_pid, 0)
except:
# interrupted, send middleman TERM signal which will terminate children
os.kill(middleman_pid, signal.SIGTERM)
while True:
try:
_, status = os.waitpid(middleman_pid, 0)
break
except:
# interrupted, wait for middleman to finish
pass
stdout_fwd.join()
stderr_fwd.join()
exit_code = status >> 8
return exit_code
This diff is collapsed. Click to expand it.
import torch
from torch.optim.optimizer import Optimizer
class RMSpropTF(Optimizer):
r"""Implements RMSprop algorithm.
Reimplement original formulation to match TF rmsprop
Proposed by G. Hinton in his
`course <http://www.cs.toronto.edu/~tijmen/csc321/slides/lecture_slides_lec6.pdf>`_.
The centered version first appears in `Generating Sequences
With Recurrent Neural Networks <https://arxiv.org/pdf/1308.0850v5.pdf>`_.
The implementation here takes the square root of the gradient average before
adding epsilon (note that TensorFlow interchanges these two operations). The effective
learning rate is thus :math:`\alpha/(\sqrt{v + \epsilon})` where :math:`\alpha` from :math:`\alpha/(\sqrt{v} + \epsilon)` where :math:`\alpha`
is the scheduled learning rate and :math:`v` is the weighted moving average
of the squared gradient.
Arguments:
params (iterable): iterable of parameters to optimize or dicts defining
parameter groups
lr (float, optional): learning rate (default: 1e-2)
momentum (float, optional): momentum factor (default: 0)
alpha (float, optional): smoothing constant (default: 0.99)
eps (float, optional): term added to the denominator to improve
numerical stability (default: 1e-8)
centered (bool, optional) : if ``True``, compute the centered RMSProp,
the gradient is normalized by an estimation of its variance
weight_decay (float, optional): weight decay (L2 penalty) (default: 0)
"""
def __init__(self, params, lr=1e-2, alpha=0.99, eps=1e-8, momentum=0, weight_decay=0.0):
if not 0.0 <= lr:
raise ValueError("Invalid learning rate: {}".format(lr))
if not 0.0 <= eps:
raise ValueError("Invalid epsilon value: {}".format(eps))
if not 0.0 < momentum:
raise ValueError("Invalid momentum value: {}".format(momentum))
if not 0.0 <= alpha:
raise ValueError("Invalid alpha value: {}".format(alpha))
assert momentum > 0.0
defaults = dict(lr=lr, momentum=momentum, alpha=alpha, eps=eps, weight_decay=weight_decay)
super(RMSpropTF, self).__init__(params, defaults)
self.initialized = False
def __setstate__(self, state):
super(RMSpropTF, self).__setstate__(state)
for group in self.param_groups:
group.setdefault('momentum', 0)
def load_state_dict(self, state_dict):
super(RMSpropTF, self).load_state_dict(state_dict)
self.initialized = True
def step(self, closure=None):
"""Performs a single optimization step.
We modified pytorch's RMSProp to be same as Tensorflow's
See : https://github.com/tensorflow/tensorflow/blob/master/tensorflow/core/kernels/training_ops.cc#L485
Arguments:
closure (callable, optional): A closure that reevaluates the model
and returns the loss.
"""
loss = None
if closure is not None:
loss = closure()
for group in self.param_groups:
for p in group['params']:
if p.grad is None:
continue
grad = p.grad.data
if grad.is_sparse:
raise RuntimeError('RMSprop does not support sparse gradients')
state = self.state[p]
# State initialization
if len(state) == 0:
assert not self.initialized
state['step'] = 0
state['ms'] = torch.ones_like(p.data) #, memory_format=torch.preserve_format)
state['mom'] = torch.zeros_like(p.data) #, memory_format=torch.preserve_format)
# weight decay -----
if group['weight_decay'] > 0:
grad = grad.add(group['weight_decay'], p.data)
rho = group['alpha']
ms = state['ms']
mom = state['mom']
state['step'] += 1
# ms.mul_(rho).addcmul_(1 - rho, grad, grad)
ms.add_(torch.mul(grad, grad).add_(-ms) * (1. - rho))
assert group['momentum'] > 0
# new rmsprop
mom.mul_(group['momentum']).addcdiv_(group['lr'], grad, (ms + group['eps']).sqrt())
p.data.add_(-1.0, mom)
return loss
import torch
from torch.nn import BatchNorm2d
from torch.nn.parameter import Parameter
import torch.distributed as dist
from torch import nn
class TpuBatchNormalization(nn.Module):
# Ref : https://github.com/tensorflow/tpu/blob/master/models/official/efficientnet/utils.py#L113
def __init__(self, num_features, eps=1e-5, momentum=0.1, affine=True,
track_running_stats=True):
super(TpuBatchNormalization, self).__init__() # num_features, eps, momentum, affine, track_running_stats)
self.weight = Parameter(torch.ones(num_features))
self.bias = Parameter(torch.zeros(num_features))
self.register_buffer('running_mean', torch.zeros(num_features))
self.register_buffer('running_var', torch.ones(num_features))
self.register_buffer('num_batches_tracked', torch.tensor(0, dtype=torch.long))
self.eps = eps
self.momentum = momentum
def _reduce_avg(self, t):
dist.all_reduce(t, dist.ReduceOp.SUM)
t.mul_(1. / dist.get_world_size())
def forward(self, input):
if not self.training or not dist.is_initialized():
bn = (input - self.running_mean.view(1, self.running_mean.shape[0], 1, 1)) / \
(torch.sqrt(self.running_var.view(1, self.running_var.shape[0], 1, 1) + self.eps))
# print(self.weight.shape, self.bias.shape)
return bn.mul(self.weight.view(1, self.weight.shape[0], 1, 1)).add(self.bias.view(1, self.bias.shape[0], 1, 1))
shard_mean, shard_invstd = torch.batch_norm_stats(input, self.eps)
shard_vars = (1. / shard_invstd) ** 2 - self.eps
shard_square_of_mean = torch.mul(shard_mean, shard_mean)
shard_mean_of_square = shard_vars + shard_square_of_mean
group_mean = shard_mean.clone().detach()
self._reduce_avg(group_mean)
group_mean_of_square = shard_mean_of_square.clone().detach()
self._reduce_avg(group_mean_of_square)
group_vars = group_mean_of_square - torch.mul(group_mean, group_mean)
group_mean = group_mean.detach()
group_vars = group_vars.detach()
# print(self.running_mean.shape, self.running_var.shape)
self.running_mean.mul_(1. - self.momentum).add_(group_mean.mul(self.momentum))
self.running_var.mul_(1. - self.momentum).add_(group_vars.mul(self.momentum))
self.num_batches_tracked.add_(1)
# print(input.shape, group_mean.view(1, group_mean.shape[0], 1, 1).shape, group_vars.view(1, group_vars.shape[0], 1, 1).shape, self.eps)
bn = (input - group_mean.view(1, group_mean.shape[0], 1, 1)) / (torch.sqrt(group_vars.view(1, group_vars.shape[0], 1, 1) + self.eps))
# print(self.weight.shape, self.bias.shape)
return bn.mul(self.weight.view(1, self.weight.shape[0], 1, 1)).add(self.bias.view(1, self.bias.shape[0], 1, 1))
This diff is collapsed. Click to expand it.
import sys
sys.path.append('/data/private/fast-autoaugment-public') # TODO
import time
import os
import threading
import six
from six.moves import queue
from FastAutoAugment import safe_shell_exec
def _exec_command(command):
host_output = six.StringIO()
try:
exit_code = safe_shell_exec.execute(command,
stdout=host_output,
stderr=host_output)
if exit_code != 0:
print('Launching task function was not successful:\n{host_output}'.format(host_output=host_output.getvalue()))
os._exit(exit_code)
finally:
host_output.close()
return exit_code
def execute_function_multithreaded(fn,
args_list,
block_until_all_done=True,
max_concurrent_executions=1000):
"""
Executes fn in multiple threads each with one set of the args in the
args_list.
:param fn: function to be executed
:type fn:
:param args_list:
:type args_list: list(list)
:param block_until_all_done: if is True, function will block until all the
threads are done and will return the results of each thread's execution.
:type block_until_all_done: bool
:param max_concurrent_executions:
:type max_concurrent_executions: int
:return:
If block_until_all_done is False, returns None. If block_until_all_done is
True, function returns the dict of results.
{
index: execution result of fn with args_list[index]
}
:rtype: dict
"""
result_queue = queue.Queue()
worker_queue = queue.Queue()
for i, arg in enumerate(args_list):
arg.append(i)
worker_queue.put(arg)
def fn_execute():
while True:
try:
arg = worker_queue.get(block=False)
except queue.Empty:
return
exec_index = arg[-1]
res = fn(*arg[:-1])
result_queue.put((exec_index, res))
threads = []
number_of_threads = min(max_concurrent_executions, len(args_list))
for _ in range(number_of_threads):
thread = threading.Thread(target=fn_execute)
if not block_until_all_done:
thread.daemon = True
thread.start()
threads.append(thread)
# Returns the results only if block_until_all_done is set.
results = None
if block_until_all_done:
# Because join() cannot be interrupted by signal, a single join()
# needs to be separated into join()s with timeout in a while loop.
have_alive_child = True
while have_alive_child:
have_alive_child = False
for t in threads:
t.join(0.1)
if t.is_alive():
have_alive_child = True
results = {}
while not result_queue.empty():
item = result_queue.get()
results[item[0]] = item[1]
if len(results) != len(args_list):
raise RuntimeError(
'Some threads for func {func} did not complete '
'successfully.'.format(func=fn.__name__))
return results
if __name__ == '__main__':
import argparse
parser = argparse.ArgumentParser()
parser.add_argument('--host', type=str)
parser.add_argument('--num-gpus', type=int, default=4)
parser.add_argument('--master', type=str, default='task1')
parser.add_argument('--port', type=int, default=1958)
parser.add_argument('-c', '--conf', type=str)
parser.add_argument('--args', type=str, default='')
args = parser.parse_args()
try:
hosts = ['task%d' % (x + 1) for x in range(int(args.host))]
except:
hosts = args.host.split(',')
cwd = os.getcwd()
command_list = []
for node_rank, host in enumerate(hosts):
ssh_cmd = f'ssh -t -t -o StrictHostKeyChecking=no {host} -p 22 ' \
f'\'bash -O huponexit -c "cd {cwd} && ' \
f'python -m torch.distributed.launch --nproc_per_node={args.num_gpus} --nnodes={len(hosts)} ' \
f'--master_addr={args.master} --master_port={args.port} --node_rank={node_rank} ' \
f'FastAutoAugment/train.py -c {args.conf} {args.args}"' \
'\''
print(ssh_cmd)
command_list.append([ssh_cmd])
execute_function_multithreaded(_exec_command,
command_list[1:],
block_until_all_done=False)
print(command_list[0])
while True:
time.sleep(1)
# thread = threading.Thread(target=safe_shell_exec.execute, args=(command_list[0][0],))
# thread.start()
# thread.join()
# while True:
# time.sleep(1)
MIT License
Copyright (c) 2019 Ildoo Kim
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
# Fast AutoAugment **(Accepted at NeurIPS 2019)**
Official [Fast AutoAugment](https://arxiv.org/abs/1905.00397) implementation in PyTorch.
- Fast AutoAugment learns augmentation policies using a more efficient search strategy based on density matching.
- Fast AutoAugment speeds up the search time by orders of magnitude while maintaining the comparable performances.
<p align="center">
<img src="etc/search.jpg" height=350>
</p>
## Results
### CIFAR-10 / 100
Search : **3.5 GPU Hours (1428x faster than AutoAugment)**, WResNet-40x2 on Reduced CIFAR-10
| Model(CIFAR-10) | Baseline | Cutout | AutoAugment | Fast AutoAugment<br/>(transfer/direct) | |
|-------------------------|------------|------------|-------------|------------------|----|
| Wide-ResNet-40-2 | 5.3 | 4.1 | 3.7 | 3.6 / 3.7 | [Download](https://arena.kakaocdn.net/brainrepo/fast-autoaugment/cifar10_wresnet40x2_top1_3.52.pth) |
| Wide-ResNet-28-10 | 3.9 | 3.1 | 2.6 | 2.7 / 2.7 | [Download](https://arena.kakaocdn.net/brainrepo/fast-autoaugment/cifar10_wresnet28x10_top1.pth) |
| Shake-Shake(26 2x32d) | 3.6 | 3.0 | 2.5 | 2.7 / 2.5 | [Download](https://arena.kakaocdn.net/brainrepo/fast-autoaugment/cifar10_shake26_2x32d_top1_2.68.pth) |
| Shake-Shake(26 2x96d) | 2.9 | 2.6 | 2.0 | 2.0 / 2.0 | [Download](https://arena.kakaocdn.net/brainrepo/fast-autoaugment/cifar10_shake26_2x96d_top1_1.97.pth) |
| Shake-Shake(26 2x112d) | 2.8 | 2.6 | 1.9 | 2.0 / 1.9 | [Download](https://arena.kakaocdn.net/brainrepo/fast-autoaugment/cifar10_shake26_2x112d_top1_2.04.pth) |
| PyramidNet+ShakeDrop | 2.7 | 2.3 | 1.5 | 1.8 / 1.7 | [Download](https://arena.kakaocdn.net/brainrepo/fast-autoaugment/cifar10_pyramid272_top1_1.44.pth) |
| Model(CIFAR-100) | Baseline | Cutout | AutoAugment | Fast AutoAugment<br/>(transfer/direct) | |
|-----------------------|------------|------------|-------------|------------------|----|
| Wide-ResNet-40-2 | 26.0 | 25.2 | 20.7 | 20.7 / 20.6 | [Download](https://arena.kakaocdn.net/brainrepo/fast-autoaugment/cifar100_wresnet40x2_top1_20.43.pth) |
| Wide-ResNet-28-10 | 18.8 | 18.4 | 17.1 | 17.3 / 17.3 | [Download](https://arena.kakaocdn.net/brainrepo/fast-autoaugment/cifar100_wresnet28x10_top1_17.17.pth) |
| Shake-Shake(26 2x96d) | 17.1 | 16.0 | 14.3 | 14.9 / 14.6 | [Download](https://arena.kakaocdn.net/brainrepo/fast-autoaugment/cifar100_shake26_2x96d_top1_15.15.pth) |
| PyramidNet+ShakeDrop | 14.0 | 12.2 | 10.7 | 11.9 / 11.7 | [Download](https://arena.kakaocdn.net/brainrepo/fast-autoaugment/cifar100_pyramid272_top1_11.74.pth) |
### ImageNet
Search : **450 GPU Hours (33x faster than AutoAugment)**, ResNet-50 on Reduced ImageNet
| Model | Baseline | AutoAugment | Fast AutoAugment<br/>(Top1/Top5) | |
|------------|------------|-------------|------------------|----|
| ResNet-50 | 23.7 / 6.9 | 22.4 / 6.2 | **22.4 / 6.3** | [Download](https://arena.kakaocdn.net/brainrepo/fast-autoaugment/imagenet_resnet50_top1_22.2.pth) |
| ResNet-200 | 21.5 / 5.8 | 20.0 / 5.0 | **19.4 / 4.7** | [Download](https://arena.kakaocdn.net/brainrepo/fast-autoaugment/imagenet_resnet200_top1_19.4.pth) |
Notes
* We evaluated resnet-50 and resnet-200 with resolution of 224 and 320, respectively. According to the original resnet paper, resnet 200 was tested with the resolution of 320. Also our resnet-200 baseline's performance was similar when we use the resolution.
* But with recent our code clean-up and bugfixes, we've found that the baseline performs similar to the baseline even using 224x224.
* When we use 224x224, resnet-200 performs **20.0 / 5.2**. Download link for the trained model is [here](https://arena.kakaocdn.net/brainrepo/fast-autoaugment/imagenet_resnet200_res224.pth).
We have conducted additional experiments with EfficientNet.
| Model | Baseline | AutoAugment | | Our Baseline(Batch) | +Fast AA |
|-------|------------|-------------|---|---------------------|----------|
| B0 | 23.2 | 22.7 | | 22.96 | 22.68 |
### SVHN Test
Search : **1.5 GPU Hours**
| | Baseline | AutoAug / Our | Fast AutoAugment |
|----------------------------------|---------:|--------------:|--------:|
| Wide-Resnet28x10 | 1.5 | 1.1 | 1.1 |
## Run
We conducted experiments under
- python 3.6.9
- pytorch 1.2.0, torchvision 0.4.0, cuda10
### Search a augmentation policy
Please read ray's document to construct a proper ray cluster : https://github.com/ray-project/ray, and run search.py with the master's redis address.
```
$ python search.py -c confs/wresnet40x2_cifar10_b512.yaml --dataroot ... --redis ...
```
### Train a model with found policies
You can train network architectures on CIFAR-10 / 100 and ImageNet with our searched policies.
- fa_reduced_cifar10 : reduced CIFAR-10(4k images), WResNet-40x2
- fa_reduced_imagenet : reduced ImageNet(50k images, 120 classes), ResNet-50
```
$ export PYTHONPATH=$PYTHONPATH:$PWD
$ python FastAutoAugment/train.py -c confs/wresnet40x2_cifar10_b512.yaml --aug fa_reduced_cifar10 --dataset cifar10
$ python FastAutoAugment/train.py -c confs/wresnet40x2_cifar10_b512.yaml --aug fa_reduced_cifar10 --dataset cifar100
$ python FastAutoAugment/train.py -c confs/wresnet28x10_cifar10_b512.yaml --aug fa_reduced_cifar10 --dataset cifar10
$ python FastAutoAugment/train.py -c confs/wresnet28x10_cifar10_b512.yaml --aug fa_reduced_cifar10 --dataset cifar100
...
$ python FastAutoAugment/train.py -c confs/resnet50_b512.yaml --aug fa_reduced_imagenet
$ python FastAutoAugment/train.py -c confs/resnet200_b512.yaml --aug fa_reduced_imagenet
```
By adding --only-eval and --save arguments, you can test trained models without training.
If you want to train with multi-gpu/node, use `torch.distributed.launch` such as
```bash
$ python -m torch.distributed.launch --nproc_per_node={num_gpu_per_node} --nnodes={num_node} --master_addr={master} --master_port={master_port} --node_rank={0,1,2,...,num_node} FastAutoAugment/train.py -c confs/efficientnet_b4.yaml --aug fa_reduced_imagenet
```
## Citation
If you use this code in your research, please cite our [paper](https://arxiv.org/abs/1905.00397).
```
@inproceedings{lim2019fast,
title={Fast AutoAugment},
author={Lim, Sungbin and Kim, Ildoo and Kim, Taesup and Kim, Chiheon and Kim, Sungwoong},
booktitle={Advances in Neural Information Processing Systems (NeurIPS)},
year={2019}
}
```
## Contact for Issues
- Ildoo Kim, ildoo.kim@kakaobrain.com
## References & Opensources
We increase the batch size and adapt the learning rate accordingly to boost the training. Otherwise, we set other hyperparameters equal to AutoAugment if possible. For the unknown hyperparameters, we follow values from the original references or we tune them to match baseline performances.
- **ResNet** : [paper1](https://arxiv.org/abs/1512.03385), [paper2](https://arxiv.org/abs/1603.05027), [code](https://github.com/osmr/imgclsmob/tree/master/pytorch/pytorchcv/models)
- **PyramidNet** : [paper](https://arxiv.org/abs/1610.02915), [code](https://github.com/dyhan0920/PyramidNet-PyTorch)
- **Wide-ResNet** : [code](https://github.com/meliketoy/wide-resnet.pytorch)
- **Shake-Shake** : [code](https://github.com/owruby/shake-shake_pytorch)
- **ShakeDrop Regularization** : [paper](https://arxiv.org/abs/1802.02375), [code](https://github.com/owruby/shake-drop_pytorch)
- **AutoAugment** : [code](https://github.com/tensorflow/models/tree/master/research/autoaugment)
- **Ray** : [code](https://github.com/ray-project/ray)
- **HyperOpt** : [code](https://github.com/hyperopt/hyperopt)
This diff could not be displayed because it is too large.
model:
type: efficientnet-b0
condconv_num_expert: 1 # if this is greater than 1(eg. 4), it activates condconv.
dataset: imagenet
aug: fa_reduced_imagenet
cutout: 0
batch: 128 # per gpu
epoch: 350
lr: 0.008 # 0.256 for 4096 batch
lr_schedule:
type: 'efficientnet'
warmup:
multiplier: 1
epoch: 5
optimizer:
type: rmsprop
decay: 0.00001
clip: 0
ema: 0.9999
ema_interval: -1
lb_smooth: 0.1
\ No newline at end of file
model:
type: efficientnet-b0
condconv_num_expert: 8 # if this is greater than 1(eg. 4), it activates condconv.
dataset: imagenet
aug: fa_reduced_imagenet
cutout: 0
batch: 128 # per gpu
epoch: 350
lr: 0.008 # 0.256 for 4096 batch
lr_schedule:
type: 'efficientnet'
warmup:
multiplier: 1
epoch: 5
optimizer:
type: rmsprop
decay: 0.00001
clip: 0
ema: 0.9999
ema_interval: -1
lb_smooth: 0.1
mixup: 0.2
model:
type: efficientnet-b1
condconv_num_expert: 1 # if this is greater than 1(eg. 4), it activates condconv.
dataset: imagenet
aug: fa_reduced_imagenet
cutout: 0
batch: 128 # per gpu
epoch: 350
lr: 0.008 # 0.256 for 4096 batch
lr_schedule:
type: 'efficientnet'
warmup:
multiplier: 1
epoch: 5
optimizer:
type: rmsprop
decay: 0.00001
clip: 0
ema: 0.9999
ema_interval: -1
lb_smooth: 0.1
model:
type: efficientnet-b2
condconv_num_expert: 1 # if this is greater than 1(eg. 4), it activates condconv.
dataset: imagenet
aug: fa_reduced_imagenet
cutout: 0
batch: 128 # per gpu
epoch: 350
lr: 0.008 # 0.256 for 4096 batch
lr_schedule:
type: 'efficientnet'
warmup:
multiplier: 1
epoch: 5
optimizer:
type: rmsprop
decay: 0.00001
clip: 0
ema: 0.9999
ema_interval: -1
lb_smooth: 0.1
model:
type: efficientnet-b3
condconv_num_expert: 1 # if this is greater than 1(eg. 4), it activates condconv.
dataset: imagenet
aug: fa_reduced_imagenet
cutout: 0
batch: 64 # per gpu
epoch: 350
lr: 0.004 # 0.256 for 4096 batch
lr_schedule:
type: 'efficientnet'
warmup:
multiplier: 1
epoch: 5
optimizer:
type: rmsprop
decay: 0.00001
clip: 0
ema: 0.9999
ema_interval: -1
lb_smooth: 0.1
model:
type: efficientnet-b4
condconv_num_expert: 1 # if this is greater than 1(eg. 4), it activates condconv.
dataset: imagenet
aug: fa_reduced_imagenet
cutout: 0
batch: 32 # per gpu
epoch: 350
lr: 0.002 # 0.256 for 4096 batch
lr_schedule:
type: 'efficientnet'
warmup:
multiplier: 1
epoch: 5
optimizer:
type: rmsprop
decay: 0.00001
clip: 0
ema: 0.9999
ema_interval: -1
lb_smooth: 0.1
model:
type: pyramid
depth: 272
alpha: 200
bottleneck: True
dataset: cifar10
aug: fa_reduced_cifar10
cutout: 16
batch: 64
epoch: 1800
lr: 0.05
lr_schedule:
type: 'cosine'
warmup:
multiplier: 1
epoch: 5
optimizer:
type: sgd
nesterov: True
decay: 0.00005
model:
type: resnet200
dataset: imagenet
aug: fa_reduced_imagenet
cutout: 0
batch: 64
epoch: 270
lr: 0.025
lr_schedule:
type: 'resnet'
warmup:
multiplier: 1
epoch: 5
optimizer:
type: sgd
nesterov: True
decay: 0.0001
clip: 0
model:
type: resnet50
dataset: imagenet
aug: fa_reduced_imagenet
cutout: 0
batch: 128
epoch: 270
lr: 0.05
lr_schedule:
type: 'resnet'
warmup:
multiplier: 1
epoch: 5
optimizer:
type: sgd
nesterov: True
decay: 0.0001
clip: 0
ema: 0
model:
type: resnet50
dataset: imagenet
aug: fa_reduced_imagenet
cutout: 0
batch: 128
epoch: 270
lr: 0.05
lr_schedule:
type: 'resnet'
warmup:
multiplier: 1
epoch: 5
optimizer:
type: sgd
nesterov: True
decay: 0.0001
clip: 0
ema: 0
#lb_smooth: 0.1
mixup: 0.2
model:
type: shakeshake26_2x112d
dataset: cifar10
aug: fa_reduced_cifar10
cutout: 16
batch: 128
epoch: 1800
lr: 0.01
lr_schedule:
type: 'cosine'
warmup:
multiplier: 1
epoch: 5
optimizer:
type: sgd
nesterov: True
decay: 0.002
model:
type: shakeshake26_2x32d
dataset: cifar10
aug: fa_reduced_cifar10
cutout: 16
batch: 128
epoch: 1800
lr: 0.01
lr_schedule:
type: 'cosine'
warmup:
multiplier: 1
epoch: 5
optimizer:
type: sgd
nesterov: True
decay: 0.001
model:
type: shakeshake26_2x96d
dataset: cifar10
aug: fa_reduced_cifar10
cutout: 16
batch: 128
epoch: 1800
lr: 0.01
lr_schedule:
type: 'cosine'
warmup:
multiplier: 1
epoch: 5
optimizer:
type: sgd
nesterov: True
decay: 0.001
model:
type: wresnet28_10
dataset: cifar10
aug: fa_reduced_cifar10
cutout: 16
batch: 128
epoch: 200
lr: 0.1
lr_schedule:
type: 'cosine'
warmup:
multiplier: 1
epoch: 5
optimizer:
type: sgd
nesterov: True
decay: 0.0005
\ No newline at end of file
model:
type: wresnet28_10
dataset: svhn
aug: fa_reduced_svhn
cutout: 20
batch: 128
epoch: 200
lr: 0.01
lr_schedule:
type: 'cosine'
warmup:
multiplier: 1
epoch: 5
optimizer:
type: sgd
nesterov: True
decay: 0.0005
\ No newline at end of file
model:
type: wresnet40_2
dataset: cifar10
aug: fa_reduced_cifar10
cutout: 16
batch: 128
epoch: 200
lr: 0.1
lr_schedule:
type: 'cosine'
warmup:
multiplier: 1
epoch: 5
optimizer:
type: sgd
nesterov: True
decay: 0.0002
git+https://github.com/wbaek/theconf
git+https://github.com/ildoonet/pytorch-gradual-warmup-lr.git@08f7d5e
git+https://github.com/ildoonet/pystopwatch2.git
git+https://github.com/hyperopt/hyperopt.git
git+https://github.com/kakaobrain/torchlars
pretrainedmodels
tqdm
tensorboardx
sklearn
ray
matplotlib
psutil
requests
\ No newline at end of file
{
"cells": [
{
"cell_type": "code",
"execution_count": 1,
"metadata": {},
"outputs": [],
"source": [
"import pandas as pd"
]
},
{
"cell_type": "code",
"execution_count": 3,
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>id</th>\n",
" <th>Label</th>\n",
" <th>Subject</th>\n",
" <th>Date</th>\n",
" <th>Gender</th>\n",
" <th>Age</th>\n",
" <th>mmse</th>\n",
" <th>ageAtEntry</th>\n",
" <th>cdr</th>\n",
" <th>commun</th>\n",
" <th>...</th>\n",
" <th>memory</th>\n",
" <th>orient</th>\n",
" <th>perscare</th>\n",
" <th>apoe</th>\n",
" <th>sumbox</th>\n",
" <th>acsparnt</th>\n",
" <th>height</th>\n",
" <th>weight</th>\n",
" <th>primStudy</th>\n",
" <th>acsStudy</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>/@WEBAPP/images/r.gif</td>\n",
" <td>OAS30001_ClinicalData_d3025</td>\n",
" <td>OAS30001</td>\n",
" <td>NaN</td>\n",
" <td>female</td>\n",
" <td>NaN</td>\n",
" <td>30.0</td>\n",
" <td>65.149895</td>\n",
" <td>0.0</td>\n",
" <td>0.0</td>\n",
" <td>...</td>\n",
" <td>0.0</td>\n",
" <td>0.0</td>\n",
" <td>0.0</td>\n",
" <td>23.0</td>\n",
" <td>0.0</td>\n",
" <td>NaN</td>\n",
" <td>64.0</td>\n",
" <td>180.0</td>\n",
" <td>NaN</td>\n",
" <td>NaN</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1</th>\n",
" <td>/@WEBAPP/images/r.gif</td>\n",
" <td>OAS30001_ClinicalData_d3977</td>\n",
" <td>OAS30001</td>\n",
" <td>NaN</td>\n",
" <td>female</td>\n",
" <td>NaN</td>\n",
" <td>29.0</td>\n",
" <td>65.149895</td>\n",
" <td>0.0</td>\n",
" <td>0.0</td>\n",
" <td>...</td>\n",
" <td>0.0</td>\n",
" <td>0.0</td>\n",
" <td>0.0</td>\n",
" <td>23.0</td>\n",
" <td>0.0</td>\n",
" <td>NaN</td>\n",
" <td>NaN</td>\n",
" <td>NaN</td>\n",
" <td>NaN</td>\n",
" <td>NaN</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2</th>\n",
" <td>/@WEBAPP/images/r.gif</td>\n",
" <td>OAS30001_ClinicalData_d3332</td>\n",
" <td>OAS30001</td>\n",
" <td>NaN</td>\n",
" <td>female</td>\n",
" <td>NaN</td>\n",
" <td>30.0</td>\n",
" <td>65.149895</td>\n",
" <td>0.0</td>\n",
" <td>0.0</td>\n",
" <td>...</td>\n",
" <td>0.0</td>\n",
" <td>0.0</td>\n",
" <td>0.0</td>\n",
" <td>23.0</td>\n",
" <td>0.0</td>\n",
" <td>NaN</td>\n",
" <td>63.0</td>\n",
" <td>185.0</td>\n",
" <td>NaN</td>\n",
" <td>NaN</td>\n",
" </tr>\n",
" <tr>\n",
" <th>3</th>\n",
" <td>/@WEBAPP/images/r.gif</td>\n",
" <td>OAS30001_ClinicalData_d0000</td>\n",
" <td>OAS30001</td>\n",
" <td>NaN</td>\n",
" <td>female</td>\n",
" <td>NaN</td>\n",
" <td>28.0</td>\n",
" <td>65.149895</td>\n",
" <td>0.0</td>\n",
" <td>0.0</td>\n",
" <td>...</td>\n",
" <td>0.0</td>\n",
" <td>0.0</td>\n",
" <td>0.0</td>\n",
" <td>23.0</td>\n",
" <td>0.0</td>\n",
" <td>NaN</td>\n",
" <td>NaN</td>\n",
" <td>NaN</td>\n",
" <td>NaN</td>\n",
" <td>NaN</td>\n",
" </tr>\n",
" <tr>\n",
" <th>4</th>\n",
" <td>/@WEBAPP/images/r.gif</td>\n",
" <td>OAS30001_ClinicalData_d1456</td>\n",
" <td>OAS30001</td>\n",
" <td>NaN</td>\n",
" <td>female</td>\n",
" <td>NaN</td>\n",
" <td>30.0</td>\n",
" <td>65.149895</td>\n",
" <td>0.0</td>\n",
" <td>0.0</td>\n",
" <td>...</td>\n",
" <td>0.0</td>\n",
" <td>0.0</td>\n",
" <td>0.0</td>\n",
" <td>23.0</td>\n",
" <td>0.0</td>\n",
" <td>NaN</td>\n",
" <td>63.0</td>\n",
" <td>173.0</td>\n",
" <td>NaN</td>\n",
" <td>NaN</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"<p>5 rows × 27 columns</p>\n",
"</div>"
],
"text/plain": [
" id Label Subject Date Gender \\\n",
"0 /@WEBAPP/images/r.gif OAS30001_ClinicalData_d3025 OAS30001 NaN female \n",
"1 /@WEBAPP/images/r.gif OAS30001_ClinicalData_d3977 OAS30001 NaN female \n",
"2 /@WEBAPP/images/r.gif OAS30001_ClinicalData_d3332 OAS30001 NaN female \n",
"3 /@WEBAPP/images/r.gif OAS30001_ClinicalData_d0000 OAS30001 NaN female \n",
"4 /@WEBAPP/images/r.gif OAS30001_ClinicalData_d1456 OAS30001 NaN female \n",
"\n",
" Age mmse ageAtEntry cdr commun ... memory orient perscare apoe \\\n",
"0 NaN 30.0 65.149895 0.0 0.0 ... 0.0 0.0 0.0 23.0 \n",
"1 NaN 29.0 65.149895 0.0 0.0 ... 0.0 0.0 0.0 23.0 \n",
"2 NaN 30.0 65.149895 0.0 0.0 ... 0.0 0.0 0.0 23.0 \n",
"3 NaN 28.0 65.149895 0.0 0.0 ... 0.0 0.0 0.0 23.0 \n",
"4 NaN 30.0 65.149895 0.0 0.0 ... 0.0 0.0 0.0 23.0 \n",
"\n",
" sumbox acsparnt height weight primStudy acsStudy \n",
"0 0.0 NaN 64.0 180.0 NaN NaN \n",
"1 0.0 NaN NaN NaN NaN NaN \n",
"2 0.0 NaN 63.0 185.0 NaN NaN \n",
"3 0.0 NaN NaN NaN NaN NaN \n",
"4 0.0 NaN 63.0 173.0 NaN NaN \n",
"\n",
"[5 rows x 27 columns]"
]
},
"execution_count": 3,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"all_data = pd.read_csv(\"..\\data\\ADRC clinical data_all.csv\")\n",
"\n",
"all_data.head()"
]
},
{
"cell_type": "code",
"execution_count": 18,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Subject\n",
"OAS30001 0.0\n",
"OAS30002 0.0\n",
"OAS30003 0.0\n",
"OAS30004 0.0\n",
"OAS30005 0.0\n",
" ... \n",
"OAS31168 0.0\n",
"OAS31169 3.0\n",
"OAS31170 2.0\n",
"OAS31171 2.0\n",
"OAS31172 0.0\n",
"Name: cdr, Length: 1098, dtype: float64\n"
]
}
],
"source": [
"ad = all_data.groupby(['Subject'])['cdr'].max()\n",
"print(ad)"
]
},
{
"cell_type": "code",
"execution_count": 21,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"'OAS30001'"
]
},
"execution_count": 21,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"ad.index[0]"
]
},
{
"cell_type": "code",
"execution_count": 22,
"metadata": {},
"outputs": [
{
"ename": "SyntaxError",
"evalue": "unexpected EOF while parsing (<ipython-input-22-b8a078b72aca>, line 5)",
"output_type": "error",
"traceback": [
"\u001b[1;36m File \u001b[1;32m\"<ipython-input-22-b8a078b72aca>\"\u001b[1;36m, line \u001b[1;32m5\u001b[0m\n\u001b[1;33m #print(filtered)\u001b[0m\n\u001b[1;37m ^\u001b[0m\n\u001b[1;31mSyntaxError\u001b[0m\u001b[1;31m:\u001b[0m unexpected EOF while parsing\n"
]
}
],
"source": [
"filtered = []\n",
"for i, val in enumerate(ad):\n",
" if ad[i] == 0:\n",
" filtered.append(ad.index[i])\n",
"#print(filtered)"
]
},
{
"cell_type": "code",
"execution_count": 23,
"metadata": {},
"outputs": [],
"source": [
"df_filtered = pd.DataFrame(filtered)\n",
"df_filtered.to_csv('..\\data\\ADRC clinical data_normal.csv')"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "ML",
"language": "python",
"name": "ml"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.7.4"
}
},
"nbformat": 4,
"nbformat_minor": 2
}
This diff could not be displayed because it is too large.
%load('..\data\MICCAI_BraTS_2019_Data_Training\name_mapping.csv')
inputheader = '..\data\MICCAI_BraTS_2019_Data_Training\HGG\';
outfolder = '..\testfolder\';
id = 'BraTS19_2013_2_1';
type = 'flair.nii';
filename = strcat(id,'_', type); % BraTS19_2013_2_1_flair.nii
flair_path = strcat(inputheader, id, '\', filename,'\', filename);
%disp(path);
flair = niftiread(flair_path); %size 240x240x155
cp_flair = flair;
type = 'seg.nii';
filename = strcat(id,'_', type); % BraTS19_2013_2_1_seg.nii
seg_path = strcat(inputheader, id, '\', filename, '\', filename);
seg = niftiread(seg_path);
[x,y,z] = size(seg);
% copy flair, segment flair data
cp_flair(seg == 0) = 0;
% save a segmented data
type = 'seg_flair.nii';
filename = strcat(id,'_', type); % BraTS19_2013_2_1_seg_flair.nii
outpath = strcat(outfolder, filename);
%niftiwrite(cp_flair, outpath);
%whos seg
%extract = seg(84, :, 86);
% cp84 = cp_flair(84,:,86);
% flair84 = flair(84,:, 86);
%[flair,info] = ReadData3D(filename)
% whos flair
%volumeViewer(flair);
\ No newline at end of file
inputheader = '..\data\MICCAI_BraTS_2019_Data_Training\HGG\';
outfolder = strcat('..\data\MICCAI_BraTS_2019_Data_Training\HGG_seg_flair\');
files = dir(inputheader);
id = {files.name};
% files + dir dir
dirFlag = [files.isdir] & ~strcmp(id, '.') & ~strcmp(id, '..');
subFolders = files(dirFlag);
disp(length(subFolders));
% for k = 1 : length(subFolders)
% fprintf('Sub folder #%d = %s\n', k, subFolders(k).name);
% end
for i = 1 : length(subFolders)
id = subFolders(i).name;
fprintf('Sub folder #%d = %s\n', i, id);
type = 'flair.nii';
filename = strcat(id,'_', type); % BraTS19_2013_2_1_flair.nii
flair_path = strcat(inputheader, id, '\', filename,'\', filename);
flair = niftiread(flair_path); %size 240x240x155
cp_flair = flair;
type = 'seg.nii';
filename = strcat(id,'_', type); % BraTS19_2013_2_1_seg.nii
seg_path = strcat(inputheader, id, '\', filename, '\', filename);
seg = niftiread(seg_path);
[x,y,z] = size(seg);
% copy flair, segment flair data
cp_flair(seg == 0) = 0;
% save a segmented data
type = 'seg_flair.nii';
filename = strcat(id,'_', type); % BraTS19_2013_2_1_seg_flair.nii
outpath = strcat(outfolder, filename);
niftiwrite(cp_flair, outpath);
end
inputheader = '..\data\MICCAI_BraTS_2019_Data_Training\HGG\';
outfolder = '..\testfolder\';
id = 'BraTS19_2013_2_1';
type = 'seg_flair.nii';
filename = strcat(id,'_', type); % BraTS19_2013_2_1_flair.nii
flair_path = strcat(outfolder, filename);
%disp(path);
ffffff = niftiread(flair_path); %size 240x240x155
fff84 = ffffff(84, :, 86);
\ No newline at end of file