flair2seg & filter normal

+<?xml version="1.0" encoding="UTF-8"?>
+<module type="PYTHON_MODULE" version="4">
+  <component name="NewModuleRootManager">
+    <content url="file://$MODULE_DIR$" />
+    <orderEntry type="inheritedJdk" />
+    <orderEntry type="sourceFolder" forTests="false" />
+  </component>
+  <component name="TestRunnerService">
+    <option name="PROJECT_TEST_RUNNER" value="Unittests" />
+  </component>
+</module>
\ No newline at end of file

+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="Encoding" addBOMForNewFiles="with NO BOM" />
+</project>
\ No newline at end of file

+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="JavaScriptSettings">
+    <option name="languageLevel" value="ES6" />
+  </component>
+  <component name="ProjectRootManager" version="2" project-jdk-name="Python 3.7 (code) (3)" project-jdk-type="Python SDK" />
+</project>
\ No newline at end of file

+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="ProjectModuleManager">
+    <modules>
+      <module fileurl="file://$PROJECT_DIR$/.idea/code.iml" filepath="$PROJECT_DIR$/.idea/code.iml" />
+    </modules>
+  </component>
+</project>
\ No newline at end of file

+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="ChangeListManager">
+    <list default="true" id="38bbd81d-cf79-4a3f-8979-7c3ceb27bc32" name="Default Changelist" comment="" />
+    <option name="EXCLUDED_CONVERTED_TO_IGNORED" value="true" />
+    <option name="SHOW_DIALOG" value="false" />
+    <option name="HIGHLIGHT_CONFLICTS" value="true" />
+    <option name="HIGHLIGHT_NON_ACTIVE_CHANGELIST" value="false" />
+    <option name="LAST_RESOLUTION" value="IGNORE" />
+  </component>
+  <component name="ProjectFrameBounds">
+    <option name="x" value="-9" />
+    <option name="width" value="1825" />
+    <option name="height" value="1039" />
+  </component>
+  <component name="ProjectView">
+    <navigator proportions="" version="1">
+      <foldersAlwaysOnTop value="true" />
+    </navigator>
+    <panes>
+      <pane id="Scope" />
+      <pane id="ProjectPane" />
+    </panes>
+  </component>
+  <component name="PropertiesComponent">
+    <property name="WebServerToolWindowFactoryState" value="false" />
+    <property name="nodejs_interpreter_path.stuck_in_default_project" value="undefined stuck path" />
+    <property name="nodejs_npm_path_reset_for_default_project" value="true" />
+    <property name="settings.editor.selected.configurable" value="com.jetbrains.python.configuration.PyActiveSdkModuleConfigurable" />
+  </component>
+  <component name="RunDashboard">
+    <option name="ruleStates">
+      <list>
+        <RuleState>
+          <option name="name" value="ConfigurationTypeDashboardGroupingRule" />
+        </RuleState>
+        <RuleState>
+          <option name="name" value="StatusDashboardGroupingRule" />
+        </RuleState>
+      </list>
+    </option>
+  </component>
+  <component name="SvnConfiguration">
+    <configuration />
+  </component>
+  <component name="TaskManager">
+    <task active="true" id="Default" summary="Default task">
+      <changelist id="38bbd81d-cf79-4a3f-8979-7c3ceb27bc32" name="Default Changelist" comment="" />
+      <created>1584871834688</created>
+      <option name="number" value="Default" />
+      <option name="presentableId" value="Default" />
+      <updated>1584871834688</updated>
+      <workItem from="1584871840396" duration="72000" />
+    </task>
+    <servers />
+  </component>
+  <component name="TimeTrackingManager">
+    <option name="totallyTimeSpent" value="72000" />
+  </component>
+  <component name="ToolWindowManager">
+    <frame x="-7" y="0" width="1460" height="831" extended-state="0" />
+    <layout>
+      <window_info id="Favorites" side_tool="true" />
+      <window_info active="true" content_ui="combo" id="Project" order="0" visible="true" weight="0.25" />
+      <window_info id="Structure" order="1" side_tool="true" weight="0.25" />
+      <window_info anchor="bottom" id="Docker" show_stripe_button="false" />
+      <window_info anchor="bottom" id="Database Changes" />
+      <window_info anchor="bottom" id="Version Control" />
+      <window_info anchor="bottom" id="Python Console" />
+      <window_info anchor="bottom" id="Terminal" />
+      <window_info anchor="bottom" id="Event Log" side_tool="true" />
+      <window_info anchor="bottom" id="Message" order="0" />
+      <window_info anchor="bottom" id="Find" order="1" />
+      <window_info anchor="bottom" id="Run" order="2" />
+      <window_info anchor="bottom" id="Debug" order="3" weight="0.4" />
+      <window_info anchor="bottom" id="Cvs" order="4" weight="0.25" />
+      <window_info anchor="bottom" id="Inspection" order="5" weight="0.4" />
+      <window_info anchor="bottom" id="TODO" order="6" />
+      <window_info anchor="right" id="SciView" />
+      <window_info anchor="right" id="Database" />
+      <window_info anchor="right" id="Commander" internal_type="SLIDING" order="0" type="SLIDING" weight="0.4" />
+      <window_info anchor="right" id="Ant Build" order="1" weight="0.25" />
+      <window_info anchor="right" content_ui="combo" id="Hierarchy" order="2" weight="0.25" />
+    </layout>
+  </component>
+  <component name="TypeScriptGeneratedFilesManager">
+    <option name="version" value="1" />
+  </component>
+</project>
\ No newline at end of file

+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import pandas as pd"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>id</th>\n",
+       "      <th>Label</th>\n",
+       "      <th>Subject</th>\n",
+       "      <th>Date</th>\n",
+       "      <th>Gender</th>\n",
+       "      <th>Age</th>\n",
+       "      <th>mmse</th>\n",
+       "      <th>ageAtEntry</th>\n",
+       "      <th>cdr</th>\n",
+       "      <th>commun</th>\n",
+       "      <th>...</th>\n",
+       "      <th>memory</th>\n",
+       "      <th>orient</th>\n",
+       "      <th>perscare</th>\n",
+       "      <th>apoe</th>\n",
+       "      <th>sumbox</th>\n",
+       "      <th>acsparnt</th>\n",
+       "      <th>height</th>\n",
+       "      <th>weight</th>\n",
+       "      <th>primStudy</th>\n",
+       "      <th>acsStudy</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>0</th>\n",
+       "      <td>/@WEBAPP/images/r.gif</td>\n",
+       "      <td>OAS30001_ClinicalData_d3025</td>\n",
+       "      <td>OAS30001</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>female</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>30.0</td>\n",
+       "      <td>65.149895</td>\n",
+       "      <td>0.0</td>\n",
+       "      <td>0.0</td>\n",
+       "      <td>...</td>\n",
+       "      <td>0.0</td>\n",
+       "      <td>0.0</td>\n",
+       "      <td>0.0</td>\n",
+       "      <td>23.0</td>\n",
+       "      <td>0.0</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>64.0</td>\n",
+       "      <td>180.0</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1</th>\n",
+       "      <td>/@WEBAPP/images/r.gif</td>\n",
+       "      <td>OAS30001_ClinicalData_d3977</td>\n",
+       "      <td>OAS30001</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>female</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>29.0</td>\n",
+       "      <td>65.149895</td>\n",
+       "      <td>0.0</td>\n",
+       "      <td>0.0</td>\n",
+       "      <td>...</td>\n",
+       "      <td>0.0</td>\n",
+       "      <td>0.0</td>\n",
+       "      <td>0.0</td>\n",
+       "      <td>23.0</td>\n",
+       "      <td>0.0</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2</th>\n",
+       "      <td>/@WEBAPP/images/r.gif</td>\n",
+       "      <td>OAS30001_ClinicalData_d3332</td>\n",
+       "      <td>OAS30001</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>female</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>30.0</td>\n",
+       "      <td>65.149895</td>\n",
+       "      <td>0.0</td>\n",
+       "      <td>0.0</td>\n",
+       "      <td>...</td>\n",
+       "      <td>0.0</td>\n",
+       "      <td>0.0</td>\n",
+       "      <td>0.0</td>\n",
+       "      <td>23.0</td>\n",
+       "      <td>0.0</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>63.0</td>\n",
+       "      <td>185.0</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>3</th>\n",
+       "      <td>/@WEBAPP/images/r.gif</td>\n",
+       "      <td>OAS30001_ClinicalData_d0000</td>\n",
+       "      <td>OAS30001</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>female</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>28.0</td>\n",
+       "      <td>65.149895</td>\n",
+       "      <td>0.0</td>\n",
+       "      <td>0.0</td>\n",
+       "      <td>...</td>\n",
+       "      <td>0.0</td>\n",
+       "      <td>0.0</td>\n",
+       "      <td>0.0</td>\n",
+       "      <td>23.0</td>\n",
+       "      <td>0.0</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>4</th>\n",
+       "      <td>/@WEBAPP/images/r.gif</td>\n",
+       "      <td>OAS30001_ClinicalData_d1456</td>\n",
+       "      <td>OAS30001</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>female</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>30.0</td>\n",
+       "      <td>65.149895</td>\n",
+       "      <td>0.0</td>\n",
+       "      <td>0.0</td>\n",
+       "      <td>...</td>\n",
+       "      <td>0.0</td>\n",
+       "      <td>0.0</td>\n",
+       "      <td>0.0</td>\n",
+       "      <td>23.0</td>\n",
+       "      <td>0.0</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>63.0</td>\n",
+       "      <td>173.0</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "<p>5 rows × 27 columns</p>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "                      id                        Label   Subject  Date  Gender  \\\n",
+       "0  /@WEBAPP/images/r.gif  OAS30001_ClinicalData_d3025  OAS30001   NaN  female   \n",
+       "1  /@WEBAPP/images/r.gif  OAS30001_ClinicalData_d3977  OAS30001   NaN  female   \n",
+       "2  /@WEBAPP/images/r.gif  OAS30001_ClinicalData_d3332  OAS30001   NaN  female   \n",
+       "3  /@WEBAPP/images/r.gif  OAS30001_ClinicalData_d0000  OAS30001   NaN  female   \n",
+       "4  /@WEBAPP/images/r.gif  OAS30001_ClinicalData_d1456  OAS30001   NaN  female   \n",
+       "\n",
+       "   Age  mmse  ageAtEntry  cdr  commun  ... memory orient perscare  apoe  \\\n",
+       "0  NaN  30.0   65.149895  0.0     0.0  ...    0.0    0.0      0.0  23.0   \n",
+       "1  NaN  29.0   65.149895  0.0     0.0  ...    0.0    0.0      0.0  23.0   \n",
+       "2  NaN  30.0   65.149895  0.0     0.0  ...    0.0    0.0      0.0  23.0   \n",
+       "3  NaN  28.0   65.149895  0.0     0.0  ...    0.0    0.0      0.0  23.0   \n",
+       "4  NaN  30.0   65.149895  0.0     0.0  ...    0.0    0.0      0.0  23.0   \n",
+       "\n",
+       "  sumbox  acsparnt  height  weight  primStudy  acsStudy  \n",
+       "0    0.0       NaN    64.0   180.0        NaN       NaN  \n",
+       "1    0.0       NaN     NaN     NaN        NaN       NaN  \n",
+       "2    0.0       NaN    63.0   185.0        NaN       NaN  \n",
+       "3    0.0       NaN     NaN     NaN        NaN       NaN  \n",
+       "4    0.0       NaN    63.0   173.0        NaN       NaN  \n",
+       "\n",
+       "[5 rows x 27 columns]"
+      ]
+     },
+     "execution_count": 3,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "all_data = pd.read_csv(\"..\\data\\ADRC clinical data_all.csv\")\n",
+    "\n",
+    "all_data.head()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 18,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Subject\n",
+      "OAS30001    0.0\n",
+      "OAS30002    0.0\n",
+      "OAS30003    0.0\n",
+      "OAS30004    0.0\n",
+      "OAS30005    0.0\n",
+      "           ... \n",
+      "OAS31168    0.0\n",
+      "OAS31169    3.0\n",
+      "OAS31170    2.0\n",
+      "OAS31171    2.0\n",
+      "OAS31172    0.0\n",
+      "Name: cdr, Length: 1098, dtype: float64\n"
+     ]
+    }
+   ],
+   "source": [
+    "ad = all_data.groupby(['Subject'])['cdr'].max()\n",
+    "print(ad)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 21,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "'OAS30001'"
+      ]
+     },
+     "execution_count": 21,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "ad.index[0]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 22,
+   "metadata": {},
+   "outputs": [
+    {
+     "ename": "SyntaxError",
+     "evalue": "unexpected EOF while parsing (<ipython-input-22-b8a078b72aca>, line 5)",
+     "output_type": "error",
+     "traceback": [
+      "\u001b[1;36m  File \u001b[1;32m\"<ipython-input-22-b8a078b72aca>\"\u001b[1;36m, line \u001b[1;32m5\u001b[0m\n\u001b[1;33m    #print(filtered)\u001b[0m\n\u001b[1;37m                    ^\u001b[0m\n\u001b[1;31mSyntaxError\u001b[0m\u001b[1;31m:\u001b[0m unexpected EOF while parsing\n"
+     ]
+    }
+   ],
+   "source": [
+    "filtered = []\n",
+    "for i, val in enumerate(ad):\n",
+    "    if ad[i] == 0:\n",
+    "        filtered.append(ad.index[i])\n",
+    "#print(filtered)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 23,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "df_filtered = pd.DataFrame(filtered)\n",
+    "df_filtered.to_csv('..\\data\\ADRC clinical data_normal.csv')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "ML",
+   "language": "python",
+   "name": "ml"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.7.4"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+.hypothesis/
+.pytest_cache/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# pyenv
+.python-version
+
+# celery beat schedule file
+celerybeat-schedule
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/

+"""
+Reference :
+- https://github.com/hysts/pytorch_image_classification/blob/master/augmentations/mixup.py
+- https://github.com/tensorflow/tpu/blob/master/models/official/efficientnet/imagenet_input.py#L120
+"""
+
+import numpy as np
+import torch
+
+from FastAutoAugment.metrics import CrossEntropyLabelSmooth
+
+
+def mixup(data, targets, alpha):
+    indices = torch.randperm(data.size(0))
+    shuffled_data = data[indices]
+    shuffled_targets = targets[indices]
+
+    lam = np.random.beta(alpha, alpha)
+    lam = max(lam, 1. - lam)
+    assert 0.0 <= lam <= 1.0, lam
+    data = data * lam + shuffled_data * (1 - lam)
+
+    return data, targets, shuffled_targets, lam
+
+
+class CrossEntropyMixUpLabelSmooth(torch.nn.Module):
+    def __init__(self, num_classes, epsilon, reduction='mean'):
+        super(CrossEntropyMixUpLabelSmooth, self).__init__()
+        self.ce = CrossEntropyLabelSmooth(num_classes, epsilon, reduction=reduction)
+
+    def forward(self, input, target1, target2, lam):  # pylint: disable=redefined-builtin
+        return lam * self.ce(input, target1) + (1 - lam) * self.ce(input, target2)

+# code in this file is adpated from rpmcruz/autoaugment
+# https://github.com/rpmcruz/autoaugment/blob/master/transformations.py
+import random
+
+import PIL, PIL.ImageOps, PIL.ImageEnhance, PIL.ImageDraw
+import numpy as np
+import torch
+from torchvision.transforms.transforms import Compose
+
+random_mirror = True
+
+
+def ShearX(img, v):  # [-0.3, 0.3]
+    assert -0.3 <= v <= 0.3
+    if random_mirror and random.random() > 0.5:
+        v = -v
+    return img.transform(img.size, PIL.Image.AFFINE, (1, v, 0, 0, 1, 0))
+
+
+def ShearY(img, v):  # [-0.3, 0.3]
+    assert -0.3 <= v <= 0.3
+    if random_mirror and random.random() > 0.5:
+        v = -v
+    return img.transform(img.size, PIL.Image.AFFINE, (1, 0, 0, v, 1, 0))
+
+
+def TranslateX(img, v):  # [-150, 150] => percentage: [-0.45, 0.45]
+    assert -0.45 <= v <= 0.45
+    if random_mirror and random.random() > 0.5:
+        v = -v
+    v = v * img.size[0]
+    return img.transform(img.size, PIL.Image.AFFINE, (1, 0, v, 0, 1, 0))
+
+
+def TranslateY(img, v):  # [-150, 150] => percentage: [-0.45, 0.45]
+    assert -0.45 <= v <= 0.45
+    if random_mirror and random.random() > 0.5:
+        v = -v
+    v = v * img.size[1]
+    return img.transform(img.size, PIL.Image.AFFINE, (1, 0, 0, 0, 1, v))
+
+
+def TranslateXAbs(img, v):  # [-150, 150] => percentage: [-0.45, 0.45]
+    assert 0 <= v <= 10
+    if random.random() > 0.5:
+        v = -v
+    return img.transform(img.size, PIL.Image.AFFINE, (1, 0, v, 0, 1, 0))
+
+
+def TranslateYAbs(img, v):  # [-150, 150] => percentage: [-0.45, 0.45]
+    assert 0 <= v <= 10
+    if random.random() > 0.5:
+        v = -v
+    return img.transform(img.size, PIL.Image.AFFINE, (1, 0, 0, 0, 1, v))
+
+
+def Rotate(img, v):  # [-30, 30]
+    assert -30 <= v <= 30
+    if random_mirror and random.random() > 0.5:
+        v = -v
+    return img.rotate(v)
+
+
+def AutoContrast(img, _):
+    return PIL.ImageOps.autocontrast(img)
+
+
+def Invert(img, _):
+    return PIL.ImageOps.invert(img)
+
+
+def Equalize(img, _):
+    return PIL.ImageOps.equalize(img)
+
+
+def Flip(img, _):  # not from the paper
+    return PIL.ImageOps.mirror(img)
+
+
+def Solarize(img, v):  # [0, 256]
+    assert 0 <= v <= 256
+    return PIL.ImageOps.solarize(img, v)
+
+
+def Posterize(img, v):  # [4, 8]
+    assert 4 <= v <= 8
+    v = int(v)
+    return PIL.ImageOps.posterize(img, v)
+
+
+def Posterize2(img, v):  # [0, 4]
+    assert 0 <= v <= 4
+    v = int(v)
+    return PIL.ImageOps.posterize(img, v)
+
+
+def Contrast(img, v):  # [0.1,1.9]
+    assert 0.1 <= v <= 1.9
+    return PIL.ImageEnhance.Contrast(img).enhance(v)
+
+
+def Color(img, v):  # [0.1,1.9]
+    assert 0.1 <= v <= 1.9
+    return PIL.ImageEnhance.Color(img).enhance(v)
+
+
+def Brightness(img, v):  # [0.1,1.9]
+    assert 0.1 <= v <= 1.9
+    return PIL.ImageEnhance.Brightness(img).enhance(v)
+
+
+def Sharpness(img, v):  # [0.1,1.9]
+    assert 0.1 <= v <= 1.9
+    return PIL.ImageEnhance.Sharpness(img).enhance(v)
+
+
+def Cutout(img, v):  # [0, 60] => percentage: [0, 0.2]
+    assert 0.0 <= v <= 0.2
+    if v <= 0.:
+        return img
+
+    v = v * img.size[0]
+    return CutoutAbs(img, v)
+
+
+def CutoutAbs(img, v):  # [0, 60] => percentage: [0, 0.2]
+    # assert 0 <= v <= 20
+    if v < 0:
+        return img
+    w, h = img.size
+    x0 = np.random.uniform(w)
+    y0 = np.random.uniform(h)
+
+    x0 = int(max(0, x0 - v / 2.))
+    y0 = int(max(0, y0 - v / 2.))
+    x1 = min(w, x0 + v)
+    y1 = min(h, y0 + v)
+
+    xy = (x0, y0, x1, y1)
+    color = (125, 123, 114)
+    # color = (0, 0, 0)
+    img = img.copy()
+    PIL.ImageDraw.Draw(img).rectangle(xy, color)
+    return img
+
+
+def SamplePairing(imgs):  # [0, 0.4]
+    def f(img1, v):
+        i = np.random.choice(len(imgs))
+        img2 = PIL.Image.fromarray(imgs[i])
+        return PIL.Image.blend(img1, img2, v)
+
+    return f
+
+
+def augment_list(for_autoaug=True):  # 16 oeprations and their ranges
+    l = [
+        (ShearX, -0.3, 0.3),  # 0
+        (ShearY, -0.3, 0.3),  # 1
+        (TranslateX, -0.45, 0.45),  # 2
+        (TranslateY, -0.45, 0.45),  # 3
+        (Rotate, -30, 30),  # 4
+        (AutoContrast, 0, 1),  # 5
+        (Invert, 0, 1),  # 6
+        (Equalize, 0, 1),  # 7
+        (Solarize, 0, 256),  # 8
+        (Posterize, 4, 8),  # 9
+        (Contrast, 0.1, 1.9),  # 10
+        (Color, 0.1, 1.9),  # 11
+        (Brightness, 0.1, 1.9),  # 12
+        (Sharpness, 0.1, 1.9),  # 13
+        (Cutout, 0, 0.2),  # 14
+        # (SamplePairing(imgs), 0, 0.4),  # 15
+    ]
+    if for_autoaug:
+        l += [
+            (CutoutAbs, 0, 20),  # compatible with auto-augment
+            (Posterize2, 0, 4),  # 9
+            (TranslateXAbs, 0, 10),  # 9
+            (TranslateYAbs, 0, 10),  # 9
+        ]
+    return l
+
+
+augment_dict = {fn.__name__: (fn, v1, v2) for fn, v1, v2 in augment_list()}
+
+
+def get_augment(name):
+    return augment_dict[name]
+
+
+def apply_augment(img, name, level):
+    augment_fn, low, high = get_augment(name)
+    return augment_fn(img.copy(), level * (high - low) + low)
+
+
+class Lighting(object):
+    """Lighting noise(AlexNet - style PCA - based noise)"""
+
+    def __init__(self, alphastd, eigval, eigvec):
+        self.alphastd = alphastd
+        self.eigval = torch.Tensor(eigval)
+        self.eigvec = torch.Tensor(eigvec)
+
+    def __call__(self, img):
+        if self.alphastd == 0:
+            return img
+
+        alpha = img.new().resize_(3).normal_(0, self.alphastd)
+        rgb = self.eigvec.type_as(img).clone() \
+            .mul(alpha.view(1, 3).expand(3, 3)) \
+            .mul(self.eigval.view(1, 3).expand(3, 3)) \
+            .sum(1).squeeze()
+
+        return img.add(rgb.view(3, 1, 1).expand_as(img))

+import copy
+import logging
+import warnings
+
+formatter = logging.Formatter('[%(asctime)s] [%(name)s] [%(levelname)s] %(message)s')
+warnings.filterwarnings("ignore", "(Possibly )?corrupt EXIF data", UserWarning)
+warnings.filterwarnings("ignore", "DeprecationWarning: 'saved_variables' is deprecated", UserWarning)
+
+
+def get_logger(name, level=logging.DEBUG):
+    logger = logging.getLogger(name)
+    logger.handlers.clear()
+    logger.setLevel(level)
+    ch = logging.StreamHandler()
+    ch.setLevel(level)
+    ch.setFormatter(formatter)
+    logger.addHandler(ch)
+    return logger
+
+
+def add_filehandler(logger, filepath, level=logging.DEBUG):
+    fh = logging.FileHandler(filepath)
+    fh.setLevel(level)
+    fh.setFormatter(formatter)
+    logger.addHandler(fh)
+
+
+class EMA:
+    def __init__(self, mu):
+        self.mu = mu
+        self.shadow = {}
+
+    def state_dict(self):
+        return copy.deepcopy(self.shadow)
+
+    def __len__(self):
+        return len(self.shadow)
+
+    def __call__(self, module, step=None):
+        if step is None:
+            mu = self.mu
+        else:
+            # see : https://www.tensorflow.org/versions/r1.15/api_docs/python/tf/train/ExponentialMovingAverage?hl=PL
+            mu = min(self.mu, (1. + step) / (10 + step))
+
+        for name, x in module.state_dict().items():
+            if name in self.shadow:
+                new_average = (1.0 - mu) * x + mu * self.shadow[name]
+                self.shadow[name] = new_average.clone()
+            else:
+                self.shadow[name] = x.clone()

+from __future__ import print_function
+import os
+import shutil
+import torch
+
+ARCHIVE_DICT = {
+    'train': {
+        'url': 'http://www.image-net.org/challenges/LSVRC/2012/nnoupb/ILSVRC2012_img_train.tar',
+        'md5': '1d675b47d978889d74fa0da5fadfb00e',
+    },
+    'val': {
+        'url': 'http://www.image-net.org/challenges/LSVRC/2012/nnoupb/ILSVRC2012_img_val.tar',
+        'md5': '29b22e2961454d5413ddabcf34fc5622',
+    },
+    'devkit': {
+        'url': 'http://www.image-net.org/challenges/LSVRC/2012/nnoupb/ILSVRC2012_devkit_t12.tar.gz',
+        'md5': 'fa75699e90414af021442c21a62c3abf',
+    }
+}
+
+
+import torchvision
+from torchvision.datasets.utils import check_integrity, download_url
+
+
+# copy ILSVRC/ImageSets/CLS-LOC/train_cls.txt to ./root/
+# to skip os walk (it's too slow) using ILSVRC/ImageSets/CLS-LOC/train_cls.txt file
+class ImageNet(torchvision.datasets.ImageFolder):
+    """`ImageNet <http://image-net.org/>`_ 2012 Classification Dataset.
+
+    Args:
+        root (string): Root directory of the ImageNet Dataset.
+        split (string, optional): The dataset split, supports ``train``, or ``val``.
+        download (bool, optional): If true, downloads the dataset from the internet and
+            puts it in root directory. If dataset is already downloaded, it is not
+            downloaded again.
+        transform (callable, optional): A function/transform that  takes in an PIL image
+            and returns a transformed version. E.g, ``transforms.RandomCrop``
+        target_transform (callable, optional): A function/transform that takes in the
+            target and transforms it.
+        loader (callable, optional): A function to load an image given its path.
+
+     Attributes:
+        classes (list): List of the class names.
+        class_to_idx (dict): Dict with items (class_name, class_index).
+        wnids (list): List of the WordNet IDs.
+        wnid_to_idx (dict): Dict with items (wordnet_id, class_index).
+        imgs (list): List of (image path, class_index) tuples
+        targets (list): The class_index value for each image in the dataset
+    """
+
+    def __init__(self, root, split='train', download=False, **kwargs):
+        root = self.root = os.path.expanduser(root)
+        self.split = self._verify_split(split)
+
+        if download:
+            self.download()
+        wnid_to_classes = self._load_meta_file()[0]
+
+        # to skip os walk (it's too slow) using ILSVRC/ImageSets/CLS-LOC/train_cls.txt file
+        listfile = os.path.join(root, 'train_cls.txt')
+        if split == 'train' and os.path.exists(listfile):
+            torchvision.datasets.VisionDataset.__init__(self, root, **kwargs)
+            with open(listfile, 'r') as f:
+                datalist = [
+                    line.strip().split(' ')[0]
+                    for line in f.readlines()
+                    if line.strip()
+                ]
+
+            classes = list(set([line.split('/')[0] for line in datalist]))
+            classes.sort()
+            class_to_idx = {classes[i]: i for i in range(len(classes))}
+
+            samples = [
+                (os.path.join(self.split_folder, line + '.JPEG'), class_to_idx[line.split('/')[0]])
+                for line in datalist
+            ]
+
+            self.loader = torchvision.datasets.folder.default_loader
+            self.extensions = torchvision.datasets.folder.IMG_EXTENSIONS
+
+            self.classes = classes
+            self.class_to_idx = class_to_idx
+            self.samples = samples
+            self.targets = [s[1] for s in samples]
+
+            self.imgs = self.samples
+        else:
+            super(ImageNet, self).__init__(self.split_folder, **kwargs)
+
+        self.root = root
+
+        idcs = [idx for _, idx in self.imgs]
+        self.wnids = self.classes
+        self.wnid_to_idx = {wnid: idx for idx, wnid in zip(idcs, self.wnids)}
+        self.classes = [wnid_to_classes[wnid] for wnid in self.wnids]
+        self.class_to_idx = {cls: idx
+                             for clss, idx in zip(self.classes, idcs)
+                             for cls in clss}
+
+    def download(self):
+        if not check_integrity(self.meta_file):
+            tmpdir = os.path.join(self.root, 'tmp')
+
+            archive_dict = ARCHIVE_DICT['devkit']
+            download_and_extract_tar(archive_dict['url'], self.root,
+                                     extract_root=tmpdir,
+                                     md5=archive_dict['md5'])
+            devkit_folder = _splitexts(os.path.basename(archive_dict['url']))[0]
+            meta = parse_devkit(os.path.join(tmpdir, devkit_folder))
+            self._save_meta_file(*meta)
+
+            shutil.rmtree(tmpdir)
+
+        if not os.path.isdir(self.split_folder):
+            archive_dict = ARCHIVE_DICT[self.split]
+            download_and_extract_tar(archive_dict['url'], self.root,
+                                     extract_root=self.split_folder,
+                                     md5=archive_dict['md5'])
+
+            if self.split == 'train':
+                prepare_train_folder(self.split_folder)
+            elif self.split == 'val':
+                val_wnids = self._load_meta_file()[1]
+                prepare_val_folder(self.split_folder, val_wnids)
+        else:
+            msg = ("You set download=True, but a folder '{}' already exist in "
+                   "the root directory. If you want to re-download or re-extract the "
+                   "archive, delete the folder.")
+            print(msg.format(self.split))
+
+    @property
+    def meta_file(self):
+        return os.path.join(self.root, 'meta.bin')
+
+    def _load_meta_file(self):
+        if check_integrity(self.meta_file):
+            return torch.load(self.meta_file)
+        raise RuntimeError("Meta file not found or corrupted.",
+                           "You can use download=True to create it.")
+
+    def _save_meta_file(self, wnid_to_class, val_wnids):
+        torch.save((wnid_to_class, val_wnids), self.meta_file)
+
+    def _verify_split(self, split):
+        if split not in self.valid_splits:
+            msg = "Unknown split {} .".format(split)
+            msg += "Valid splits are {{}}.".format(", ".join(self.valid_splits))
+            raise ValueError(msg)
+        return split
+
+    @property
+    def valid_splits(self):
+        return 'train', 'val'
+
+    @property
+    def split_folder(self):
+        return os.path.join(self.root, self.split)
+
+    def extra_repr(self):
+        return "Split: {split}".format(**self.__dict__)
+
+
+def extract_tar(src, dest=None, gzip=None, delete=False):
+    import tarfile
+
+    if dest is None:
+        dest = os.path.dirname(src)
+    if gzip is None:
+        gzip = src.lower().endswith('.gz')
+
+    mode = 'r:gz' if gzip else 'r'
+    with tarfile.open(src, mode) as tarfh:
+        tarfh.extractall(path=dest)
+
+    if delete:
+        os.remove(src)
+
+
+def download_and_extract_tar(url, download_root, extract_root=None, filename=None,
+                             md5=None, **kwargs):
+    download_root = os.path.expanduser(download_root)
+    if extract_root is None:
+        extract_root = download_root
+    if filename is None:
+        filename = os.path.basename(url)
+
+    if not check_integrity(os.path.join(download_root, filename), md5):
+        download_url(url, download_root, filename=filename, md5=md5)
+
+    extract_tar(os.path.join(download_root, filename), extract_root, **kwargs)
+
+
+def parse_devkit(root):
+    idx_to_wnid, wnid_to_classes = parse_meta(root)
+    val_idcs = parse_val_groundtruth(root)
+    val_wnids = [idx_to_wnid[idx] for idx in val_idcs]
+    return wnid_to_classes, val_wnids
+
+
+def parse_meta(devkit_root, path='data', filename='meta.mat'):
+    import scipy.io as sio
+
+    metafile = os.path.join(devkit_root, path, filename)
+    meta = sio.loadmat(metafile, squeeze_me=True)['synsets']
+    nums_children = list(zip(*meta))[4]
+    meta = [meta[idx] for idx, num_children in enumerate(nums_children)
+            if num_children == 0]
+    idcs, wnids, classes = list(zip(*meta))[:3]
+    classes = [tuple(clss.split(', ')) for clss in classes]
+    idx_to_wnid = {idx: wnid for idx, wnid in zip(idcs, wnids)}
+    wnid_to_classes = {wnid: clss for wnid, clss in zip(wnids, classes)}
+    return idx_to_wnid, wnid_to_classes
+
+
+def parse_val_groundtruth(devkit_root, path='data',
+                          filename='ILSVRC2012_validation_ground_truth.txt'):
+    with open(os.path.join(devkit_root, path, filename), 'r') as txtfh:
+        val_idcs = txtfh.readlines()
+    return [int(val_idx) for val_idx in val_idcs]
+
+
+def prepare_train_folder(folder):
+    for archive in [os.path.join(folder, archive) for archive in os.listdir(folder)]:
+        extract_tar(archive, os.path.splitext(archive)[0], delete=True)
+
+
+def prepare_val_folder(folder, wnids):
+    img_files = sorted([os.path.join(folder, file) for file in os.listdir(folder)])
+
+    for wnid in set(wnids):
+        os.mkdir(os.path.join(folder, wnid))
+
+    for wnid, img_file in zip(wnids, img_files):
+        shutil.move(img_file, os.path.join(folder, wnid, os.path.basename(img_file)))
+
+
+def _splitexts(root):
+    exts = []
+    ext = '.'
+    while ext:
+        root, ext = os.path.splitext(root)
+        exts.append(ext)
+    return root, ''.join(reversed(exts))

+import torch
+
+from theconf import Config as C
+
+
+def adjust_learning_rate_resnet(optimizer):
+    """
+    Sets the learning rate to the initial LR decayed by 10 on every predefined epochs
+    Ref: AutoAugment
+    """
+
+    if C.get()['epoch'] == 90:
+        return torch.optim.lr_scheduler.MultiStepLR(optimizer, [30, 60, 80])
+    elif C.get()['epoch'] == 270:   # autoaugment
+        return torch.optim.lr_scheduler.MultiStepLR(optimizer, [90, 180, 240])
+    else:
+        raise ValueError('invalid epoch=%d for resnet scheduler' % C.get()['epoch'])

+import copy
+
+import torch
+import numpy as np
+from collections import defaultdict
+
+from torch import nn
+
+
+def accuracy(output, target, topk=(1,)):
+    """Computes the precision@k for the specified values of k"""
+    maxk = max(topk)
+    batch_size = target.size(0)
+
+    _, pred = output.topk(maxk, 1, True, True)
+    pred = pred.t()
+    correct = pred.eq(target.view(1, -1).expand_as(pred))
+
+    res = []
+    for k in topk:
+        correct_k = correct[:k].view(-1).float().sum(0)
+        res.append(correct_k.mul_(1. / batch_size))
+    return res
+
+
+class CrossEntropyLabelSmooth(torch.nn.Module):
+    def __init__(self, num_classes, epsilon, reduction='mean'):
+        super(CrossEntropyLabelSmooth, self).__init__()
+        self.num_classes = num_classes
+        self.epsilon = epsilon
+        self.reduction = reduction
+        self.logsoftmax = torch.nn.LogSoftmax(dim=1)
+
+    def forward(self, input, target):  # pylint: disable=redefined-builtin
+        log_probs = self.logsoftmax(input)
+        targets = torch.zeros_like(log_probs).scatter_(1, target.unsqueeze(1), 1)
+        if self.epsilon > 0.0:
+            targets = (1 - self.epsilon) * targets + self.epsilon / self.num_classes
+        targets = targets.detach()
+        loss = (-targets * log_probs)
+
+        if self.reduction in ['avg', 'mean']:
+            loss = torch.mean(torch.sum(loss, dim=1))
+        elif self.reduction == 'sum':
+            loss = loss.sum()
+        return loss
+
+
+class Accumulator:
+    def __init__(self):
+        self.metrics = defaultdict(lambda: 0.)
+
+    def add(self, key, value):
+        self.metrics[key] += value
+
+    def add_dict(self, dict):
+        for key, value in dict.items():
+            self.add(key, value)
+
+    def __getitem__(self, item):
+        return self.metrics[item]
+
+    def __setitem__(self, key, value):
+        self.metrics[key] = value
+
+    def get_dict(self):
+        return copy.deepcopy(dict(self.metrics))
+
+    def items(self):
+        return self.metrics.items()
+
+    def __str__(self):
+        return str(dict(self.metrics))
+
+    def __truediv__(self, other):
+        newone = Accumulator()
+        for key, value in self.items():
+            if isinstance(other, str):
+                if other != key:
+                    newone[key] = value / self[other]
+                else:
+                    newone[key] = value
+            else:
+                newone[key] = value / other
+        return newone
+
+
+class SummaryWriterDummy:
+    def __init__(self, log_dir):
+        pass
+
+    def add_scalar(self, *args, **kwargs):
+        pass

+import torch
+
+from torch import nn
+from torch.nn import DataParallel
+from torch.nn.parallel import DistributedDataParallel
+import torch.backends.cudnn as cudnn
+# from torchvision import models
+import numpy as np
+
+from FastAutoAugment.networks.resnet import ResNet
+from FastAutoAugment.networks.pyramidnet import PyramidNet
+from FastAutoAugment.networks.shakeshake.shake_resnet import ShakeResNet
+from FastAutoAugment.networks.wideresnet import WideResNet
+from FastAutoAugment.networks.shakeshake.shake_resnext import ShakeResNeXt
+from FastAutoAugment.networks.efficientnet_pytorch import EfficientNet, RoutingFn
+from FastAutoAugment.tf_port.tpu_bn import TpuBatchNormalization
+
+
+def get_model(conf, num_class=10, local_rank=-1):
+    name = conf['type']
+
+    if name == 'resnet50':
+        model = ResNet(dataset='imagenet', depth=50, num_classes=num_class, bottleneck=True)
+    elif name == 'resnet200':
+        model = ResNet(dataset='imagenet', depth=200, num_classes=num_class, bottleneck=True)
+    elif name == 'wresnet40_2':
+        model = WideResNet(40, 2, dropout_rate=0.0, num_classes=num_class)
+    elif name == 'wresnet28_10':
+        model = WideResNet(28, 10, dropout_rate=0.0, num_classes=num_class)
+
+    elif name == 'shakeshake26_2x32d':
+        model = ShakeResNet(26, 32, num_class)
+    elif name == 'shakeshake26_2x64d':
+        model = ShakeResNet(26, 64, num_class)
+    elif name == 'shakeshake26_2x96d':
+        model = ShakeResNet(26, 96, num_class)
+    elif name == 'shakeshake26_2x112d':
+        model = ShakeResNet(26, 112, num_class)
+
+    elif name == 'shakeshake26_2x96d_next':
+        model = ShakeResNeXt(26, 96, 4, num_class)
+
+    elif name == 'pyramid':
+        model = PyramidNet('cifar10', depth=conf['depth'], alpha=conf['alpha'], num_classes=num_class, bottleneck=conf['bottleneck'])
+
+    elif 'efficientnet' in name:
+        model = EfficientNet.from_name(name, condconv_num_expert=conf['condconv_num_expert'], norm_layer=None)  # TpuBatchNormalization
+        if local_rank >= 0:
+            model = nn.SyncBatchNorm.convert_sync_batchnorm(model)
+        def kernel_initializer(module):
+            def get_fan_in_out(module):
+                num_input_fmaps = module.weight.size(1)
+                num_output_fmaps = module.weight.size(0)
+                receptive_field_size = 1
+                if module.weight.dim() > 2:
+                    receptive_field_size = module.weight[0][0].numel()
+                fan_in = num_input_fmaps * receptive_field_size
+                fan_out = num_output_fmaps * receptive_field_size
+                return fan_in, fan_out
+
+            if isinstance(module, torch.nn.Conv2d):
+                # https://github.com/tensorflow/tpu/blob/master/models/official/efficientnet/efficientnet_model.py#L58
+                fan_in, fan_out = get_fan_in_out(module)
+                torch.nn.init.normal_(module.weight, mean=0.0, std=np.sqrt(2.0 / fan_out))
+                if module.bias is not None:
+                    torch.nn.init.constant_(module.bias, val=0.)
+            elif isinstance(module, RoutingFn):
+                torch.nn.init.xavier_uniform_(module.weight)
+                torch.nn.init.constant_(module.bias, val=0.)
+            elif isinstance(module, torch.nn.Linear):
+                # https://github.com/tensorflow/tpu/blob/master/models/official/efficientnet/efficientnet_model.py#L82
+                fan_in, fan_out = get_fan_in_out(module)
+                delta = 1.0 / np.sqrt(fan_out)
+                torch.nn.init.uniform_(module.weight, a=-delta, b=delta)
+                if module.bias is not None:
+                    torch.nn.init.constant_(module.bias, val=0.)
+        model.apply(kernel_initializer)
+    else:
+        raise NameError('no model named, %s' % name)
+
+    if local_rank >= 0:
+        device = torch.device('cuda', local_rank)
+        model = model.to(device)
+        model = DistributedDataParallel(model, device_ids=[local_rank], output_device=local_rank)
+    else:
+        model = model.cuda()
+#         model = DataParallel(model)
+
+    cudnn.benchmark = True
+    return model
+
+
+def num_class(dataset):
+    return {
+        'cifar10': 10,
+        'reduced_cifar10': 10,
+        'cifar10.1': 10,
+        'cifar100': 100,
+        'svhn': 10,
+        'reduced_svhn': 10,
+        'imagenet': 1000,
+        'reduced_imagenet': 120,
+    }[dataset]

+__version__ = "0.5.1"
+from .model import EfficientNet, RoutingFn
+from .utils import (
+    GlobalParams,
+    BlockArgs,
+    BlockDecoder,
+    efficientnet,
+    get_model_params,
+)
\ No newline at end of file

+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+from torch._six import container_abcs
+
+from itertools import repeat
+from functools import partial
+from typing import Union, List, Tuple, Optional, Callable
+import numpy as np
+import math
+
+
+def _ntuple(n):
+    def parse(x):
+        if isinstance(x, container_abcs.Iterable):
+            return x
+        return tuple(repeat(x, n))
+    return parse
+
+
+_single = _ntuple(1)
+_pair = _ntuple(2)
+_triple = _ntuple(3)
+_quadruple = _ntuple(4)
+
+
+def _is_static_pad(kernel_size, stride=1, dilation=1, **_):
+    return stride == 1 and (dilation * (kernel_size - 1)) % 2 == 0
+
+
+def _get_padding(kernel_size, stride=1, dilation=1, **_):
+    padding = ((stride - 1) + dilation * (kernel_size - 1)) // 2
+    return padding
+
+
+def _calc_same_pad(i: int, k: int, s: int, d: int):
+    return max((math.ceil(i / s) - 1) * s + (k - 1) * d + 1 - i, 0)
+
+
+def conv2d_same(
+        x, weight: torch.Tensor, bias: Optional[torch.Tensor] = None, stride: Tuple[int, int] = (1, 1),
+        padding: Tuple[int, int] = (0, 0), dilation: Tuple[int, int] = (1, 1), groups: int = 1):
+    ih, iw = x.size()[-2:]
+    kh, kw = weight.size()[-2:]
+    pad_h = _calc_same_pad(ih, kh, stride[0], dilation[0])
+    pad_w = _calc_same_pad(iw, kw, stride[1], dilation[1])
+    if pad_h > 0 or pad_w > 0:
+        x = F.pad(x, [pad_w // 2, pad_w - pad_w // 2, pad_h // 2, pad_h - pad_h // 2])
+    return F.conv2d(x, weight, bias, stride, (0, 0), dilation, groups)
+
+
+def get_padding_value(padding, kernel_size, **kwargs):
+    dynamic = False
+    if isinstance(padding, str):
+        # for any string padding, the padding will be calculated for you, one of three ways
+        padding = padding.lower()
+        if padding == 'same':
+            # TF compatible 'SAME' padding, has a performance and GPU memory allocation impact
+            if _is_static_pad(kernel_size, **kwargs):
+                # static case, no extra overhead
+                padding = _get_padding(kernel_size, **kwargs)
+            else:
+                # dynamic padding
+                padding = 0
+                dynamic = True
+        elif padding == 'valid':
+            # 'VALID' padding, same as padding=0
+            padding = 0
+        else:
+            # Default to PyTorch style 'same'-ish symmetric padding
+            padding = _get_padding(kernel_size, **kwargs)
+    return padding, dynamic
+
+
+def get_condconv_initializer(initializer, num_experts, expert_shape):
+    def condconv_initializer(weight):
+        """CondConv initializer function."""
+        num_params = np.prod(expert_shape)
+        if (len(weight.shape) != 2 or weight.shape[0] != num_experts or weight.shape[1] != num_params):
+            raise (ValueError('CondConv variables must have shape [num_experts, num_params]'))
+        for i in range(num_experts):
+            initializer(weight[i].view(expert_shape))
+    return condconv_initializer
+
+
+class CondConv2d(nn.Module):
+    """ Conditional Convolution
+    Inspired by: https://github.com/tensorflow/tpu/blob/master/models/official/efficientnet/condconv/condconv_layers.py
+    Grouped convolution hackery for parallel execution of the per-sample kernel filters inspired by this discussion:
+    https://github.com/pytorch/pytorch/issues/17983
+    """
+    __constants__ = ['bias', 'in_channels', 'out_channels', 'dynamic_padding']
+
+    def __init__(self, in_channels, out_channels, kernel_size=3,
+                 stride=1, padding='', dilation=1, groups=1, bias=False, num_experts=4):
+        super(CondConv2d, self).__init__()
+        assert num_experts > 1
+
+        if isinstance(stride, container_abcs.Iterable) and len(stride) == 1:
+            stride = stride[0]
+        # print('CondConv', num_experts)
+
+        self.in_channels = in_channels
+        self.out_channels = out_channels
+        self.kernel_size = _pair(kernel_size)
+        self.stride = _pair(stride)
+        padding_val, is_padding_dynamic = get_padding_value(padding, kernel_size, stride=stride, dilation=dilation)
+        self.dynamic_padding = is_padding_dynamic  # if in forward to work with torchscript
+        self.padding = _pair(padding_val)
+        self.dilation = _pair(dilation)
+        self.groups = groups
+        self.num_experts = num_experts
+
+        self.weight_shape = (self.out_channels, self.in_channels // self.groups) + self.kernel_size
+        weight_num_param = 1
+        for wd in self.weight_shape:
+            weight_num_param *= wd
+        self.weight = torch.nn.Parameter(torch.Tensor(self.num_experts, weight_num_param))
+
+        if bias:
+            self.bias_shape = (self.out_channels,)
+            self.bias = torch.nn.Parameter(torch.Tensor(self.num_experts, self.out_channels))
+        else:
+            self.register_parameter('bias', None)
+
+        self.reset_parameters()
+
+    def reset_parameters(self):
+        num_input_fmaps = self.weight.size(1)
+        num_output_fmaps = self.weight.size(0)
+        receptive_field_size = 1
+        if self.weight.dim() > 2:
+            receptive_field_size = self.weight[0][0].numel()
+        fan_in = num_input_fmaps * receptive_field_size
+        fan_out = num_output_fmaps * receptive_field_size
+
+        init_weight = get_condconv_initializer(partial(nn.init.normal_, mean=0.0, std=np.sqrt(2.0 / fan_out)), self.num_experts, self.weight_shape)
+        init_weight(self.weight)
+        if self.bias is not None:
+            # fan_in = np.prod(self.weight_shape[1:])
+            # bound = 1 / math.sqrt(fan_in)
+            init_bias = get_condconv_initializer(partial(nn.init.constant_, val=0), self.num_experts, self.bias_shape)
+            init_bias(self.bias)
+
+    def forward(self, x, routing_weights):
+        x_orig = x
+        B, C, H, W = x.shape
+        weight = torch.matmul(routing_weights, self.weight)     # (Expert x out x in x 3x3) --> (B x out x in x 3x3)
+        new_weight_shape = (B * self.out_channels, self.in_channels // self.groups) + self.kernel_size
+        weight = weight.view(new_weight_shape)                  # (B*out x in x 3 x 3)
+        bias = None
+        if self.bias is not None:
+            bias = torch.matmul(routing_weights, self.bias)
+            bias = bias.view(B * self.out_channels)
+        # move batch elements with channels so each batch element can be efficiently convolved with separate kernel
+        x = x.view(1, B * C, H, W)
+        if self.dynamic_padding:
+            out = conv2d_same(
+                x, weight, bias, stride=self.stride, padding=self.padding,
+                dilation=self.dilation, groups=self.groups * B)
+        else:
+            out = F.conv2d(
+                x, weight, bias, stride=self.stride, padding=self.padding,
+                dilation=self.dilation, groups=self.groups * B)
+
+        # out : (1 x B*out x ...)
+        out = out.permute([1, 0, 2, 3]).view(B, self.out_channels, out.shape[-2], out.shape[-1])
+
+        # out2 = self.forward_legacy(x_orig, routing_weights)
+        # lt = torch.lt(torch.abs(torch.add(out, -out2)), 1e-8)
+        # assert torch.all(lt), torch.abs(torch.add(out, -out2))[lt]
+        # print('checked')
+        return out
+
+    def forward_legacy(self, x, routing_weights):
+        # Literal port (from TF definition)
+        B, C, H, W = x.shape
+        weight = torch.matmul(routing_weights, self.weight)  # (Expert x out x in x 3x3) --> (B x out x in x 3x3)
+        x = torch.split(x, 1, 0)
+        weight = torch.split(weight, 1, 0)
+        if self.bias is not None:
+            bias = torch.matmul(routing_weights, self.bias)
+            bias = torch.split(bias, 1, 0)
+        else:
+            bias = [None] * B
+        out = []
+        if self.dynamic_padding:
+            conv_fn = conv2d_same
+        else:
+            conv_fn = F.conv2d
+        for xi, wi, bi in zip(x, weight, bias):
+            wi = wi.view(*self.weight_shape)
+            if bi is not None:
+                bi = bi.view(*self.bias_shape)
+            out.append(conv_fn(
+                xi, wi, bi, stride=self.stride, padding=self.padding,
+                dilation=self.dilation, groups=self.groups))
+        out = torch.cat(out, 0)
+        return out

+import torch
+import torch.nn as nn
+import math
+
+from FastAutoAugment.networks.shakedrop import ShakeDrop
+
+
+def conv3x3(in_planes, out_planes, stride=1):
+    """
+    3x3 convolution with padding
+    """
+    return nn.Conv2d(in_planes, out_planes, kernel_size=3, stride=stride, padding=1, bias=False)
+
+
+class BasicBlock(nn.Module):
+    outchannel_ratio = 1
+
+    def __init__(self, inplanes, planes, stride=1, downsample=None, p_shakedrop=1.0):
+        super(BasicBlock, self).__init__()
+        self.bn1 = nn.BatchNorm2d(inplanes)
+        self.conv1 = conv3x3(inplanes, planes, stride)
+        self.bn2 = nn.BatchNorm2d(planes)
+        self.conv2 = conv3x3(planes, planes)
+        self.bn3 = nn.BatchNorm2d(planes)
+        self.relu = nn.ReLU(inplace=True)
+        self.downsample = downsample
+        self.stride = stride
+        self.shake_drop = ShakeDrop(p_shakedrop)
+
+    def forward(self, x):
+
+        out = self.bn1(x)
+        out = self.conv1(out)
+        out = self.bn2(out)
+        out = self.relu(out)
+        out = self.conv2(out)
+        out = self.bn3(out)
+
+        out = self.shake_drop(out)
+
+        if self.downsample is not None:
+            shortcut = self.downsample(x)
+            featuremap_size = shortcut.size()[2:4]
+        else:
+            shortcut = x
+            featuremap_size = out.size()[2:4]
+
+        batch_size = out.size()[0]
+        residual_channel = out.size()[1]
+        shortcut_channel = shortcut.size()[1]
+
+        if residual_channel != shortcut_channel:
+            padding = torch.autograd.Variable(
+                torch.cuda.FloatTensor(batch_size, residual_channel - shortcut_channel, featuremap_size[0],
+                                       featuremap_size[1]).fill_(0))
+            out += torch.cat((shortcut, padding), 1)
+        else:
+            out += shortcut
+
+        return out
+
+
+class Bottleneck(nn.Module):
+    outchannel_ratio = 4
+
+    def __init__(self, inplanes, planes, stride=1, downsample=None, p_shakedrop=1.0):
+        super(Bottleneck, self).__init__()
+        self.bn1 = nn.BatchNorm2d(inplanes)
+        self.conv1 = nn.Conv2d(inplanes, planes, kernel_size=1, bias=False)
+        self.bn2 = nn.BatchNorm2d(planes)
+        self.conv2 = nn.Conv2d(planes, (planes * 1), kernel_size=3, stride=stride,
+                               padding=1, bias=False)
+        self.bn3 = nn.BatchNorm2d((planes * 1))
+        self.conv3 = nn.Conv2d((planes * 1), planes * Bottleneck.outchannel_ratio, kernel_size=1, bias=False)
+        self.bn4 = nn.BatchNorm2d(planes * Bottleneck.outchannel_ratio)
+        self.relu = nn.ReLU(inplace=True)
+        self.downsample = downsample
+        self.stride = stride
+        self.shake_drop = ShakeDrop(p_shakedrop)
+
+    def forward(self, x):
+
+        out = self.bn1(x)
+        out = self.conv1(out)
+
+        out = self.bn2(out)
+        out = self.relu(out)
+        out = self.conv2(out)
+
+        out = self.bn3(out)
+        out = self.relu(out)
+        out = self.conv3(out)
+
+        out = self.bn4(out)
+
+        out = self.shake_drop(out)
+
+        if self.downsample is not None:
+            shortcut = self.downsample(x)
+            featuremap_size = shortcut.size()[2:4]
+        else:
+            shortcut = x
+            featuremap_size = out.size()[2:4]
+
+        batch_size = out.size()[0]
+        residual_channel = out.size()[1]
+        shortcut_channel = shortcut.size()[1]
+
+        if residual_channel != shortcut_channel:
+            padding = torch.autograd.Variable(
+                torch.cuda.FloatTensor(batch_size, residual_channel - shortcut_channel, featuremap_size[0],
+                                       featuremap_size[1]).fill_(0))
+            out += torch.cat((shortcut, padding), 1)
+        else:
+            out += shortcut
+
+        return out
+
+
+class PyramidNet(nn.Module):
+
+    def __init__(self, dataset, depth, alpha, num_classes, bottleneck=True):
+        super(PyramidNet, self).__init__()
+        self.dataset = dataset
+        if self.dataset.startswith('cifar'):
+            self.inplanes = 16
+            if bottleneck:
+                n = int((depth - 2) / 9)
+                block = Bottleneck
+            else:
+                n = int((depth - 2) / 6)
+                block = BasicBlock
+
+            self.addrate = alpha / (3 * n * 1.0)
+            self.ps_shakedrop = [1. - (1.0 - (0.5 / (3 * n)) * (i + 1)) for i in range(3 * n)]
+
+            self.input_featuremap_dim = self.inplanes
+            self.conv1 = nn.Conv2d(3, self.input_featuremap_dim, kernel_size=3, stride=1, padding=1, bias=False)
+            self.bn1 = nn.BatchNorm2d(self.input_featuremap_dim)
+
+            self.featuremap_dim = self.input_featuremap_dim
+            self.layer1 = self.pyramidal_make_layer(block, n)
+            self.layer2 = self.pyramidal_make_layer(block, n, stride=2)
+            self.layer3 = self.pyramidal_make_layer(block, n, stride=2)
+
+            self.final_featuremap_dim = self.input_featuremap_dim
+            self.bn_final = nn.BatchNorm2d(self.final_featuremap_dim)
+            self.relu_final = nn.ReLU(inplace=True)
+            self.avgpool = nn.AvgPool2d(8)
+            self.fc = nn.Linear(self.final_featuremap_dim, num_classes)
+
+        elif dataset == 'imagenet':
+            blocks = {18: BasicBlock, 34: BasicBlock, 50: Bottleneck, 101: Bottleneck, 152: Bottleneck, 200: Bottleneck}
+            layers = {18: [2, 2, 2, 2], 34: [3, 4, 6, 3], 50: [3, 4, 6, 3], 101: [3, 4, 23, 3], 152: [3, 8, 36, 3],
+                      200: [3, 24, 36, 3]}
+
+            if layers.get(depth) is None:
+                if bottleneck == True:
+                    blocks[depth] = Bottleneck
+                    temp_cfg = int((depth - 2) / 12)
+                else:
+                    blocks[depth] = BasicBlock
+                    temp_cfg = int((depth - 2) / 8)
+
+                layers[depth] = [temp_cfg, temp_cfg, temp_cfg, temp_cfg]
+                print('=> the layer configuration for each stage is set to', layers[depth])
+
+            self.inplanes = 64
+            self.addrate = alpha / (sum(layers[depth]) * 1.0)
+
+            self.input_featuremap_dim = self.inplanes
+            self.conv1 = nn.Conv2d(3, self.input_featuremap_dim, kernel_size=7, stride=2, padding=3, bias=False)
+            self.bn1 = nn.BatchNorm2d(self.input_featuremap_dim)
+            self.relu = nn.ReLU(inplace=True)
+            self.maxpool = nn.MaxPool2d(kernel_size=3, stride=2, padding=1)
+
+            self.featuremap_dim = self.input_featuremap_dim
+            self.layer1 = self.pyramidal_make_layer(blocks[depth], layers[depth][0])
+            self.layer2 = self.pyramidal_make_layer(blocks[depth], layers[depth][1], stride=2)
+            self.layer3 = self.pyramidal_make_layer(blocks[depth], layers[depth][2], stride=2)
+            self.layer4 = self.pyramidal_make_layer(blocks[depth], layers[depth][3], stride=2)
+
+            self.final_featuremap_dim = self.input_featuremap_dim
+            self.bn_final = nn.BatchNorm2d(self.final_featuremap_dim)
+            self.relu_final = nn.ReLU(inplace=True)
+            self.avgpool = nn.AvgPool2d(7)
+            self.fc = nn.Linear(self.final_featuremap_dim, num_classes)
+
+        for m in self.modules():
+            if isinstance(m, nn.Conv2d):
+                n = m.kernel_size[0] * m.kernel_size[1] * m.out_channels
+                m.weight.data.normal_(0, math.sqrt(2. / n))
+            elif isinstance(m, nn.BatchNorm2d):
+                m.weight.data.fill_(1)
+                m.bias.data.zero_()
+
+        assert len(self.ps_shakedrop) == 0, self.ps_shakedrop
+
+    def pyramidal_make_layer(self, block, block_depth, stride=1):
+        downsample = None
+        if stride != 1:  # or self.inplanes != int(round(featuremap_dim_1st)) * block.outchannel_ratio:
+            downsample = nn.AvgPool2d((2, 2), stride=(2, 2), ceil_mode=True)
+
+        layers = []
+        self.featuremap_dim = self.featuremap_dim + self.addrate
+        layers.append(block(self.input_featuremap_dim, int(round(self.featuremap_dim)), stride, downsample, p_shakedrop=self.ps_shakedrop.pop(0)))
+        for i in range(1, block_depth):
+            temp_featuremap_dim = self.featuremap_dim + self.addrate
+            layers.append(
+                block(int(round(self.featuremap_dim)) * block.outchannel_ratio, int(round(temp_featuremap_dim)), 1, p_shakedrop=self.ps_shakedrop.pop(0)))
+            self.featuremap_dim = temp_featuremap_dim
+        self.input_featuremap_dim = int(round(self.featuremap_dim)) * block.outchannel_ratio
+
+        return nn.Sequential(*layers)
+
+    def forward(self, x):
+        if self.dataset == 'cifar10' or self.dataset == 'cifar100':
+            x = self.conv1(x)
+            x = self.bn1(x)
+
+            x = self.layer1(x)
+            x = self.layer2(x)
+            x = self.layer3(x)
+
+            x = self.bn_final(x)
+            x = self.relu_final(x)
+            x = self.avgpool(x)
+            x = x.view(x.size(0), -1)
+            x = self.fc(x)
+
+        elif self.dataset == 'imagenet':
+            x = self.conv1(x)
+            x = self.bn1(x)
+            x = self.relu(x)
+            x = self.maxpool(x)
+
+            x = self.layer1(x)
+            x = self.layer2(x)
+            x = self.layer3(x)
+            x = self.layer4(x)
+
+            x = self.bn_final(x)
+            x = self.relu_final(x)
+            x = self.avgpool(x)
+            x = x.view(x.size(0), -1)
+            x = self.fc(x)
+
+        return x

+# Original code: https://github.com/pytorch/vision/blob/master/torchvision/models/resnet.py
+
+import torch.nn as nn
+import math
+
+
+def conv3x3(in_planes, out_planes, stride=1):
+    "3x3 convolution with padding"
+    return nn.Conv2d(in_planes, out_planes, kernel_size=3, stride=stride,
+                     padding=1, bias=False)
+
+
+class BasicBlock(nn.Module):
+    expansion = 1
+
+    def __init__(self, inplanes, planes, stride=1, downsample=None):
+        super(BasicBlock, self).__init__()
+        self.conv1 = conv3x3(inplanes, planes, stride)
+        self.bn1 = nn.BatchNorm2d(planes)
+        self.conv2 = conv3x3(planes, planes)
+        self.bn2 = nn.BatchNorm2d(planes)
+        self.relu = nn.ReLU(inplace=True)
+
+        self.downsample = downsample
+        self.stride = stride
+
+    def forward(self, x):
+        residual = x
+
+        out = self.conv1(x)
+        out = self.bn1(out)
+        out = self.relu(out)
+
+        out = self.conv2(out)
+        out = self.bn2(out)
+
+        if self.downsample is not None:
+            residual = self.downsample(x)
+
+        out += residual
+        out = self.relu(out)
+
+        return out
+
+
+class Bottleneck(nn.Module):
+    expansion = 4
+
+    def __init__(self, inplanes, planes, stride=1, downsample=None):
+        super(Bottleneck, self).__init__()
+
+        self.conv1 = nn.Conv2d(inplanes, planes, kernel_size=1, bias=False)
+        self.bn1 = nn.BatchNorm2d(planes)
+        self.conv2 = nn.Conv2d(planes, planes, kernel_size=3, stride=stride, padding=1, bias=False)
+        self.bn2 = nn.BatchNorm2d(planes)
+        self.conv3 = nn.Conv2d(planes, planes * Bottleneck.expansion, kernel_size=1, bias=False)
+        self.bn3 = nn.BatchNorm2d(planes * Bottleneck.expansion)
+        self.relu = nn.ReLU(inplace=True)
+
+        self.downsample = downsample
+        self.stride = stride
+
+    def forward(self, x):
+        residual = x
+
+        out = self.conv1(x)
+        out = self.bn1(out)
+        out = self.relu(out)
+
+        out = self.conv2(out)
+        out = self.bn2(out)
+        out = self.relu(out)
+
+        out = self.conv3(out)
+        out = self.bn3(out)
+        if self.downsample is not None:
+            residual = self.downsample(x)
+
+        out += residual
+        out = self.relu(out)
+
+        return out
+
+class ResNet(nn.Module):
+    def __init__(self, dataset, depth, num_classes, bottleneck=False):
+        super(ResNet, self).__init__()        
+        self.dataset = dataset
+        if self.dataset.startswith('cifar'):
+            self.inplanes = 16
+            print(bottleneck)
+            if bottleneck == True:
+                n = int((depth - 2) / 9)
+                block = Bottleneck
+            else:
+                n = int((depth - 2) / 6)
+                block = BasicBlock
+
+            self.conv1 = nn.Conv2d(3, self.inplanes, kernel_size=3, stride=1, padding=1, bias=False)
+            self.bn1 = nn.BatchNorm2d(self.inplanes)
+            self.relu = nn.ReLU(inplace=True)
+            self.layer1 = self._make_layer(block, 16, n)
+            self.layer2 = self._make_layer(block, 32, n, stride=2)
+            self.layer3 = self._make_layer(block, 64, n, stride=2) 
+            # self.avgpool = nn.AvgPool2d(8)
+            self.avgpool = nn.AdaptiveAvgPool2d((1, 1))
+            self.fc = nn.Linear(64 * block.expansion, num_classes)
+
+        elif dataset == 'imagenet':
+            blocks ={18: BasicBlock, 34: BasicBlock, 50: Bottleneck, 101: Bottleneck, 152: Bottleneck, 200: Bottleneck}
+            layers ={18: [2, 2, 2, 2], 34: [3, 4, 6, 3], 50: [3, 4, 6, 3], 101: [3, 4, 23, 3], 152: [3, 8, 36, 3], 200: [3, 24, 36, 3]}
+            assert layers[depth], 'invalid detph for ResNet (depth should be one of 18, 34, 50, 101, 152, and 200)'
+
+            self.inplanes = 64
+            self.conv1 = nn.Conv2d(3, self.inplanes, kernel_size=7, stride=2, padding=3, bias=False)
+            self.bn1 = nn.BatchNorm2d(64)
+            self.relu = nn.ReLU(inplace=True)
+            self.maxpool = nn.MaxPool2d(kernel_size=3, stride=2, padding=1)
+            self.layer1 = self._make_layer(blocks[depth], 64, layers[depth][0])
+            self.layer2 = self._make_layer(blocks[depth], 128, layers[depth][1], stride=2)
+            self.layer3 = self._make_layer(blocks[depth], 256, layers[depth][2], stride=2)
+            self.layer4 = self._make_layer(blocks[depth], 512, layers[depth][3], stride=2)
+            # self.avgpool = nn.AvgPool2d(7)
+            self.avgpool = nn.AdaptiveAvgPool2d((1, 1))
+            self.fc = nn.Linear(512 * blocks[depth].expansion, num_classes)
+
+        for m in self.modules():
+            if isinstance(m, nn.Conv2d):
+                n = m.kernel_size[0] * m.kernel_size[1] * m.out_channels
+                m.weight.data.normal_(0, math.sqrt(2. / n))
+            elif isinstance(m, nn.BatchNorm2d):
+                m.weight.data.fill_(1)
+                m.bias.data.zero_()
+
+    def _make_layer(self, block, planes, blocks, stride=1):
+        downsample = None
+        if stride != 1 or self.inplanes != planes * block.expansion:
+            downsample = nn.Sequential(
+                nn.Conv2d(self.inplanes, planes * block.expansion,
+                          kernel_size=1, stride=stride, bias=False),
+                nn.BatchNorm2d(planes * block.expansion),
+            )
+
+        layers = []
+        layers.append(block(self.inplanes, planes, stride, downsample))
+        self.inplanes = planes * block.expansion
+        for i in range(1, blocks):
+            layers.append(block(self.inplanes, planes))
+
+        return nn.Sequential(*layers)
+
+    def forward(self, x):
+        if self.dataset == 'cifar10' or self.dataset == 'cifar100':
+            x = self.conv1(x)
+            x = self.bn1(x)
+            x = self.relu(x)
+            
+            x = self.layer1(x)
+            x = self.layer2(x)
+            x = self.layer3(x)
+
+            x = self.avgpool(x)
+            x = x.view(x.size(0), -1)
+            x = self.fc(x)
+
+        elif self.dataset == 'imagenet':
+            x = self.conv1(x)
+            x = self.bn1(x)
+            x = self.relu(x)
+            x = self.maxpool(x)
+
+            x = self.layer1(x)
+            x = self.layer2(x)
+            x = self.layer3(x)
+            x = self.layer4(x)
+
+            x = self.avgpool(x)
+            x = x.view(x.size(0), -1)
+            x = self.fc(x)
+    
+        return x

+# -*- coding: utf-8 -*-
+
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+from torch.autograd import Variable
+
+
+class ShakeDropFunction(torch.autograd.Function):
+
+    @staticmethod
+    def forward(ctx, x, training=True, p_drop=0.5, alpha_range=[-1, 1]):
+        if training:
+            gate = torch.cuda.FloatTensor([0]).bernoulli_(1 - p_drop)
+            ctx.save_for_backward(gate)
+            if gate.item() == 0:
+                alpha = torch.cuda.FloatTensor(x.size(0)).uniform_(*alpha_range)
+                alpha = alpha.view(alpha.size(0), 1, 1, 1).expand_as(x)
+                return alpha * x
+            else:
+                return x
+        else:
+            return (1 - p_drop) * x
+
+    @staticmethod
+    def backward(ctx, grad_output):
+        gate = ctx.saved_tensors[0]
+        if gate.item() == 0:
+            beta = torch.cuda.FloatTensor(grad_output.size(0)).uniform_(0, 1)
+            beta = beta.view(beta.size(0), 1, 1, 1).expand_as(grad_output)
+            beta = Variable(beta)
+            return beta * grad_output, None, None, None
+        else:
+            return grad_output, None, None, None
+
+
+class ShakeDrop(nn.Module):
+
+    def __init__(self, p_drop=0.5, alpha_range=[-1, 1]):
+        super(ShakeDrop, self).__init__()
+        self.p_drop = p_drop
+        self.alpha_range = alpha_range
+
+    def forward(self, x):
+        return ShakeDropFunction.apply(x, self.training, self.p_drop, self.alpha_range)

+# -*- coding: utf-8 -*-
+
+import math
+
+import torch.nn as nn
+import torch.nn.functional as F
+
+from FastAutoAugment.networks.shakeshake.shakeshake import ShakeShake
+from FastAutoAugment.networks.shakeshake.shakeshake import Shortcut
+
+
+class ShakeBlock(nn.Module):
+
+    def __init__(self, in_ch, out_ch, stride=1):
+        super(ShakeBlock, self).__init__()
+        self.equal_io = in_ch == out_ch
+        self.shortcut = self.equal_io and None or Shortcut(in_ch, out_ch, stride=stride)
+
+        self.branch1 = self._make_branch(in_ch, out_ch, stride)
+        self.branch2 = self._make_branch(in_ch, out_ch, stride)
+
+    def forward(self, x):
+        h1 = self.branch1(x)
+        h2 = self.branch2(x)
+        h = ShakeShake.apply(h1, h2, self.training)
+        h0 = x if self.equal_io else self.shortcut(x)
+        return h + h0
+
+    def _make_branch(self, in_ch, out_ch, stride=1):
+        return nn.Sequential(
+            nn.ReLU(inplace=False),
+            nn.Conv2d(in_ch, out_ch, 3, padding=1, stride=stride, bias=False),
+            nn.BatchNorm2d(out_ch),
+            nn.ReLU(inplace=False),
+            nn.Conv2d(out_ch, out_ch, 3, padding=1, stride=1, bias=False),
+            nn.BatchNorm2d(out_ch))
+
+
+class ShakeResNet(nn.Module):
+
+    def __init__(self, depth, w_base, label):
+        super(ShakeResNet, self).__init__()
+        n_units = (depth - 2) / 6
+
+        in_chs = [16, w_base, w_base * 2, w_base * 4]
+        self.in_chs = in_chs
+
+        self.c_in = nn.Conv2d(3, in_chs[0], 3, padding=1)
+        self.layer1 = self._make_layer(n_units, in_chs[0], in_chs[1])
+        self.layer2 = self._make_layer(n_units, in_chs[1], in_chs[2], 2)
+        self.layer3 = self._make_layer(n_units, in_chs[2], in_chs[3], 2)
+        self.fc_out = nn.Linear(in_chs[3], label)
+
+        # Initialize paramters
+        for m in self.modules():
+            if isinstance(m, nn.Conv2d):
+                n = m.kernel_size[0] * m.kernel_size[1] * m.out_channels
+                m.weight.data.normal_(0, math.sqrt(2. / n))
+            elif isinstance(m, nn.BatchNorm2d):
+                m.weight.data.fill_(1)
+                m.bias.data.zero_()
+            elif isinstance(m, nn.Linear):
+                m.bias.data.zero_()
+
+    def forward(self, x):
+        h = self.c_in(x)
+        h = self.layer1(h)
+        h = self.layer2(h)
+        h = self.layer3(h)
+        h = F.relu(h)
+        h = F.avg_pool2d(h, 8)
+        h = h.view(-1, self.in_chs[3])
+        h = self.fc_out(h)
+        return h
+
+    def _make_layer(self, n_units, in_ch, out_ch, stride=1):
+        layers = []
+        for i in range(int(n_units)):
+            layers.append(ShakeBlock(in_ch, out_ch, stride=stride))
+            in_ch, stride = out_ch, 1
+        return nn.Sequential(*layers)

+# -*- coding: utf-8 -*-
+
+import math
+
+import torch.nn as nn
+import torch.nn.functional as F
+
+from FastAutoAugment.networks.shakeshake.shakeshake import ShakeShake
+from FastAutoAugment.networks.shakeshake.shakeshake import Shortcut
+
+
+class ShakeBottleNeck(nn.Module):
+
+    def __init__(self, in_ch, mid_ch, out_ch, cardinary, stride=1):
+        super(ShakeBottleNeck, self).__init__()
+        self.equal_io = in_ch == out_ch
+        self.shortcut = None if self.equal_io else Shortcut(in_ch, out_ch, stride=stride)
+
+        self.branch1 = self._make_branch(in_ch, mid_ch, out_ch, cardinary, stride)
+        self.branch2 = self._make_branch(in_ch, mid_ch, out_ch, cardinary, stride)
+
+    def forward(self, x):
+        h1 = self.branch1(x)
+        h2 = self.branch2(x)
+        h = ShakeShake.apply(h1, h2, self.training)
+        h0 = x if self.equal_io else self.shortcut(x)
+        return h + h0
+
+    def _make_branch(self, in_ch, mid_ch, out_ch, cardinary, stride=1):
+        return nn.Sequential(
+            nn.Conv2d(in_ch, mid_ch, 1, padding=0, bias=False),
+            nn.BatchNorm2d(mid_ch),
+            nn.ReLU(inplace=False),
+            nn.Conv2d(mid_ch, mid_ch, 3, padding=1, stride=stride, groups=cardinary, bias=False),
+            nn.BatchNorm2d(mid_ch),
+            nn.ReLU(inplace=False),
+            nn.Conv2d(mid_ch, out_ch, 1, padding=0, bias=False),
+            nn.BatchNorm2d(out_ch))
+
+
+class ShakeResNeXt(nn.Module):
+
+    def __init__(self, depth, w_base, cardinary, label):
+        super(ShakeResNeXt, self).__init__()
+        n_units = (depth - 2) // 9
+        n_chs = [64, 128, 256, 1024]
+        self.n_chs = n_chs
+        self.in_ch = n_chs[0]
+
+        self.c_in = nn.Conv2d(3, n_chs[0], 3, padding=1)
+        self.layer1 = self._make_layer(n_units, n_chs[0], w_base, cardinary)
+        self.layer2 = self._make_layer(n_units, n_chs[1], w_base, cardinary, 2)
+        self.layer3 = self._make_layer(n_units, n_chs[2], w_base, cardinary, 2)
+        self.fc_out = nn.Linear(n_chs[3], label)
+
+        # Initialize paramters
+        for m in self.modules():
+            if isinstance(m, nn.Conv2d):
+                n = m.kernel_size[0] * m.kernel_size[1] * m.out_channels
+                m.weight.data.normal_(0, math.sqrt(2. / n))
+            elif isinstance(m, nn.BatchNorm2d):
+                m.weight.data.fill_(1)
+                m.bias.data.zero_()
+            elif isinstance(m, nn.Linear):
+                m.bias.data.zero_()
+
+    def forward(self, x):
+        h = self.c_in(x)
+        h = self.layer1(h)
+        h = self.layer2(h)
+        h = self.layer3(h)
+        h = F.relu(h)
+        h = F.avg_pool2d(h, 8)
+        h = h.view(-1, self.n_chs[3])
+        h = self.fc_out(h)
+        return h
+
+    def _make_layer(self, n_units, n_ch, w_base, cardinary, stride=1):
+        layers = []
+        mid_ch, out_ch = n_ch * (w_base // 64) * cardinary, n_ch * 4
+        for i in range(n_units):
+            layers.append(ShakeBottleNeck(self.in_ch, mid_ch, out_ch, cardinary, stride=stride))
+            self.in_ch, stride = out_ch, 1
+        return nn.Sequential(*layers)

+# -*- coding: utf-8 -*-
+
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+from torch.autograd import Variable
+
+
+class ShakeShake(torch.autograd.Function):
+
+    @staticmethod
+    def forward(ctx, x1, x2, training=True):
+        if training:
+            alpha = torch.cuda.FloatTensor(x1.size(0)).uniform_()
+            alpha = alpha.view(alpha.size(0), 1, 1, 1).expand_as(x1)
+        else:
+            alpha = 0.5
+        return alpha * x1 + (1 - alpha) * x2
+
+    @staticmethod
+    def backward(ctx, grad_output):
+        beta = torch.cuda.FloatTensor(grad_output.size(0)).uniform_()
+        beta = beta.view(beta.size(0), 1, 1, 1).expand_as(grad_output)
+        beta = Variable(beta)
+
+        return beta * grad_output, (1 - beta) * grad_output, None
+
+
+class Shortcut(nn.Module):
+
+    def __init__(self, in_ch, out_ch, stride):
+        super(Shortcut, self).__init__()
+        self.stride = stride
+        self.conv1 = nn.Conv2d(in_ch, out_ch // 2, 1, stride=1, padding=0, bias=False)
+        self.conv2 = nn.Conv2d(in_ch, out_ch // 2, 1, stride=1, padding=0, bias=False)
+        self.bn = nn.BatchNorm2d(out_ch)
+
+    def forward(self, x):
+        h = F.relu(x)
+
+        h1 = F.avg_pool2d(h, 1, self.stride)
+        h1 = self.conv1(h1)
+
+        h2 = F.avg_pool2d(F.pad(h, (-1, 1, -1, 1)), 1, self.stride)
+        h2 = self.conv2(h2)
+
+        h = torch.cat((h1, h2), 1)
+        return self.bn(h)

+import torch.nn as nn
+import torch.nn.init as init
+import torch.nn.functional as F
+import numpy as np
+
+
+def conv3x3(in_planes, out_planes, stride=1):
+    return nn.Conv2d(in_planes, out_planes, kernel_size=3, stride=stride, padding=1, bias=True)
+
+
+def conv_init(m):
+    classname = m.__class__.__name__
+    if classname.find('Conv') != -1:
+        init.xavier_uniform_(m.weight, gain=np.sqrt(2))
+        init.constant_(m.bias, 0)
+    elif classname.find('BatchNorm') != -1:
+        init.constant_(m.weight, 1)
+        init.constant_(m.bias, 0)
+
+
+class WideBasic(nn.Module):
+    def __init__(self, in_planes, planes, dropout_rate, stride=1):
+        super(WideBasic, self).__init__()
+        self.bn1 = nn.BatchNorm2d(in_planes, momentum=0.9)
+        self.conv1 = nn.Conv2d(in_planes, planes, kernel_size=3, padding=1, bias=True)
+        self.dropout = nn.Dropout(p=dropout_rate)
+        self.bn2 = nn.BatchNorm2d(planes, momentum=0.9)
+        self.conv2 = nn.Conv2d(planes, planes, kernel_size=3, stride=stride, padding=1, bias=True)
+
+        self.shortcut = nn.Sequential()
+        if stride != 1 or in_planes != planes:
+            self.shortcut = nn.Sequential(
+                nn.Conv2d(in_planes, planes, kernel_size=1, stride=stride, bias=True),
+            )
+
+    def forward(self, x):
+        out = self.dropout(self.conv1(F.relu(self.bn1(x))))
+        out = self.conv2(F.relu(self.bn2(out)))
+        out += self.shortcut(x)
+
+        return out
+
+
+class WideResNet(nn.Module):
+    def __init__(self, depth, widen_factor, dropout_rate, num_classes):
+        super(WideResNet, self).__init__()
+        self.in_planes = 16
+
+        assert ((depth - 4) % 6 == 0), 'Wide-resnet depth should be 6n+4'
+        n = int((depth - 4) / 6)
+        k = widen_factor
+
+        nStages = [16, 16*k, 32*k, 64*k]
+
+        self.conv1 = conv3x3(3, nStages[0])
+        self.layer1 = self._wide_layer(WideBasic, nStages[1], n, dropout_rate, stride=1)
+        self.layer2 = self._wide_layer(WideBasic, nStages[2], n, dropout_rate, stride=2)
+        self.layer3 = self._wide_layer(WideBasic, nStages[3], n, dropout_rate, stride=2)
+        self.bn1 = nn.BatchNorm2d(nStages[3], momentum=0.9)
+        self.linear = nn.Linear(nStages[3], num_classes)
+
+        # self.apply(conv_init)
+
+    def _wide_layer(self, block, planes, num_blocks, dropout_rate, stride):
+        strides = [stride] + [1]*(num_blocks-1)
+        layers = []
+
+        for stride in strides:
+            layers.append(block(self.in_planes, planes, dropout_rate, stride))
+            self.in_planes = planes
+
+        return nn.Sequential(*layers)
+
+    def forward(self, x):
+        out = self.conv1(x)
+        out = self.layer1(out)
+        out = self.layer2(out)
+        out = self.layer3(out)
+        out = F.relu(self.bn1(out))
+        # out = F.avg_pool2d(out, 8)
+        out = F.adaptive_avg_pool2d(out, (1, 1))
+        out = out.view(out.size(0), -1)
+        out = self.linear(out)
+
+        return out

+# Copyright 2019 Uber Technologies, Inc. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+
+import os
+import psutil
+import re
+import signal
+import subprocess
+import sys
+import threading
+import time
+
+
+GRACEFUL_TERMINATION_TIME_S = 5
+
+
+def terminate_executor_shell_and_children(pid):
+    print('terminate_executor_shell_and_children+', pid)
+    # If the shell already ends, no need to terminate its child.
+    try:
+        p = psutil.Process(pid)
+    except psutil.NoSuchProcess:
+        print('nosuchprocess')
+        return
+
+    # Terminate children gracefully.
+    for child in p.children():
+        try:
+            child.terminate()
+        except psutil.NoSuchProcess:
+            pass
+
+    # Wait for graceful termination.
+    time.sleep(GRACEFUL_TERMINATION_TIME_S)
+
+    # Send STOP to executor shell to stop progress.
+    p.send_signal(signal.SIGSTOP)
+
+    # Kill children recursively.
+    for child in p.children(recursive=True):
+        try:
+            child.kill()
+        except psutil.NoSuchProcess:
+            pass
+
+    # Kill shell itself.
+    p.kill()
+    print('terminate_executor_shell_and_children-', pid)
+
+
+def forward_stream(src_fd, dst_stream, prefix, index):
+    with os.fdopen(src_fd, 'r') as src:
+        line_buffer = ''
+        while True:
+            text = os.read(src.fileno(), 1000)
+            if not isinstance(text, str):
+                text = text.decode('utf-8')
+            if not text:
+                break
+
+            for line in re.split('([\r\n])', text):
+                line_buffer += line
+                if line == '\r' or line == '\n':
+                    if index is not None:
+                        localtime = time.asctime(time.localtime(time.time()))
+                        line_buffer = '{time}[{rank}]<{prefix}>:{line}'.format(
+                            time=localtime,
+                            rank=str(index),
+                            prefix=prefix,
+                            line=line_buffer
+                        )
+
+                    dst_stream.write(line_buffer)
+                    dst_stream.flush()
+                    line_buffer = ''
+
+
+def execute(command, env=None, stdout=None, stderr=None, index=None, event=None):
+    # Make a pipe for the subprocess stdout/stderr.
+    (stdout_r, stdout_w) = os.pipe()
+    (stderr_r, stderr_w) = os.pipe()
+
+    # Make a pipe for notifying the child that parent has died.
+    (r, w) = os.pipe()
+
+    middleman_pid = os.fork()
+    if middleman_pid == 0:
+        # Close unused file descriptors to enforce PIPE behavior.
+        os.close(w)
+        os.setsid()
+
+        executor_shell = subprocess.Popen(command, shell=True, env=env,
+                                          stdout=stdout_w, stderr=stderr_w)
+
+        sigterm_received = threading.Event()
+
+        def set_sigterm_received(signum, frame):
+            sigterm_received.set()
+
+        signal.signal(signal.SIGINT, set_sigterm_received)
+        signal.signal(signal.SIGTERM, set_sigterm_received)
+
+        def kill_executor_children_if_parent_dies():
+            # This read blocks until the pipe is closed on the other side
+            # due to the process termination.
+            os.read(r, 1)
+            terminate_executor_shell_and_children(executor_shell.pid)
+
+        bg = threading.Thread(target=kill_executor_children_if_parent_dies)
+        bg.daemon = True
+        bg.start()
+
+        def kill_executor_children_if_sigterm_received():
+            sigterm_received.wait()
+            terminate_executor_shell_and_children(executor_shell.pid)
+
+        bg = threading.Thread(target=kill_executor_children_if_sigterm_received)
+        bg.daemon = True
+        bg.start()
+
+        exit_code = executor_shell.wait()
+        os._exit(exit_code)
+
+    # Close unused file descriptors to enforce PIPE behavior.
+    os.close(r)
+    os.close(stdout_w)
+    os.close(stderr_w)
+
+    # Redirect command stdout & stderr to provided streams or sys.stdout/sys.stderr.
+    # This is useful for Jupyter Notebook that uses custom sys.stdout/sys.stderr or
+    # for redirecting to a file on disk.
+    if stdout is None:
+        stdout = sys.stdout
+    if stderr is None:
+        stderr = sys.stderr
+    stdout_fwd = threading.Thread(target=forward_stream, args=(stdout_r, stdout, 'stdout', index))
+    stderr_fwd = threading.Thread(target=forward_stream, args=(stderr_r, stderr, 'stderr', index))
+    stdout_fwd.start()
+    stderr_fwd.start()
+
+    def kill_middleman_if_master_thread_terminate():
+        event.wait()
+        try:
+            os.kill(middleman_pid, signal.SIGTERM)
+        except:
+            # The process has already been killed elsewhere
+            pass
+
+    # TODO: Currently this requires explicitly declaration of the event and signal handler to set
+    #  the event (gloo_run.py:_launch_jobs()). Need to figure out a generalized way to hide this behind
+    #  interfaces.
+    if event is not None:
+        bg_thread = threading.Thread(target=kill_middleman_if_master_thread_terminate)
+        bg_thread.daemon = True
+        bg_thread.start()
+
+    try:
+        res, status = os.waitpid(middleman_pid, 0)
+    except:
+        # interrupted, send middleman TERM signal which will terminate children
+        os.kill(middleman_pid, signal.SIGTERM)
+        while True:
+            try:
+                _, status = os.waitpid(middleman_pid, 0)
+                break
+            except:
+                # interrupted, wait for middleman to finish
+                pass
+
+    stdout_fwd.join()
+    stderr_fwd.join()
+    exit_code = status >> 8
+    return exit_code

+import torch
+from torch.optim.optimizer import Optimizer
+
+
+class RMSpropTF(Optimizer):
+    r"""Implements RMSprop algorithm.
+    Reimplement original formulation to match TF rmsprop
+    Proposed by G. Hinton in his
+    `course <http://www.cs.toronto.edu/~tijmen/csc321/slides/lecture_slides_lec6.pdf>`_.
+    The centered version first appears in `Generating Sequences
+    With Recurrent Neural Networks <https://arxiv.org/pdf/1308.0850v5.pdf>`_.
+    The implementation here takes the square root of the gradient average before
+    adding epsilon (note that TensorFlow interchanges these two operations). The effective
+    learning rate is thus :math:`\alpha/(\sqrt{v + \epsilon})` where :math:`\alpha` from :math:`\alpha/(\sqrt{v} + \epsilon)` where :math:`\alpha`
+    is the scheduled learning rate and :math:`v` is the weighted moving average
+    of the squared gradient.
+    Arguments:
+        params (iterable): iterable of parameters to optimize or dicts defining
+            parameter groups
+        lr (float, optional): learning rate (default: 1e-2)
+        momentum (float, optional): momentum factor (default: 0)
+        alpha (float, optional): smoothing constant (default: 0.99)
+        eps (float, optional): term added to the denominator to improve
+            numerical stability (default: 1e-8)
+        centered (bool, optional) : if ``True``, compute the centered RMSProp,
+            the gradient is normalized by an estimation of its variance
+        weight_decay (float, optional): weight decay (L2 penalty) (default: 0)
+    """
+
+    def __init__(self, params, lr=1e-2, alpha=0.99, eps=1e-8, momentum=0, weight_decay=0.0):
+        if not 0.0 <= lr:
+            raise ValueError("Invalid learning rate: {}".format(lr))
+        if not 0.0 <= eps:
+            raise ValueError("Invalid epsilon value: {}".format(eps))
+        if not 0.0 < momentum:
+            raise ValueError("Invalid momentum value: {}".format(momentum))
+        if not 0.0 <= alpha:
+            raise ValueError("Invalid alpha value: {}".format(alpha))
+        assert momentum > 0.0
+
+        defaults = dict(lr=lr, momentum=momentum, alpha=alpha, eps=eps, weight_decay=weight_decay)
+        super(RMSpropTF, self).__init__(params, defaults)
+        self.initialized = False
+
+    def __setstate__(self, state):
+        super(RMSpropTF, self).__setstate__(state)
+        for group in self.param_groups:
+            group.setdefault('momentum', 0)
+
+    def load_state_dict(self, state_dict):
+        super(RMSpropTF, self).load_state_dict(state_dict)
+        self.initialized = True
+
+    def step(self, closure=None):
+        """Performs a single optimization step.
+        We modified pytorch's RMSProp to be same as Tensorflow's
+        See : https://github.com/tensorflow/tensorflow/blob/master/tensorflow/core/kernels/training_ops.cc#L485
+
+        Arguments:
+            closure (callable, optional): A closure that reevaluates the model
+                and returns the loss.
+        """
+        loss = None
+        if closure is not None:
+            loss = closure()
+
+        for group in self.param_groups:
+            for p in group['params']:
+                if p.grad is None:
+                    continue
+                grad = p.grad.data
+                if grad.is_sparse:
+                    raise RuntimeError('RMSprop does not support sparse gradients')
+                state = self.state[p]
+
+                # State initialization
+                if len(state) == 0:
+                    assert not self.initialized
+                    state['step'] = 0
+                    state['ms'] = torch.ones_like(p.data)  #, memory_format=torch.preserve_format)
+                    state['mom'] = torch.zeros_like(p.data)  #, memory_format=torch.preserve_format)
+
+                # weight decay -----
+                if group['weight_decay'] > 0:
+                    grad = grad.add(group['weight_decay'], p.data)
+
+                rho = group['alpha']
+                ms = state['ms']
+                mom = state['mom']
+                state['step'] += 1
+
+                # ms.mul_(rho).addcmul_(1 - rho, grad, grad)
+                ms.add_(torch.mul(grad, grad).add_(-ms) * (1. - rho))
+                assert group['momentum'] > 0
+
+                # new rmsprop
+                mom.mul_(group['momentum']).addcdiv_(group['lr'], grad, (ms + group['eps']).sqrt())
+
+                p.data.add_(-1.0, mom)
+
+        return loss

+import torch
+from torch.nn import BatchNorm2d
+from torch.nn.parameter import Parameter
+import torch.distributed as dist
+from torch import nn
+
+
+class TpuBatchNormalization(nn.Module):
+    # Ref : https://github.com/tensorflow/tpu/blob/master/models/official/efficientnet/utils.py#L113
+    def __init__(self, num_features, eps=1e-5, momentum=0.1, affine=True,
+                 track_running_stats=True):
+        super(TpuBatchNormalization, self).__init__()   # num_features, eps, momentum, affine, track_running_stats)
+
+        self.weight = Parameter(torch.ones(num_features))
+        self.bias = Parameter(torch.zeros(num_features))
+
+        self.register_buffer('running_mean', torch.zeros(num_features))
+        self.register_buffer('running_var', torch.ones(num_features))
+        self.register_buffer('num_batches_tracked', torch.tensor(0, dtype=torch.long))
+
+        self.eps = eps
+        self.momentum = momentum
+
+    def _reduce_avg(self, t):
+        dist.all_reduce(t, dist.ReduceOp.SUM)
+        t.mul_(1. / dist.get_world_size())
+
+    def forward(self, input):
+        if not self.training or not dist.is_initialized():
+            bn = (input - self.running_mean.view(1, self.running_mean.shape[0], 1, 1)) / \
+                 (torch.sqrt(self.running_var.view(1, self.running_var.shape[0], 1, 1) + self.eps))
+            # print(self.weight.shape, self.bias.shape)
+            return bn.mul(self.weight.view(1, self.weight.shape[0], 1, 1)).add(self.bias.view(1, self.bias.shape[0], 1, 1))
+
+        shard_mean, shard_invstd = torch.batch_norm_stats(input, self.eps)
+        shard_vars = (1. / shard_invstd) ** 2 - self.eps
+
+        shard_square_of_mean = torch.mul(shard_mean, shard_mean)
+        shard_mean_of_square = shard_vars + shard_square_of_mean
+
+        group_mean = shard_mean.clone().detach()
+        self._reduce_avg(group_mean)
+        group_mean_of_square = shard_mean_of_square.clone().detach()
+        self._reduce_avg(group_mean_of_square)
+        group_vars = group_mean_of_square - torch.mul(group_mean, group_mean)
+
+        group_mean = group_mean.detach()
+        group_vars = group_vars.detach()
+
+        # print(self.running_mean.shape, self.running_var.shape)
+        self.running_mean.mul_(1. - self.momentum).add_(group_mean.mul(self.momentum))
+        self.running_var.mul_(1. - self.momentum).add_(group_vars.mul(self.momentum))
+        self.num_batches_tracked.add_(1)
+
+        # print(input.shape, group_mean.view(1, group_mean.shape[0], 1, 1).shape, group_vars.view(1, group_vars.shape[0], 1, 1).shape, self.eps)
+        bn = (input - group_mean.view(1, group_mean.shape[0], 1, 1)) / (torch.sqrt(group_vars.view(1, group_vars.shape[0], 1, 1) + self.eps))
+        # print(self.weight.shape, self.bias.shape)
+        return bn.mul(self.weight.view(1, self.weight.shape[0], 1, 1)).add(self.bias.view(1, self.bias.shape[0], 1, 1))

+import sys
+sys.path.append('/data/private/fast-autoaugment-public')    # TODO
+
+import time
+import os
+import threading
+import six
+from six.moves import queue
+
+from FastAutoAugment import safe_shell_exec
+
+
+def _exec_command(command):
+    host_output = six.StringIO()
+    try:
+        exit_code = safe_shell_exec.execute(command,
+                                            stdout=host_output,
+                                            stderr=host_output)
+        if exit_code != 0:
+            print('Launching task function was not successful:\n{host_output}'.format(host_output=host_output.getvalue()))
+            os._exit(exit_code)
+    finally:
+        host_output.close()
+    return exit_code
+
+
+def execute_function_multithreaded(fn,
+                                   args_list,
+                                   block_until_all_done=True,
+                                   max_concurrent_executions=1000):
+    """
+    Executes fn in multiple threads each with one set of the args in the
+    args_list.
+    :param fn: function to be executed
+    :type fn:
+    :param args_list:
+    :type args_list: list(list)
+    :param block_until_all_done: if is True, function will block until all the
+    threads are done and will return the results of each thread's execution.
+    :type block_until_all_done: bool
+    :param max_concurrent_executions:
+    :type max_concurrent_executions: int
+    :return:
+    If block_until_all_done is False, returns None. If block_until_all_done is
+    True, function returns the dict of results.
+        {
+            index: execution result of fn with args_list[index]
+        }
+    :rtype: dict
+    """
+    result_queue = queue.Queue()
+    worker_queue = queue.Queue()
+
+    for i, arg in enumerate(args_list):
+        arg.append(i)
+        worker_queue.put(arg)
+
+    def fn_execute():
+        while True:
+            try:
+                arg = worker_queue.get(block=False)
+            except queue.Empty:
+                return
+            exec_index = arg[-1]
+            res = fn(*arg[:-1])
+            result_queue.put((exec_index, res))
+
+    threads = []
+    number_of_threads = min(max_concurrent_executions, len(args_list))
+
+    for _ in range(number_of_threads):
+        thread = threading.Thread(target=fn_execute)
+        if not block_until_all_done:
+            thread.daemon = True
+        thread.start()
+        threads.append(thread)
+
+    # Returns the results only if block_until_all_done is set.
+    results = None
+    if block_until_all_done:
+        # Because join() cannot be interrupted by signal, a single join()
+        # needs to be separated into join()s with timeout in a while loop.
+        have_alive_child = True
+        while have_alive_child:
+            have_alive_child = False
+            for t in threads:
+                t.join(0.1)
+                if t.is_alive():
+                    have_alive_child = True
+
+        results = {}
+        while not result_queue.empty():
+            item = result_queue.get()
+            results[item[0]] = item[1]
+
+        if len(results) != len(args_list):
+            raise RuntimeError(
+                'Some threads for func {func} did not complete '
+                'successfully.'.format(func=fn.__name__))
+    return results
+
+
+if __name__ == '__main__':
+    import argparse
+
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--host', type=str)
+    parser.add_argument('--num-gpus', type=int, default=4)
+    parser.add_argument('--master', type=str, default='task1')
+    parser.add_argument('--port', type=int, default=1958)
+    parser.add_argument('-c', '--conf', type=str)
+    parser.add_argument('--args', type=str, default='')
+
+    args = parser.parse_args()
+
+    try:
+        hosts = ['task%d' % (x + 1) for x in range(int(args.host))]
+    except:
+        hosts = args.host.split(',')
+
+    cwd = os.getcwd()
+    command_list = []
+    for node_rank, host in enumerate(hosts):
+        ssh_cmd = f'ssh -t -t -o StrictHostKeyChecking=no {host} -p 22 ' \
+                  f'\'bash -O huponexit -c "cd {cwd} && ' \
+                  f'python -m torch.distributed.launch --nproc_per_node={args.num_gpus} --nnodes={len(hosts)} ' \
+                  f'--master_addr={args.master} --master_port={args.port} --node_rank={node_rank} ' \
+                  f'FastAutoAugment/train.py -c {args.conf} {args.args}"' \
+                  '\''
+        print(ssh_cmd)
+
+        command_list.append([ssh_cmd])
+
+    execute_function_multithreaded(_exec_command,
+                                   command_list[1:],
+                                   block_until_all_done=False)
+
+    print(command_list[0])
+
+    while True:
+        time.sleep(1)
+
+    # thread = threading.Thread(target=safe_shell_exec.execute, args=(command_list[0][0],))
+    # thread.start()
+    # thread.join()
+
+    # while True:
+    #     time.sleep(1)

+MIT License
+
+Copyright (c) 2019 Ildoo Kim
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

+# Fast AutoAugment **(Accepted at NeurIPS 2019)**
+
+Official [Fast AutoAugment](https://arxiv.org/abs/1905.00397) implementation in PyTorch.
+
+- Fast AutoAugment learns augmentation policies using a more efficient search strategy based on density matching.
+- Fast AutoAugment speeds up the search time by orders of magnitude while maintaining the comparable performances.
+
+<p align="center">
+<img src="etc/search.jpg" height=350>
+</p>
+
+## Results
+
+### CIFAR-10 / 100
+
+Search : **3.5 GPU Hours (1428x faster than AutoAugment)**, WResNet-40x2 on Reduced CIFAR-10
+
+| Model(CIFAR-10)         | Baseline   | Cutout     | AutoAugment | Fast AutoAugment<br/>(transfer/direct) |   |
+|-------------------------|------------|------------|-------------|------------------|----|
+| Wide-ResNet-40-2        | 5.3        | 4.1        | 3.7         | 3.6 / 3.7        | [Download](https://arena.kakaocdn.net/brainrepo/fast-autoaugment/cifar10_wresnet40x2_top1_3.52.pth) |
+| Wide-ResNet-28-10       | 3.9        | 3.1        | 2.6         | 2.7 / 2.7        | [Download](https://arena.kakaocdn.net/brainrepo/fast-autoaugment/cifar10_wresnet28x10_top1.pth) |
+| Shake-Shake(26 2x32d)   | 3.6        | 3.0        | 2.5         | 2.7 / 2.5        | [Download](https://arena.kakaocdn.net/brainrepo/fast-autoaugment/cifar10_shake26_2x32d_top1_2.68.pth) |
+| Shake-Shake(26 2x96d)   | 2.9        | 2.6        | 2.0         | 2.0 / 2.0        | [Download](https://arena.kakaocdn.net/brainrepo/fast-autoaugment/cifar10_shake26_2x96d_top1_1.97.pth) |
+| Shake-Shake(26 2x112d)  | 2.8        | 2.6        | 1.9         | 2.0 / 1.9        | [Download](https://arena.kakaocdn.net/brainrepo/fast-autoaugment/cifar10_shake26_2x112d_top1_2.04.pth) |
+| PyramidNet+ShakeDrop    | 2.7        | 2.3        | 1.5         | 1.8 / 1.7        | [Download](https://arena.kakaocdn.net/brainrepo/fast-autoaugment/cifar10_pyramid272_top1_1.44.pth) |
+
+| Model(CIFAR-100)      | Baseline   | Cutout     | AutoAugment | Fast AutoAugment<br/>(transfer/direct) |    |
+|-----------------------|------------|------------|-------------|------------------|----|
+| Wide-ResNet-40-2      | 26.0       | 25.2       | 20.7        | 20.7 / 20.6      | [Download](https://arena.kakaocdn.net/brainrepo/fast-autoaugment/cifar100_wresnet40x2_top1_20.43.pth) |
+| Wide-ResNet-28-10     | 18.8       | 18.4       | 17.1        | 17.3 / 17.3      | [Download](https://arena.kakaocdn.net/brainrepo/fast-autoaugment/cifar100_wresnet28x10_top1_17.17.pth) |
+| Shake-Shake(26 2x96d) | 17.1       | 16.0       | 14.3        | 14.9 / 14.6      | [Download](https://arena.kakaocdn.net/brainrepo/fast-autoaugment/cifar100_shake26_2x96d_top1_15.15.pth) |
+| PyramidNet+ShakeDrop  | 14.0       | 12.2       | 10.7        | 11.9 / 11.7      | [Download](https://arena.kakaocdn.net/brainrepo/fast-autoaugment/cifar100_pyramid272_top1_11.74.pth) |
+
+### ImageNet
+
+Search : **450 GPU Hours (33x faster than AutoAugment)**, ResNet-50 on Reduced ImageNet
+
+| Model      | Baseline   | AutoAugment | Fast AutoAugment<br/>(Top1/Top5) |    |
+|------------|------------|-------------|------------------|----|
+| ResNet-50  | 23.7 / 6.9 | 22.4 / 6.2  | **22.4 / 6.3**   | [Download](https://arena.kakaocdn.net/brainrepo/fast-autoaugment/imagenet_resnet50_top1_22.2.pth) |
+| ResNet-200 | 21.5 / 5.8 | 20.0 / 5.0  | **19.4 / 4.7**   | [Download](https://arena.kakaocdn.net/brainrepo/fast-autoaugment/imagenet_resnet200_top1_19.4.pth) |
+
+Notes
+* We evaluated resnet-50 and resnet-200 with resolution of 224 and 320, respectively. According to the original resnet paper, resnet 200 was tested with the resolution of 320. Also our resnet-200 baseline's performance was similar when we use the resolution.
+* But with recent our code clean-up and bugfixes, we've found that the baseline performs similar to the baseline even using 224x224.
+* When we use 224x224, resnet-200 performs **20.0 / 5.2**. Download link for the trained model is [here](https://arena.kakaocdn.net/brainrepo/fast-autoaugment/imagenet_resnet200_res224.pth).
+
+We have conducted additional experiments with EfficientNet.
+
+| Model | Baseline   | AutoAugment |   | Our Baseline(Batch) | +Fast AA |
+|-------|------------|-------------|---|---------------------|----------|
+| B0    | 23.2       | 22.7        |   | 22.96               | 22.68    |
+
+### SVHN Test
+
+Search : **1.5 GPU Hours**
+
+|                                  | Baseline | AutoAug / Our | Fast AutoAugment  |
+|----------------------------------|---------:|--------------:|--------:|
+| Wide-Resnet28x10                 | 1.5      | 1.1           | 1.1     |
+
+## Run
+
+We conducted experiments under
+
+- python 3.6.9
+- pytorch 1.2.0, torchvision 0.4.0, cuda10
+
+### Search a augmentation policy
+
+Please read ray's document to construct a proper ray cluster : https://github.com/ray-project/ray, and run search.py with the master's redis address.
+
+```
+$ python search.py -c confs/wresnet40x2_cifar10_b512.yaml --dataroot ... --redis ...
+```
+
+### Train a model with found policies
+
+You can train network architectures on CIFAR-10 / 100 and ImageNet with our searched policies.
+
+- fa_reduced_cifar10 : reduced CIFAR-10(4k images), WResNet-40x2
+- fa_reduced_imagenet : reduced ImageNet(50k images, 120 classes), ResNet-50
+
+```
+$ export PYTHONPATH=$PYTHONPATH:$PWD
+$ python FastAutoAugment/train.py -c confs/wresnet40x2_cifar10_b512.yaml --aug fa_reduced_cifar10 --dataset cifar10
+$ python FastAutoAugment/train.py -c confs/wresnet40x2_cifar10_b512.yaml --aug fa_reduced_cifar10 --dataset cifar100
+$ python FastAutoAugment/train.py -c confs/wresnet28x10_cifar10_b512.yaml --aug fa_reduced_cifar10 --dataset cifar10
+$ python FastAutoAugment/train.py -c confs/wresnet28x10_cifar10_b512.yaml --aug fa_reduced_cifar10 --dataset cifar100
+...
+$ python FastAutoAugment/train.py -c confs/resnet50_b512.yaml --aug fa_reduced_imagenet
+$ python FastAutoAugment/train.py -c confs/resnet200_b512.yaml --aug fa_reduced_imagenet
+```
+
+By adding --only-eval and --save arguments, you can test trained models without training.
+
+If you want to train with multi-gpu/node, use `torch.distributed.launch` such as
+
+```bash
+$ python -m torch.distributed.launch --nproc_per_node={num_gpu_per_node} --nnodes={num_node} --master_addr={master} --master_port={master_port} --node_rank={0,1,2,...,num_node} FastAutoAugment/train.py -c confs/efficientnet_b4.yaml --aug fa_reduced_imagenet
+```
+
+## Citation
+
+If you use this code in your research, please cite our [paper](https://arxiv.org/abs/1905.00397).
+
+```
+@inproceedings{lim2019fast,
+  title={Fast AutoAugment},
+  author={Lim, Sungbin and Kim, Ildoo and Kim, Taesup and Kim, Chiheon and Kim, Sungwoong},
+  booktitle={Advances in Neural Information Processing Systems (NeurIPS)},
+  year={2019}
+}
+```
+
+## Contact for Issues
+- Ildoo Kim, ildoo.kim@kakaobrain.com
+
+## References & Opensources
+
+We increase the batch size and adapt the learning rate accordingly to boost the training. Otherwise, we set other hyperparameters equal to AutoAugment if possible. For the unknown hyperparameters, we follow values from the original references or we tune them to match baseline performances.
+
+- **ResNet** : [paper1](https://arxiv.org/abs/1512.03385), [paper2](https://arxiv.org/abs/1603.05027), [code](https://github.com/osmr/imgclsmob/tree/master/pytorch/pytorchcv/models)
+- **PyramidNet** : [paper](https://arxiv.org/abs/1610.02915), [code](https://github.com/dyhan0920/PyramidNet-PyTorch)
+- **Wide-ResNet** : [code](https://github.com/meliketoy/wide-resnet.pytorch)
+- **Shake-Shake** : [code](https://github.com/owruby/shake-shake_pytorch)
+- **ShakeDrop Regularization** : [paper](https://arxiv.org/abs/1802.02375), [code](https://github.com/owruby/shake-drop_pytorch)
+- **AutoAugment** : [code](https://github.com/tensorflow/models/tree/master/research/autoaugment)
+- **Ray** : [code](https://github.com/ray-project/ray)
+- **HyperOpt** : [code](https://github.com/hyperopt/hyperopt)

+model:
+  type: efficientnet-b0
+  condconv_num_expert: 1  # if this is greater than 1(eg. 4), it activates condconv.
+dataset: imagenet
+aug: fa_reduced_imagenet
+cutout: 0
+batch: 128    # per gpu
+epoch: 350
+lr: 0.008     # 0.256 for 4096 batch
+lr_schedule:
+  type: 'efficientnet'
+  warmup:
+    multiplier: 1
+    epoch: 5
+optimizer:
+  type: rmsprop
+  decay: 0.00001
+  clip: 0
+  ema: 0.9999
+  ema_interval: -1
+lb_smooth: 0.1
\ No newline at end of file

+model:
+  type: efficientnet-b0
+  condconv_num_expert: 8  # if this is greater than 1(eg. 4), it activates condconv.
+dataset: imagenet
+aug: fa_reduced_imagenet
+cutout: 0
+batch: 128    # per gpu
+epoch: 350
+lr: 0.008     # 0.256 for 4096 batch
+lr_schedule:
+  type: 'efficientnet'
+  warmup:
+    multiplier: 1
+    epoch: 5
+optimizer:
+  type: rmsprop
+  decay: 0.00001
+  clip: 0
+  ema: 0.9999
+  ema_interval: -1
+lb_smooth: 0.1
+mixup: 0.2

+model:
+  type: efficientnet-b1
+  condconv_num_expert: 1  # if this is greater than 1(eg. 4), it activates condconv.
+dataset: imagenet
+aug: fa_reduced_imagenet
+cutout: 0
+batch: 128    # per gpu
+epoch: 350
+lr: 0.008     # 0.256 for 4096 batch
+lr_schedule:
+  type: 'efficientnet'
+  warmup:
+    multiplier: 1
+    epoch: 5
+optimizer:
+  type: rmsprop
+  decay: 0.00001
+  clip: 0
+  ema: 0.9999
+  ema_interval: -1
+lb_smooth: 0.1

+model:
+  type: efficientnet-b2
+  condconv_num_expert: 1  # if this is greater than 1(eg. 4), it activates condconv.
+dataset: imagenet
+aug: fa_reduced_imagenet
+cutout: 0
+batch: 128    # per gpu
+epoch: 350
+lr: 0.008     # 0.256 for 4096 batch
+lr_schedule:
+  type: 'efficientnet'
+  warmup:
+    multiplier: 1
+    epoch: 5
+optimizer:
+  type: rmsprop
+  decay: 0.00001
+  clip: 0
+  ema: 0.9999
+  ema_interval: -1
+lb_smooth: 0.1

+model:
+  type: efficientnet-b3
+  condconv_num_expert: 1  # if this is greater than 1(eg. 4), it activates condconv.
+dataset: imagenet
+aug: fa_reduced_imagenet
+cutout: 0
+batch: 64    # per gpu
+epoch: 350
+lr: 0.004     # 0.256 for 4096 batch
+lr_schedule:
+  type: 'efficientnet'
+  warmup:
+    multiplier: 1
+    epoch: 5
+optimizer:
+  type: rmsprop
+  decay: 0.00001
+  clip: 0
+  ema: 0.9999
+  ema_interval: -1
+lb_smooth: 0.1

+model:
+  type: efficientnet-b4
+  condconv_num_expert: 1  # if this is greater than 1(eg. 4), it activates condconv.
+dataset: imagenet
+aug: fa_reduced_imagenet
+cutout: 0
+batch: 32    # per gpu
+epoch: 350
+lr: 0.002     # 0.256 for 4096 batch
+lr_schedule:
+  type: 'efficientnet'
+  warmup:
+    multiplier: 1
+    epoch: 5
+optimizer:
+  type: rmsprop
+  decay: 0.00001
+  clip: 0
+  ema: 0.9999
+  ema_interval: -1
+lb_smooth: 0.1

+model:
+  type: pyramid
+  depth: 272
+  alpha: 200
+  bottleneck: True
+dataset: cifar10
+aug: fa_reduced_cifar10
+cutout: 16
+batch: 64
+epoch: 1800
+lr: 0.05
+lr_schedule:
+  type: 'cosine'
+  warmup:
+    multiplier: 1
+    epoch: 5
+optimizer:
+  type: sgd
+  nesterov: True
+  decay: 0.00005

+model:
+  type: resnet200
+dataset: imagenet
+aug: fa_reduced_imagenet
+cutout: 0
+batch: 64
+epoch: 270
+lr: 0.025
+lr_schedule:
+  type: 'resnet'
+  warmup:
+    multiplier: 1
+    epoch: 5
+optimizer:
+  type: sgd
+  nesterov: True
+  decay: 0.0001
+  clip: 0

+model:
+  type: resnet50
+dataset: imagenet
+aug: fa_reduced_imagenet
+cutout: 0
+batch: 128
+epoch: 270
+lr: 0.05
+lr_schedule:
+  type: 'resnet'
+  warmup:
+    multiplier: 1
+    epoch: 5
+optimizer:
+  type: sgd
+  nesterov: True
+  decay: 0.0001
+  clip: 0
+  ema: 0

+model:
+  type: resnet50
+dataset: imagenet
+aug: fa_reduced_imagenet
+cutout: 0
+batch: 128
+epoch: 270
+lr: 0.05
+lr_schedule:
+  type: 'resnet'
+  warmup:
+    multiplier: 1
+    epoch: 5
+optimizer:
+  type: sgd
+  nesterov: True
+  decay: 0.0001
+  clip: 0
+  ema: 0
+#lb_smooth: 0.1
+mixup: 0.2

+model:
+  type: shakeshake26_2x112d
+dataset: cifar10
+aug: fa_reduced_cifar10
+cutout: 16
+batch: 128
+epoch: 1800
+lr: 0.01
+lr_schedule:
+  type: 'cosine'
+  warmup:
+    multiplier: 1
+    epoch: 5
+optimizer:
+  type: sgd
+  nesterov: True
+  decay: 0.002

+model:
+  type: shakeshake26_2x32d
+dataset: cifar10
+aug: fa_reduced_cifar10
+cutout: 16
+batch: 128
+epoch: 1800
+lr: 0.01
+lr_schedule:
+  type: 'cosine'
+  warmup:
+    multiplier: 1
+    epoch: 5
+optimizer:
+  type: sgd
+  nesterov: True
+  decay: 0.001

+model:
+  type: shakeshake26_2x96d
+dataset: cifar10
+aug: fa_reduced_cifar10
+cutout: 16
+batch: 128
+epoch: 1800
+lr: 0.01
+lr_schedule:
+  type: 'cosine'
+  warmup:
+    multiplier: 1
+    epoch: 5
+optimizer:
+  type: sgd
+  nesterov: True
+  decay: 0.001

+model:
+  type: wresnet28_10
+dataset: cifar10
+aug: fa_reduced_cifar10
+cutout: 16
+batch: 128
+epoch: 200
+lr: 0.1
+lr_schedule:
+  type: 'cosine'
+  warmup:
+    multiplier: 1
+    epoch: 5
+optimizer:
+  type: sgd
+  nesterov: True
+  decay: 0.0005
\ No newline at end of file

+model:
+  type: wresnet28_10
+dataset: svhn
+aug: fa_reduced_svhn
+cutout: 20
+batch: 128
+epoch: 200
+lr: 0.01
+lr_schedule:
+  type: 'cosine'
+  warmup:
+    multiplier: 1
+    epoch: 5
+optimizer:
+  type: sgd
+  nesterov: True
+  decay: 0.0005
\ No newline at end of file

+model:
+  type: wresnet40_2
+dataset: cifar10
+aug: fa_reduced_cifar10
+cutout: 16
+batch: 128
+epoch: 200
+lr: 0.1
+lr_schedule:
+  type: 'cosine'
+  warmup:
+    multiplier: 1
+    epoch: 5
+optimizer:
+  type: sgd
+  nesterov: True
+  decay: 0.0002

+git+https://github.com/wbaek/theconf
+git+https://github.com/ildoonet/pytorch-gradual-warmup-lr.git@08f7d5e
+git+https://github.com/ildoonet/pystopwatch2.git
+git+https://github.com/hyperopt/hyperopt.git
+git+https://github.com/kakaobrain/torchlars
+
+pretrainedmodels
+tqdm
+tensorboardx
+sklearn
+ray
+matplotlib
+psutil
+requests
\ No newline at end of file

+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import pandas as pd"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>id</th>\n",
+       "      <th>Label</th>\n",
+       "      <th>Subject</th>\n",
+       "      <th>Date</th>\n",
+       "      <th>Gender</th>\n",
+       "      <th>Age</th>\n",
+       "      <th>mmse</th>\n",
+       "      <th>ageAtEntry</th>\n",
+       "      <th>cdr</th>\n",
+       "      <th>commun</th>\n",
+       "      <th>...</th>\n",
+       "      <th>memory</th>\n",
+       "      <th>orient</th>\n",
+       "      <th>perscare</th>\n",
+       "      <th>apoe</th>\n",
+       "      <th>sumbox</th>\n",
+       "      <th>acsparnt</th>\n",
+       "      <th>height</th>\n",
+       "      <th>weight</th>\n",
+       "      <th>primStudy</th>\n",
+       "      <th>acsStudy</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>0</th>\n",
+       "      <td>/@WEBAPP/images/r.gif</td>\n",
+       "      <td>OAS30001_ClinicalData_d3025</td>\n",
+       "      <td>OAS30001</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>female</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>30.0</td>\n",
+       "      <td>65.149895</td>\n",
+       "      <td>0.0</td>\n",
+       "      <td>0.0</td>\n",
+       "      <td>...</td>\n",
+       "      <td>0.0</td>\n",
+       "      <td>0.0</td>\n",
+       "      <td>0.0</td>\n",
+       "      <td>23.0</td>\n",
+       "      <td>0.0</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>64.0</td>\n",
+       "      <td>180.0</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1</th>\n",
+       "      <td>/@WEBAPP/images/r.gif</td>\n",
+       "      <td>OAS30001_ClinicalData_d3977</td>\n",
+       "      <td>OAS30001</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>female</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>29.0</td>\n",
+       "      <td>65.149895</td>\n",
+       "      <td>0.0</td>\n",
+       "      <td>0.0</td>\n",
+       "      <td>...</td>\n",
+       "      <td>0.0</td>\n",
+       "      <td>0.0</td>\n",
+       "      <td>0.0</td>\n",
+       "      <td>23.0</td>\n",
+       "      <td>0.0</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2</th>\n",
+       "      <td>/@WEBAPP/images/r.gif</td>\n",
+       "      <td>OAS30001_ClinicalData_d3332</td>\n",
+       "      <td>OAS30001</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>female</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>30.0</td>\n",
+       "      <td>65.149895</td>\n",
+       "      <td>0.0</td>\n",
+       "      <td>0.0</td>\n",
+       "      <td>...</td>\n",
+       "      <td>0.0</td>\n",
+       "      <td>0.0</td>\n",
+       "      <td>0.0</td>\n",
+       "      <td>23.0</td>\n",
+       "      <td>0.0</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>63.0</td>\n",
+       "      <td>185.0</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>3</th>\n",
+       "      <td>/@WEBAPP/images/r.gif</td>\n",
+       "      <td>OAS30001_ClinicalData_d0000</td>\n",
+       "      <td>OAS30001</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>female</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>28.0</td>\n",
+       "      <td>65.149895</td>\n",
+       "      <td>0.0</td>\n",
+       "      <td>0.0</td>\n",
+       "      <td>...</td>\n",
+       "      <td>0.0</td>\n",
+       "      <td>0.0</td>\n",
+       "      <td>0.0</td>\n",
+       "      <td>23.0</td>\n",
+       "      <td>0.0</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>4</th>\n",
+       "      <td>/@WEBAPP/images/r.gif</td>\n",
+       "      <td>OAS30001_ClinicalData_d1456</td>\n",
+       "      <td>OAS30001</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>female</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>30.0</td>\n",
+       "      <td>65.149895</td>\n",
+       "      <td>0.0</td>\n",
+       "      <td>0.0</td>\n",
+       "      <td>...</td>\n",
+       "      <td>0.0</td>\n",
+       "      <td>0.0</td>\n",
+       "      <td>0.0</td>\n",
+       "      <td>23.0</td>\n",
+       "      <td>0.0</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>63.0</td>\n",
+       "      <td>173.0</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "<p>5 rows × 27 columns</p>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "                      id                        Label   Subject  Date  Gender  \\\n",
+       "0  /@WEBAPP/images/r.gif  OAS30001_ClinicalData_d3025  OAS30001   NaN  female   \n",
+       "1  /@WEBAPP/images/r.gif  OAS30001_ClinicalData_d3977  OAS30001   NaN  female   \n",
+       "2  /@WEBAPP/images/r.gif  OAS30001_ClinicalData_d3332  OAS30001   NaN  female   \n",
+       "3  /@WEBAPP/images/r.gif  OAS30001_ClinicalData_d0000  OAS30001   NaN  female   \n",
+       "4  /@WEBAPP/images/r.gif  OAS30001_ClinicalData_d1456  OAS30001   NaN  female   \n",
+       "\n",
+       "   Age  mmse  ageAtEntry  cdr  commun  ... memory orient perscare  apoe  \\\n",
+       "0  NaN  30.0   65.149895  0.0     0.0  ...    0.0    0.0      0.0  23.0   \n",
+       "1  NaN  29.0   65.149895  0.0     0.0  ...    0.0    0.0      0.0  23.0   \n",
+       "2  NaN  30.0   65.149895  0.0     0.0  ...    0.0    0.0      0.0  23.0   \n",
+       "3  NaN  28.0   65.149895  0.0     0.0  ...    0.0    0.0      0.0  23.0   \n",
+       "4  NaN  30.0   65.149895  0.0     0.0  ...    0.0    0.0      0.0  23.0   \n",
+       "\n",
+       "  sumbox  acsparnt  height  weight  primStudy  acsStudy  \n",
+       "0    0.0       NaN    64.0   180.0        NaN       NaN  \n",
+       "1    0.0       NaN     NaN     NaN        NaN       NaN  \n",
+       "2    0.0       NaN    63.0   185.0        NaN       NaN  \n",
+       "3    0.0       NaN     NaN     NaN        NaN       NaN  \n",
+       "4    0.0       NaN    63.0   173.0        NaN       NaN  \n",
+       "\n",
+       "[5 rows x 27 columns]"
+      ]
+     },
+     "execution_count": 3,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "all_data = pd.read_csv(\"..\\data\\ADRC clinical data_all.csv\")\n",
+    "\n",
+    "all_data.head()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 18,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Subject\n",
+      "OAS30001    0.0\n",
+      "OAS30002    0.0\n",
+      "OAS30003    0.0\n",
+      "OAS30004    0.0\n",
+      "OAS30005    0.0\n",
+      "           ... \n",
+      "OAS31168    0.0\n",
+      "OAS31169    3.0\n",
+      "OAS31170    2.0\n",
+      "OAS31171    2.0\n",
+      "OAS31172    0.0\n",
+      "Name: cdr, Length: 1098, dtype: float64\n"
+     ]
+    }
+   ],
+   "source": [
+    "ad = all_data.groupby(['Subject'])['cdr'].max()\n",
+    "print(ad)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 21,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "'OAS30001'"
+      ]
+     },
+     "execution_count": 21,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "ad.index[0]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 22,
+   "metadata": {},
+   "outputs": [
+    {
+     "ename": "SyntaxError",
+     "evalue": "unexpected EOF while parsing (<ipython-input-22-b8a078b72aca>, line 5)",
+     "output_type": "error",
+     "traceback": [
+      "\u001b[1;36m  File \u001b[1;32m\"<ipython-input-22-b8a078b72aca>\"\u001b[1;36m, line \u001b[1;32m5\u001b[0m\n\u001b[1;33m    #print(filtered)\u001b[0m\n\u001b[1;37m                    ^\u001b[0m\n\u001b[1;31mSyntaxError\u001b[0m\u001b[1;31m:\u001b[0m unexpected EOF while parsing\n"
+     ]
+    }
+   ],
+   "source": [
+    "filtered = []\n",
+    "for i, val in enumerate(ad):\n",
+    "    if ad[i] == 0:\n",
+    "        filtered.append(ad.index[i])\n",
+    "#print(filtered)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 23,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "df_filtered = pd.DataFrame(filtered)\n",
+    "df_filtered.to_csv('..\\data\\ADRC clinical data_normal.csv')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "ML",
+   "language": "python",
+   "name": "ml"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.7.4"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

+%load('..\data\MICCAI_BraTS_2019_Data_Training\name_mapping.csv')
+
+inputheader = '..\data\MICCAI_BraTS_2019_Data_Training\HGG\';
+outfolder = '..\testfolder\';
+id = 'BraTS19_2013_2_1';
+
+type = 'flair.nii';
+filename = strcat(id,'_', type); % BraTS19_2013_2_1_flair.nii
+flair_path = strcat(inputheader, id, '\', filename,'\', filename);
+%disp(path);
+flair = niftiread(flair_path); %size 240x240x155 
+cp_flair = flair;
+
+type = 'seg.nii';
+filename = strcat(id,'_', type); % BraTS19_2013_2_1_seg.nii 
+seg_path = strcat(inputheader, id, '\', filename, '\', filename);
+seg = niftiread(seg_path);
+
+[x,y,z] = size(seg);
+
+% copy flair, segment flair data
+
+cp_flair(seg == 0) = 0;
+
+% save a segmented data
+type = 'seg_flair.nii';
+filename = strcat(id,'_', type); % BraTS19_2013_2_1_seg_flair.nii 
+outpath = strcat(outfolder, filename);
+%niftiwrite(cp_flair, outpath);
+
+
+%whos seg
+
+
+%extract = seg(84, :, 86);
+
+
+
+
+% cp84 = cp_flair(84,:,86);
+% flair84 = flair(84,:, 86);
+
+
+
+%[flair,info] = ReadData3D(filename)
+% whos flair
+
+%volumeViewer(flair);
\ No newline at end of file

+inputheader = '..\data\MICCAI_BraTS_2019_Data_Training\HGG\';
+outfolder = strcat('..\data\MICCAI_BraTS_2019_Data_Training\HGG_seg_flair\');
+
+files = dir(inputheader);
+id = {files.name};
+%  files + dir  dir 
+dirFlag = [files.isdir] & ~strcmp(id, '.') & ~strcmp(id, '..'); 
+subFolders = files(dirFlag);
+disp(length(subFolders));
+
+% for k = 1 : length(subFolders)
+%   fprintf('Sub folder #%d = %s\n', k, subFolders(k).name);
+% end
+
+for i = 1 : length(subFolders)
+    
+    id = subFolders(i).name;
+    fprintf('Sub folder #%d = %s\n', i, id);
+
+    type = 'flair.nii';
+    filename = strcat(id,'_', type); % BraTS19_2013_2_1_flair.nii
+    flair_path = strcat(inputheader, id, '\', filename,'\', filename);
+    flair = niftiread(flair_path); %size 240x240x155 
+    cp_flair = flair;
+
+    type = 'seg.nii';
+    filename = strcat(id,'_', type); % BraTS19_2013_2_1_seg.nii 
+    seg_path = strcat(inputheader, id, '\', filename, '\', filename);
+    seg = niftiread(seg_path);
+
+    [x,y,z] = size(seg);
+
+    % copy flair, segment flair data
+
+    cp_flair(seg == 0) = 0;
+
+    % save a segmented data
+    type = 'seg_flair.nii';
+    filename = strcat(id,'_', type); % BraTS19_2013_2_1_seg_flair.nii 
+    outpath = strcat(outfolder, filename);
+    niftiwrite(cp_flair, outpath);
+
+end

+inputheader = '..\data\MICCAI_BraTS_2019_Data_Training\HGG\';
+outfolder = '..\testfolder\';
+id = 'BraTS19_2013_2_1';
+
+type = 'seg_flair.nii';
+filename = strcat(id,'_', type); % BraTS19_2013_2_1_flair.nii
+flair_path = strcat(outfolder, filename);
+%disp(path);
+ffffff = niftiread(flair_path); %size 240x240x155 
+
+fff84 = ffffff(84, :, 86);
\ No newline at end of file