{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'\\n평가방법 : positive를 높이는방식\\n\\n본논문\\n- STFT magnitude Spectrun\\n- n=40 log mel filter bank\\n\\n다른논문\\n-STFT maginitude spectogram\\n- n=80 mel scaled filter bank\\n- scale log magnitude\\n- batch nomalization (0,1)\\n- subtract mean overtime on spectogram (for remove frequency dependency noise = colored noise)\\n'"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "'''\n",
    "평가방법 : positive를 높이는방식\n",
    "\n",
    "본논문\n",
    "- STFT magnitude Spectrun\n",
    "- n=40 log mel filter bank\n",
    "\n",
    "다른논문\n",
    "-STFT maginitude spectogram\n",
    "- n=80 mel scaled filter bank\n",
    "- scale log magnitude\n",
    "- batch nomalization (0,1)\n",
    "- subtract mean overtime on spectogram (for remove frequency dependency noise = colored noise)\n",
    "'''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[name: \"/device:CPU:0\"\n",
      "device_type: \"CPU\"\n",
      "memory_limit: 268435456\n",
      "locality {\n",
      "}\n",
      "incarnation: 1817485338440268463\n",
      "]\n"
     ]
    }
   ],
   "source": [
    "from tensorflow.python.client import device_lib\n",
    "print(device_lib.list_local_devices())\n",
    "\n",
    "from keras.utils.training_utils import multi_gpu_model\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "gpunum = 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "#path 관련 라이브러리\n",
    "import glob\n",
    "import csv\n",
    "\n",
    "#csv저장 라이브러리\n",
    "import pandas as pd\n",
    "\n",
    "# Scientific Math 라이브러리  \n",
    "import numpy as np\n",
    "import librosa\n",
    "import librosa.display\n",
    "import os\n",
    "\n",
    "# Visualization 라이브러리\n",
    "import matplotlib.pyplot as plt\n",
    "import IPython.display as ipd\n",
    "\n",
    "#keras\n",
    "from keras.utils import np_utils\n",
    "from keras.models import Sequential\n",
    "from keras.layers import Dense, Conv2D, MaxPooling2D, GRU,Dropout, Flatten,Reshape,BatchNormalization\n",
    "from keras.callbacks import ModelCheckpoint, EarlyStopping\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.metrics import roc_curve, auc, roc_auc_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "audio_path = './SOUNDS/ff1010bird/'\n",
    "audio_path2= './SOUNDS/warbler/'\n",
    "n_mels = 40\n",
    "n_frame = 500\n",
    "window_size=1024\n",
    "hop_size=512\n",
    "sample_rate=25600\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Preparing ff1010_labels.\n",
      "Done.\n",
      "Preparing walber_labels.\n",
      "Done.\n"
     ]
    }
   ],
   "source": [
    "#prepare labels\n",
    "print(\"Preparing ff1010_labels.\")\n",
    "labels_1=[]\n",
    "with open(audio_path+'labels.csv', mode='r',encoding='utf-8') as f:\n",
    "    reader = csv.reader(f)\n",
    "    for row in reader : \n",
    "        labels_1.append(row)\n",
    "labels_1.sort(key=lambda x:x[0])\n",
    "labels_1 = np.array(labels_1) #아...그냥이렇게하면 넘피배열로 바꿀수있구나ㅠ\n",
    "labels_1 = labels_1[0:-1,1]\n",
    "print(\"Done.\")\n",
    "\n",
    "\n",
    "print(\"Preparing walber_labels.\")\n",
    "labels_2 = []\n",
    "with open(audio_path2+'labels.csv', mode='r',encoding='utf-8') as f:\n",
    "    reader = csv.reader(f)\n",
    "    for row in reader : \n",
    "        labels_2.append(row)\n",
    "labels_2.sort(key=lambda x:x[0])\n",
    "labels_2 = np.array(labels_2) #아...그냥이렇게하면 넘피배열로 바꿀수있구나ㅠ\n",
    "labels_2 = labels_2[0:6000,1]\n",
    "print(\"Done.\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Preparing ff1010_melspectogram.\n",
      "part1_done\n",
      "part2_done.\n",
      "part3_done\n",
      "part4_done.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(7690, 20000)"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\n",
    "# prepare ff1010_melspecotograms\n",
    "print(\"Preparing ff1010_melspectogram.\")\n",
    "mel_spectogram_1 = []\n",
    "with open(audio_path+'mel_spec.csv', mode='r',encoding='utf-8') as f:\n",
    "    reader = csv.reader(f)\n",
    "    next(reader)\n",
    "    for row in reader : \n",
    "        mel_spectogram_1.append(row)\n",
    "print('Done')\n",
    "\n",
    "mel_spectogram_1 = np.array(mel_spectogram_1)\n",
    "mel_spectogram_1= mel_spectogram_1[:,1:]\n",
    "\n",
    "np.shape(mel_spectogram_1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Preparing ff1010_melspectogram.\n",
      "part1\n",
      "part2\n",
      "Done.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(6000, 20000)"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# prepare ff1010_melspecotograms\n",
    "print(\"Preparing walber_melspectogram.\")\n",
    "mel_spectogram_2 = []\n",
    "with open(audio_path2+'mel_spec.csv', mode='r',encoding='utf-8') as f:\n",
    "    reader = csv.reader(f)\n",
    "    next(reader)\n",
    "    for row in reader : \n",
    "        mel_spectogram_2.append(row)\n",
    "print('Done')\n",
    "\n",
    "\n",
    "mel_spectogram_2 = np.array(mel_spectogram_2)\n",
    "mel_spectogram_2= mel_spectogram_2[:,1:]\n",
    "\n",
    "np.shape(mel_spectogram_2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(13690,)"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\n",
    "MODEL_SAVE_FOLDER_PATH = './model/'\n",
    "\n",
    "if not os.path.exists(MODEL_SAVE_FOLDER_PATH):\n",
    "    os.mkdir(MODEL_SAVE_FOLDER_PATH)\n",
    "\n",
    "model_path = MODEL_SAVE_FOLDER_PATH + 'bird_sound-' + '{epoch:02d}-{val_loss:.4f}.hdf5'\n",
    "\n",
    "cb_checkpoint = ModelCheckpoint(filepath=model_path, monitor='val_loss',\n",
    "                                verbose=1, save_best_only=True)\n",
    "\n",
    "cb_early_stopping = EarlyStopping(monitor='val_loss', patience=50)\n",
    "\n",
    "\n",
    "ALL_Spectrogram = np.concatenate((mel_spectogram_1,mel_spectogram_2),axis=0) \n",
    "X_train = ALL_Spectrogram[:14000,-1]\n",
    "X_train = np.reshape(X_train,(14000,40,500,1))\n",
    "X_test = ALL_Spectrogram[14000:,-1]\n",
    "X_test = np.reshape(X_test,(14000,40,500,1))\n",
    "\n",
    "ALL_Labels = np.concatenate((labels_1,labels_2),axis=0) \n",
    "Y_train = ALL_Labels[:14000,-1]\n",
    "Y_train = np.reshape(Y_train,(14000))\n",
    "Y_test = ALL_Labels[14000:,-1]\n",
    "Y_test = np.reshape(Y_test,(14000))\n",
    "np.shape(Y_train)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(None, 40, 500, 96)\n",
      "(None, 8, 500, 96)\n",
      "(None, 8, 500, 96)\n",
      "(None, 4, 500, 96)\n",
      "(None, 4, 500, 96)\n",
      "(None, 2, 500, 96)\n",
      "(None, 2, 500, 96)\n",
      "(None, 1, 500, 96)\n",
      "(None, 96, 500)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/ubuntu/anaconda3/envs/tensorflow_p36/lib/python3.6/site-packages/ipykernel_launcher.py:54: UserWarning: Update your `GRU` call to the Keras 2 API: `GRU(return_sequences=True, units=500)`\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(None, 96, 500)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/ubuntu/anaconda3/envs/tensorflow_p36/lib/python3.6/site-packages/ipykernel_launcher.py:57: UserWarning: Update your `GRU` call to the Keras 2 API: `GRU(return_sequences=True, units=500)`\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(None, 96, 500)\n",
      "(None, 96, 500, 1)\n",
      "(None, 96, 1, 1)\n",
      "Train on 10267 samples, validate on 3423 samples\n",
      "Epoch 1/50\n",
      " - 431s - loss: 0.3637 - acc: 0.8467 - val_loss: 0.4091 - val_acc: 0.7885\n",
      "\n",
      "Epoch 00001: val_loss did not improve from 0.40740\n",
      "Epoch 2/50\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-9-ff0adce6f8b3>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     76\u001b[0m                         \u001b[0mvalidation_split\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m0.25\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     77\u001b[0m                         \u001b[0mepochs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m50\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbatch_size\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m200\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mverbose\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m2\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 78\u001b[0;31m                         callbacks=[cb_checkpoint, cb_early_stopping])\n\u001b[0m\u001b[1;32m     79\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     80\u001b[0m \u001b[0;31m# print('\\nAccuracy: {:.4f}'.format(model.evaluate(X_validation, Y_validation)[1]))\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/tensorflow_p36/lib/python3.6/site-packages/keras/engine/training.py\u001b[0m in \u001b[0;36mfit\u001b[0;34m(self, x, y, batch_size, epochs, verbose, callbacks, validation_split, validation_data, shuffle, class_weight, sample_weight, initial_epoch, steps_per_epoch, validation_steps, **kwargs)\u001b[0m\n\u001b[1;32m   1040\u001b[0m                                         \u001b[0minitial_epoch\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0minitial_epoch\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1041\u001b[0m                                         \u001b[0msteps_per_epoch\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0msteps_per_epoch\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1042\u001b[0;31m                                         validation_steps=validation_steps)\n\u001b[0m\u001b[1;32m   1043\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1044\u001b[0m     def evaluate(self, x=None, y=None,\n",
      "\u001b[0;32m~/anaconda3/envs/tensorflow_p36/lib/python3.6/site-packages/keras/engine/training_arrays.py\u001b[0m in \u001b[0;36mfit_loop\u001b[0;34m(model, f, ins, out_labels, batch_size, epochs, verbose, callbacks, val_f, val_ins, shuffle, callback_metrics, initial_epoch, steps_per_epoch, validation_steps)\u001b[0m\n\u001b[1;32m    183\u001b[0m                         \u001b[0;31m# Do not slice the training phase flag.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    184\u001b[0m                         ins_batch = slice_arrays(\n\u001b[0;32m--> 185\u001b[0;31m                             ins[:-1], batch_ids) + [ins[-1]]\n\u001b[0m\u001b[1;32m    186\u001b[0m                     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    187\u001b[0m                         \u001b[0mins_batch\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mslice_arrays\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mins\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbatch_ids\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/tensorflow_p36/lib/python3.6/site-packages/keras/utils/generic_utils.py\u001b[0m in \u001b[0;36mslice_arrays\u001b[0;34m(arrays, start, stop)\u001b[0m\n\u001b[1;32m    505\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mhasattr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mstart\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'shape'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    506\u001b[0m                 \u001b[0mstart\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mstart\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtolist\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 507\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;32mNone\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mx\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m \u001b[0;32melse\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mstart\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mx\u001b[0m \u001b[0;32min\u001b[0m \u001b[0marrays\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    508\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    509\u001b[0m             \u001b[0;32mreturn\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;32mNone\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mx\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m \u001b[0;32melse\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mstart\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0mstop\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mx\u001b[0m \u001b[0;32min\u001b[0m \u001b[0marrays\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/tensorflow_p36/lib/python3.6/site-packages/keras/utils/generic_utils.py\u001b[0m in \u001b[0;36m<listcomp>\u001b[0;34m(.0)\u001b[0m\n\u001b[1;32m    505\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mhasattr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mstart\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'shape'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    506\u001b[0m                 \u001b[0mstart\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mstart\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtolist\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 507\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;32mNone\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mx\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m \u001b[0;32melse\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mstart\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mx\u001b[0m \u001b[0;32min\u001b[0m \u001b[0marrays\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    508\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    509\u001b[0m             \u001b[0;32mreturn\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;32mNone\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mx\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m \u001b[0;32melse\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mstart\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0mstop\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mx\u001b[0m \u001b[0;32min\u001b[0m \u001b[0marrays\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "import keras.backend.tensorflow_backend as K\n",
    "def CRNN() : \n",
    "    model = Sequential()\n",
    "    model.add(Conv2D(96, kernel_size=(5, 5), input_shape=(40, 500,1), padding='same',activation='relu')) #어쩌면 40,500만해야할지두\n",
    "    print(model.output_shape)\n",
    "    model.add(BatchNormalization())\n",
    "    model.add(MaxPooling2D(pool_size=(5,1)))\n",
    "    model.add(Dropout(0.25))\n",
    "    print(model.output_shape)\n",
    "\n",
    "    model.add(Conv2D(96, (5, 5), padding='same',activation='relu'))\n",
    "    print(model.output_shape)\n",
    "    model.add(BatchNormalization())\n",
    "    model.add(MaxPooling2D(pool_size=(2,1)))\n",
    "    model.add(Dropout(0.25))\n",
    "    print(model.output_shape)\n",
    "\n",
    "    model.add(Conv2D(96, (5, 5), padding='same',activation='relu'))\n",
    "    print(model.output_shape)\n",
    "    model.add(BatchNormalization())\n",
    "    model.add(MaxPooling2D(pool_size=(2,1)))\n",
    "    model.add(Dropout(0.25))\n",
    "    print(model.output_shape)\n",
    "\n",
    "    model.add(Conv2D(96, (5, 5), padding='same', activation='relu'))\n",
    "    print(model.output_shape)\n",
    "    model.add(BatchNormalization())\n",
    "    model.add(MaxPooling2D(pool_size=(2,1)))\n",
    "    model.add(Dropout(0.25))\n",
    "    print(model.output_shape)\n",
    "\n",
    "    model.add(Reshape((96,500))) #문제될거같은데..\n",
    "    print(model.output_shape)\n",
    "\n",
    "    model.add(GRU(output_dim=500, return_sequences=True))\n",
    "    print(model.output_shape)\n",
    "\n",
    "    model.add(GRU(output_dim=500, return_sequences=True))\n",
    "    print(model.output_shape)\n",
    "\n",
    "    model.add(Reshape((96,500,1))) #문제될거같은데..2\n",
    "    print(model.output_shape)\n",
    "\n",
    "    model.add(MaxPooling2D(pool_size=(1,500)))\n",
    "    print(model.output_shape)\n",
    "\n",
    "    model.add(Flatten())\n",
    "    model.add(Dense(1, activation='sigmoid'))\n",
    "    model = multi_gpu_model(gpunum)\n",
    "    model.compile(loss='binary_crossentropy',\n",
    "                  optimizer='adam',\n",
    "                  metrics=['accuracy'])\n",
    "    # model.load_weights(MODEL_SAVE_FOLDER_PATH + 'bird_sound-' + '17-0.3943.hdf5')\n",
    "\n",
    "    return model\n",
    "\n",
    "\n",
    "def CRNN_Training(model) : \n",
    "\n",
    "    history = model.fit(X_train, Y_train, \n",
    "                        validation_split=0.25,\n",
    "                        epochs=50, batch_size=64, verbose=2,\n",
    "                        callbacks=[cb_checkpoint, cb_early_stopping])\n",
    "    \n",
    "    y_vloss = history.history['val_loss']\n",
    "    y_loss = history.history['loss']\n",
    "\n",
    "    x_len = numpy.arange(len(y_loss))\n",
    "    plt.plot(x_len, y_loss, marker='.', c='blue', label=\"Train-set Loss\")\n",
    "    plt.plot(x_len, y_vloss, marker='.', c='red', label=\"Validation-set Loss\")\n",
    "\n",
    "    plt.legend(loc='upper right')\n",
    "    plt.grid()\n",
    "    plt.xlabel('epoch')\n",
    "    plt.ylabel('loss')\n",
    "    plt.show()\n",
    "    \n",
    "    return model\n",
    "    \n",
    "def CRNN_EVALUATE(model):\n",
    "    score = model.evaluate(X_test,Y_test,batch_size=64,verbose=2)\n",
    "    print(\"%s: %.2f%%\" % (model.metrics_names[1], scores[1]*100))\n",
    "    \n",
    "    false_positive_rate, true_positive_rate, thresholds = roc_curve(Y_test, model.predict(X_test).ravel())\n",
    "    print auc(false_positive_rate, true_positive_rate)\n",
    "    return model\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "Layer = CRNN()\n",
    "Layer = CRNN_Training(Layer)\n",
    "Layer = CRNN_EVALUATE(Layer)\n",
    "# RECALL이 중요\n",
    "# TruePositive / ( TruePositive + TrueNegative )\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "'''\n",
    "Finally,\n",
    "for the bulbul submission, from each spectrogram we subtract\n",
    "its mean over time, as a simple way of removing frequency-\n",
    "dependent (colored) noise\n",
    "\n",
    "-> time axis의 평균\n",
    "'''\n",
    "'''\n",
    "\n",
    "the feature maps\n",
    "of the last convolutional layer are stacked over the frequency -> frequency axis\n",
    "axis and fed to 2 gated recurrent unit (GRU)\n",
    "'''\n",
    "\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}