Created
May 20, 2019 20:53
Revisions
-
charlesreid1 revised this gist
May 20, 2019 . 1 changed file with 20 additions and 3 deletions.There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters. Learn more about bidirectional Unicode charactersOriginal file line number Diff line number Diff line change @@ -5,14 +5,25 @@ "colab": { "name": "example_hyperas.ipynb", "version": "0.3.2", "provenance": [], "include_colab_link": true }, "kernelspec": { "name": "python3", "display_name": "Python 3" } }, "cells": [ { "cell_type": "markdown", "metadata": { "id": "view-in-github", "colab_type": "text" }, "source": [ "<a href=\"https://colab.research.google.com/gist/charlesreid1/3ea4561566f6e9277c29ad242cee9811/example_hyperas.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>" ] }, { "cell_type": "code", "metadata": { @@ -115,7 +126,7 @@ "colab_type": "code", "colab": { "base_uri": "https://localhost:8080/", "height": 3076 }, "outputId": "16eec283-d7ac-4aee-8167-89efb637755a" }, @@ -384,7 +395,13 @@ " - 10s - loss: 2.0434 - acc: 0.2848 - val_loss: 0.7661 - val_acc: 0.8870\n", "\n", "Epoch 2/2\n", " - 9s - loss: 1.3212 - acc: 0.5004 - val_loss: 0.4248 - val_acc: 0.9358\n", "\n", "Best validation acc of epoch:\n", "0.935833333492279\n", "Train on 54000 samples, validate on 6000 samples\n", "Epoch 1/2\n", " 60%|██████ | 3/5 [01:11<00:44, 22.49s/it, best loss: -0.935833333492279]" ], "name": "stdout" } -
May 20, 2019 .There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters. Learn more about bidirectional Unicode charactersOriginal file line number Diff line number Diff line change @@ -0,0 +1,420 @@ { "nbformat": 4, "nbformat_minor": 0, "metadata": { "colab": { "name": "example_hyperas.ipynb", "version": "0.3.2", "provenance": [] }, "kernelspec": { "name": "python3", "display_name": "Python 3" } }, "cells": [ { "cell_type": "code", "metadata": { "id": "tOG1pjCA9XKU", "colab_type": "code", "colab": {} }, "source": [ "!pip install --upgrade jupyter-console &> /dev/null" ], "execution_count": 0, "outputs": [] }, { "cell_type": "code", "metadata": { "id": "lKbEk3su9au8", "colab_type": "code", "colab": {} }, "source": [ "!pip install --upgrade hyperas &> /dev/null" ], "execution_count": 0, "outputs": [] }, { "cell_type": "code", "metadata": { "id": "tGBuNeOL_Y2F", "colab_type": "code", "colab": { "base_uri": "https://localhost:8080/", "height": 52 }, "outputId": "b43fa103-1d6d-40ac-92c5-019e33d2abd2" }, "source": [ "from google.colab import drive\n", "drive.mount('/gdrive')\n", "%ls /gdrive" ], "execution_count": 33, "outputs": [ { "output_type": "stream", "text": [ "Drive already mounted at /gdrive; to attempt to forcibly remount, call drive.mount(\"/gdrive\", force_remount=True).\n", "\u001b[0m\u001b[01;34m'My Drive'\u001b[0m/\n" ], "name": "stdout" } ] }, { "cell_type": "code", "metadata": { "id": "sP0w6p9y_jwI", "colab_type": "code", "colab": { "base_uri": "https://localhost:8080/", "height": 312 }, "outputId": "07324a32-6212-41ee-c8ec-46dd6ba97eaa" }, "source": [ "!ls '/gdrive/My Drive/Colab Notebooks'" ], "execution_count": 34, "outputs": [ { "output_type": "stream", "text": [ "'Copy of hyperparameter_cnn1d_dna_transcription.ipynb'\n", " data-exploration_dna_transcription.ipynb\n", " example_hyperas.ipynb\n", " hyperparameter_cnn1d_dna_transcription.ipynb\n", " keras_cnn1d_dna_transcription_powerx.ipynb\n", " keras_cnn1d_dna_transcription-round2.ipynb\n", " keras-mnist.ipynb\n", " keras-sklearn_cnn1d_dna_transcription.ipynb\n", " keras-sklearn_cnn1d_dna_transcription_powerx.ipynb\n", " keras-sklearn_cnn1d_dna_transcription_quantx.ipynb\n", " old_deepchem_ch6_improved.ipynb\n", " old_deepchem-ch6.ipynb\n", " old_deepchem-chp3.ipynb\n", " old_keras-ch6-improved.ipynb\n", " old_keras-ch6.ipynb\n", " old_Untitled0.ipynb\n", " Untitled0.ipynb\n" ], "name": "stdout" } ] }, { "cell_type": "code", "metadata": { "id": "DbaqFwkt9n73", "colab_type": "code", "colab": { "base_uri": "https://localhost:8080/", "height": 2989 }, "outputId": "16eec283-d7ac-4aee-8167-89efb637755a" }, "source": [ "import numpy as np\n", "import os\n", "\n", "from hyperopt import Trials, STATUS_OK, tpe\n", "from keras.datasets import mnist\n", "from keras.layers.core import Dense, Dropout, Activation\n", "from keras.models import Sequential\n", "from keras.utils import np_utils\n", "\n", "from hyperas import optim\n", "from hyperas.distributions import choice, uniform\n", "\n", "\n", "def data():\n", " \"\"\"\n", " Data providing function:\n", "\n", " This function is separated from create_model() so that hyperopt\n", " won't reload data for each evaluation run.\n", " \"\"\"\n", " (x_train, y_train), (x_test, y_test) = mnist.load_data()\n", " x_train = x_train.reshape(60000, 784)\n", " x_test = x_test.reshape(10000, 784)\n", " x_train = x_train.astype('float32')\n", " x_test = x_test.astype('float32')\n", " x_train /= 255\n", " x_test /= 255\n", " nb_classes = 10\n", " y_train = np_utils.to_categorical(y_train, nb_classes)\n", " y_test = np_utils.to_categorical(y_test, nb_classes)\n", " return x_train, y_train, x_test, y_test\n", "\n", "\n", "def create_model(x_train, y_train, x_test, y_test):\n", " \"\"\"\n", " Model providing function:\n", "\n", " Create Keras model with double curly brackets dropped-in as needed.\n", " Return value has to be a valid python dictionary with two customary keys:\n", " - loss: Specify a numeric evaluation metric to be minimized\n", " - status: Just use STATUS_OK and see hyperopt documentation if not feasible\n", " The last one is optional, though recommended, namely:\n", " - model: specify the model just created so that we can later use it again.\n", " \"\"\"\n", " model = Sequential()\n", " model.add(Dense(512, input_shape=(784,)))\n", " model.add(Activation('relu'))\n", " model.add(Dropout({{uniform(0, 1)}}))\n", " model.add(Dense({{choice([256, 512, 1024])}}))\n", " model.add(Activation({{choice(['relu', 'sigmoid'])}}))\n", " model.add(Dropout({{uniform(0, 1)}}))\n", "\n", " # If we choose 'four', add an additional fourth layer\n", " if {{choice(['three', 'four'])}} == 'four':\n", " model.add(Dense(100))\n", "\n", " # We can also choose between complete sets of layers\n", "\n", " model.add({{choice([Dropout(0.5), Activation('linear')])}})\n", " model.add(Activation('relu'))\n", "\n", " model.add(Dense(10))\n", " model.add(Activation('softmax'))\n", "\n", " model.compile(loss='categorical_crossentropy', metrics=['accuracy'],\n", " optimizer={{choice(['rmsprop', 'adam', 'sgd'])}})\n", "\n", " result = model.fit(x_train, y_train,\n", " batch_size={{choice([64, 128])}},\n", " epochs=2,\n", " verbose=2,\n", " validation_split=0.1)\n", " #get the highest validation accuracy of the training epochs\n", " validation_acc = np.amax(result.history['val_acc']) \n", " print('Best validation acc of epoch:', validation_acc)\n", " return {'loss': -validation_acc, 'status': STATUS_OK, 'model': model}\n", "\n", "if __name__ == '__main__':\n", " best_run, best_model = optim.minimize(model=create_model,\n", " data=data,\n", " algo=tpe.suggest,\n", " max_evals=5,\n", " trials=Trials(),\n", " notebook_name=os.path.join('..','gdrive','My Drive','Colab Notebooks','example_hyperas'))\n", " X_train, Y_train, X_test, Y_test = data()\n", " print(\"Evalutation of best performing model:\")\n", " print(best_model.evaluate(X_test, Y_test))\n", " print(\"Best performing model chosen hyper-parameters:\")\n", " print(best_run)" ], "execution_count": 0, "outputs": [ { "output_type": "stream", "text": [ ">>> Imports:\n", "#coding=utf-8\n", "\n", "try:\n", " from google.colab import drive\n", "except:\n", " pass\n", "\n", "try:\n", " import numpy as np\n", "except:\n", " pass\n", "\n", "try:\n", " import os\n", "except:\n", " pass\n", "\n", "try:\n", " from hyperopt import Trials, STATUS_OK, tpe\n", "except:\n", " pass\n", "\n", "try:\n", " from keras.datasets import mnist\n", "except:\n", " pass\n", "\n", "try:\n", " from keras.layers.core import Dense, Dropout, Activation\n", "except:\n", " pass\n", "\n", "try:\n", " from keras.models import Sequential\n", "except:\n", " pass\n", "\n", "try:\n", " from keras.utils import np_utils\n", "except:\n", " pass\n", "\n", "try:\n", " from hyperas import optim\n", "except:\n", " pass\n", "\n", "try:\n", " from hyperas.distributions import choice, uniform\n", "except:\n", " pass\n", "\n", ">>> Hyperas search space:\n", "\n", "def get_space():\n", " return {\n", " 'Dropout': hp.uniform('Dropout', 0, 1),\n", " 'Dense': hp.choice('Dense', [256, 512, 1024]),\n", " 'Activation': hp.choice('Activation', ['relu', 'sigmoid']),\n", " 'Dropout_1': hp.uniform('Dropout_1', 0, 1),\n", " 'Dropout_2': hp.choice('Dropout_2', ['three', 'four']),\n", " 'add': hp.choice('add', [Dropout(0.5), Activation('linear')]),\n", " 'optimizer': hp.choice('optimizer', ['rmsprop', 'adam', 'sgd']),\n", " 'batch_size': hp.choice('batch_size', [64, 128]),\n", " }\n", "\n", ">>> Data\n", " 1: \n", " 2: \"\"\"\n", " 3: Data providing function:\n", " 4: \n", " 5: This function is separated from create_model() so that hyperopt\n", " 6: won't reload data for each evaluation run.\n", " 7: \"\"\"\n", " 8: (x_train, y_train), (x_test, y_test) = mnist.load_data()\n", " 9: x_train = x_train.reshape(60000, 784)\n", " 10: x_test = x_test.reshape(10000, 784)\n", " 11: x_train = x_train.astype('float32')\n", " 12: x_test = x_test.astype('float32')\n", " 13: x_train /= 255\n", " 14: x_test /= 255\n", " 15: nb_classes = 10\n", " 16: y_train = np_utils.to_categorical(y_train, nb_classes)\n", " 17: y_test = np_utils.to_categorical(y_test, nb_classes)\n", " 18: \n", " 19: \n", " 20: \n", ">>> Resulting replaced keras model:\n", "\n", " 1: def keras_fmin_fnct(space):\n", " 2: \n", " 3: \"\"\"\n", " 4: Model providing function:\n", " 5: \n", " 6: Create Keras model with double curly brackets dropped-in as needed.\n", " 7: Return value has to be a valid python dictionary with two customary keys:\n", " 8: - loss: Specify a numeric evaluation metric to be minimized\n", " 9: - status: Just use STATUS_OK and see hyperopt documentation if not feasible\n", " 10: The last one is optional, though recommended, namely:\n", " 11: - model: specify the model just created so that we can later use it again.\n", " 12: \"\"\"\n", " 13: model = Sequential()\n", " 14: model.add(Dense(512, input_shape=(784,)))\n", " 15: model.add(Activation('relu'))\n", " 16: model.add(Dropout(space['Dropout']))\n", " 17: model.add(Dense(space['Dense']))\n", " 18: model.add(Activation(space['Activation']))\n", " 19: model.add(Dropout(space['Dropout_1']))\n", " 20: \n", " 21: # If we choose 'four', add an additional fourth layer\n", " 22: if space['Dropout_2'] == 'four':\n", " 23: model.add(Dense(100))\n", " 24: \n", " 25: # We can also choose between complete sets of layers\n", " 26: \n", " 27: model.add(space['add'])\n", " 28: model.add(Activation('relu'))\n", " 29: \n", " 30: model.add(Dense(10))\n", " 31: model.add(Activation('softmax'))\n", " 32: \n", " 33: model.compile(loss='categorical_crossentropy', metrics=['accuracy'],\n", " 34: optimizer=space['optimizer'])\n", " 35: \n", " 36: result = model.fit(x_train, y_train,\n", " 37: batch_size=space['batch_size'],\n", " 38: epochs=2,\n", " 39: verbose=2,\n", " 40: validation_split=0.1)\n", " 41: #get the highest validation accuracy of the training epochs\n", " 42: validation_acc = np.amax(result.history['val_acc']) \n", " 43: print('Best validation acc of epoch:', validation_acc)\n", " 44: return {'loss': -validation_acc, 'status': STATUS_OK, 'model': model}\n", " 45: \n", "Downloading data from https://s3.amazonaws.com/img-datasets/mnist.npz\n", "11493376/11490434 [==============================] - 1s 0us/step\n", " 0%| | 0/5 [00:00<?, ?it/s, best loss: ?]WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow/python/framework/op_def_library.py:263: colocate_with (from tensorflow.python.framework.ops) is deprecated and will be removed in a future version.\n", "Instructions for updating:\n", "Colocations handled automatically by placer.\n", "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py:3445: calling dropout (from tensorflow.python.ops.nn_ops) with keep_prob is deprecated and will be removed in a future version.\n", "Instructions for updating:\n", "Please use `rate` instead of `keep_prob`. Rate should be set to `rate = 1 - keep_prob`.\n", "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow/python/ops/math_ops.py:3066: to_int32 (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.\n", "Instructions for updating:\n", "Use tf.cast instead.\n", "Train on 54000 samples, validate on 6000 samples\n", "Epoch 1/2\n", " - 10s - loss: 1.7256 - acc: 0.4249 - val_loss: 0.8002 - val_acc: 0.8493\n", "\n", "Epoch 2/2\n", " - 10s - loss: 0.9461 - acc: 0.6979 - val_loss: 0.4520 - val_acc: 0.8923\n", "\n", "Best validation acc of epoch:\n", "0.8923333334922791\n", "Train on 54000 samples, validate on 6000 samples\n", "Epoch 1/2\n", " - 15s - loss: 2.2159 - acc: 0.2832 - val_loss: 0.7372 - val_acc: 0.7705\n", "\n", "Epoch 2/2\n", " - 15s - loss: 1.5323 - acc: 0.4610 - val_loss: 0.5590 - val_acc: 0.8627\n", "\n", "Best validation acc of epoch:\n", "0.8626666666666667\n", "Train on 54000 samples, validate on 6000 samples\n", "Epoch 1/2\n", " - 10s - loss: 2.0434 - acc: 0.2848 - val_loss: 0.7661 - val_acc: 0.8870\n", "\n", "Epoch 2/2\n", " 40%|████ | 2/5 [01:01<01:10, 23.64s/it, best loss: -0.8923333334922791]" ], "name": "stdout" } ] }, { "cell_type": "code", "metadata": { "id": "jVM817xM-bA1", "colab_type": "code", "colab": {} }, "source": [ "" ], "execution_count": 0, "outputs": [] }, { "cell_type": "code", "metadata": { "id": "S7m2fHyv-etq", "colab_type": "code", "colab": {} }, "source": [ "" ], "execution_count": 0, "outputs": [] } ] }