ML+Engine+requirements.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import time\n",
    "projectId = 'projects/PROJECT'\n",
    "bucket = 'PROJECT-ml'\n",
    "job_id = 'mlmagic__%d' % time.time()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import os\n",
    "from oauth2client.client import GoogleCredentials\n",
    "from googleapiclient import discovery\n",
    "import subprocess\n",
    "\n",
    "import tempfile\n",
    "\n",
    "credentials = GoogleCredentials.get_application_default()\n",
    "ml = discovery.build('ml', 'v1', credentials=credentials)\n",
    "\n",
    "\n",
    "tmpdir = tempfile.gettempdir() + '/' + job_id\n",
    "\n",
    "_store = \"\"\"\n",
    "import tensorflow as tf\n",
    "from sklearn import cross_validation\n",
    "import logging\n",
    "\n",
    "def run_training():\n",
    "    # Load iris dataset\n",
    "    iris = tf.contrib.learn.datasets.base.load_iris()\n",
    "    train_x, test_x, train_y, test_y = cross_validation.train_test_split(\n",
    "        iris.data, iris.target, test_size=0.2\n",
    "    )\n",
    "\n",
    "    \n",
    "    feature_columns = [tf.contrib.layers.real_valued_column(\"\", dimension=4)]\n",
    "    classifier = tf.contrib.learn.DNNClassifier(feature_columns=feature_columns,\n",
    "                                                hidden_units=[10, 20, 10],\n",
    "                                                n_classes=3,\n",
    "                                                model_dir=\"./model\")\n",
    "\n",
    "    classifier.fit(x=train_x,\n",
    "                   y=train_y,\n",
    "                   steps=2000,\n",
    "                   batch_size=50)\n",
    "if __name__ == '__main__':\n",
    "    run_training()\n",
    "\"\"\"\n",
    "\n",
    "if not os.path.exists('%s/trainer' % tmpdir):\n",
    "  os.makedirs('%s/trainer' % tmpdir)\n",
    "with open(tmpdir + '/trainer/__init__.py', \"w\") as f:\n",
    "  f.write(\"\")\n",
    "with open(tmpdir + '/trainer/task.py', \"w\") as f:\n",
    "  for r in _store:\n",
    "    f.write(r)\n",
    "\n",
    "with open(tmpdir + '/setup.py', \"w\") as f:\n",
    "  f.write(\"from setuptools import setup\\n\"\n",
    "          \"if __name__ == '__main__':\\n\"\n",
    "          \"    setup(name='trainer',\"\n",
    "          \"          packages=['trainer'],\"\n",
    "          \"          install_requires=['keras'])\\n\")\n",
    "\n",
    "gzfilepath = tmpdir + '/dist/trainer-0.0.0.tar.gz'\n",
    "gsfilepath = 'gs://%s/%s.tar.gz' % (bucket, job_id)\n",
    "# gspathlibs = ['gs://%s/mlmagic_lib/%s'%(bucket, f) for f in os.listdir('/var/tmp/tmppip')]\n",
    "\n",
    "subprocess.call(['python', 'setup.py', 'sdist'], cwd=tmpdir)\n",
    "subprocess.call(['gsutil', 'cp', gzfilepath, gsfilepath])\n",
    "job_req = ml.projects().jobs().create(\n",
    "            parent=projectId,\n",
    "            body={'jobId': job_id,\n",
    "                  'trainingInput': {'scaleTier': 'BASIC',\n",
    "                                    # 'masterType': 'standard_gpu',\n",
    "                                    # 'workerType': 'standard_gpu',\n",
    "                                    'packageUris': [gsfilepath],\n",
    "                                    'pythonModule': 'trainer.task',\n",
    "                                    'region': 'us-central1'\n",
    "                                    }\n",
    "                  }\n",
    "          )\n",
    "response = job_req.execute()\n",
    "print(response)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}