Playing with SQLAlchemy

sqlalchemy_play.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import sqlalchemy as sa\n",
    "from sqlalchemy.schema import CreateTable"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "connection_uri='mysql://user:pwd@server/db'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "engine = sa.create_engine(connection_uri)\n",
    "metadata = sa.MetaData(bind=engine)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "insp = sa.inspect(engine)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['information_schema',\n",
       " 'data',\n",
       " 'fds',\n",
       " 'fds_it',\n",
       " 'fds_model',\n",
       " 'maxwell',\n",
       " 'mysql',\n",
       " 'ods',\n",
       " 'percona',\n",
       " 'performance_schema',\n",
       " 'test']"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "insp.get_schema_names()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[u'bootstrap',\n",
       " u'columns',\n",
       " u'databases',\n",
       " u'positions',\n",
       " u'schemas',\n",
       " u't1',\n",
       " u't1a',\n",
       " u'tables',\n",
       " u'tables_copy']"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "insp.get_table_names('maxwell')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[u'bootstrap',\n",
       " u'columns',\n",
       " u'databases',\n",
       " u'positions',\n",
       " u'schemas',\n",
       " u't1',\n",
       " u't1a',\n",
       " u'tables']"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "insp.get_table_names()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'constrained_columns': [u'id'], 'name': None}"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "insp.get_pk_constraint('columns')\n",
    "#[key.name for key in insp.get_primary_keys]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{u'mysql_default charset': u'utf8', u'mysql_engine': u'InnoDB'}"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "insp.get_table_options('databases','maxwell')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[{'autoincrement': False,\n",
       "  'default': None,\n",
       "  'name': u'server_id',\n",
       "  'nullable': False,\n",
       "  'type': INTEGER(display_width=10, unsigned=True)},\n",
       " {'default': None,\n",
       "  'name': u'binlog_file',\n",
       "  'nullable': True,\n",
       "  'type': VARCHAR(length=255)},\n",
       " {'autoincrement': False,\n",
       "  'default': None,\n",
       "  'name': u'binlog_position',\n",
       "  'nullable': True,\n",
       "  'type': INTEGER(display_width=10, unsigned=True)},\n",
       " {'default': u\"'maxwell'\",\n",
       "  'name': u'client_id',\n",
       "  'nullable': False,\n",
       "  'type': VARCHAR(charset=u'latin1', length=255)},\n",
       " {'autoincrement': False,\n",
       "  'default': None,\n",
       "  'name': u'heartbeat_at',\n",
       "  'nullable': True,\n",
       "  'type': BIGINT(display_width=20)},\n",
       " {'autoincrement': False,\n",
       "  'default': None,\n",
       "  'name': u'last_heartbeat_read',\n",
       "  'nullable': True,\n",
       "  'type': BIGINT(display_width=20)}]"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "insp.get_columns('positions','maxwell')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "metadata = sa.MetaData(bind=engine)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[]"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "metadata.tables.keys()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Table: bootstrap\n",
      "    Column: id\n",
      "    Column: database_name\n",
      "    Column: table_name\n",
      "    Column: is_complete\n",
      "    Column: inserted_rows\n",
      "    Column: total_rows\n",
      "    Column: created_at\n",
      "    Column: started_at\n",
      "    Column: completed_at\n",
      "    Column: binlog_file\n",
      "    Column: binlog_position\n",
      "Table: columns\n",
      "    Column: id\n",
      "    Column: schema_id\n",
      "    Column: table_id\n",
      "    Column: name\n",
      "    Column: charset\n",
      "    Column: coltype\n",
      "    Column: is_signed\n",
      "    Column: enum_values\n",
      "    Column: column_length\n",
      "Table: databases\n",
      "    Column: id\n",
      "    Column: schema_id\n",
      "    Column: name\n",
      "    Column: charset\n",
      "Table: positions\n",
      "    Column: server_id\n",
      "    Column: binlog_file\n",
      "    Column: binlog_position\n",
      "    Column: client_id\n",
      "    Column: heartbeat_at\n",
      "    Column: last_heartbeat_read\n",
      "Table: schemas\n",
      "    Column: id\n",
      "    Column: binlog_file\n",
      "    Column: binlog_position\n",
      "    Column: base_schema_id\n",
      "    Column: deltas\n",
      "    Column: server_id\n",
      "    Column: position_sha\n",
      "    Column: charset\n",
      "    Column: version\n",
      "    Column: deleted\n",
      "Table: t1\n",
      "    Column: c1\n",
      "    Column: c2\n",
      "Table: t1a\n",
      "    Column: c1\n",
      "    Column: c2\n",
      "Table: tables\n",
      "    Column: id\n",
      "    Column: schema_id\n",
      "    Column: database_id\n",
      "    Column: name\n",
      "    Column: charset\n",
      "    Column: pk\n"
     ]
    }
   ],
   "source": [
    "for table_name in insp.get_table_names():\n",
    "    print('Table: %s' % table_name)\n",
    "    for column in insp.get_columns(table_name):\n",
    "        print(\"    Column: %s\" % column['name'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "list"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "type(insp.get_columns('positions','maxwell'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from sqlalchemy import create_engine, inspect\n",
    "class target_table(object):\n",
    "    def __init__(self, table_name, database_name, conn_str):\n",
    "        engine = create_engine(conn_str)\n",
    "        inspector = inspect(engine)\n",
    "        if table_name not in inspector.get_table_names(database_name):\n",
    "            raise ValueError\n",
    "        else:\n",
    "            self.columns = inspector.get_columns(table_name, database_name)\n",
    "            self.column_names = [c.get('name') for c in self.columns]\n",
    "    \n",
    "    def __repr__(self):\n",
    "        return ','.join(self.column_names)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "server_id,binlog_file,binlog_position,client_id,heartbeat_at,last_heartbeat_read\n"
     ]
    }
   ],
   "source": [
    "positions = target_table('positions', 'maxwell', connection_uri)\n",
    "print(positions)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "bootstrap = sa.Table('bootstrap', metadata, autoload=True, autoload_with=engine)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Table('bootstrap', MetaData(bind=Engine(mysql://maxwell:***@10.10.52.240/maxwell)), Column('id', INTEGER(display_width=10, unsigned=True), table=<bootstrap>, primary_key=True, nullable=False), Column('database_name', VARCHAR(length=255), table=<bootstrap>, nullable=False), Column('table_name', VARCHAR(length=255), table=<bootstrap>, nullable=False), Column('is_complete', TINYINT(display_width=1, unsigned=True), table=<bootstrap>, nullable=False, server_default=DefaultClause(<sqlalchemy.sql.elements.TextClause object at 0x10cadbdd0>, for_update=False)), Column('inserted_rows', BIGINT(display_width=20, unsigned=True), table=<bootstrap>, nullable=False, server_default=DefaultClause(<sqlalchemy.sql.elements.TextClause object at 0x10caeb810>, for_update=False)), Column('total_rows', BIGINT(display_width=20, unsigned=True), table=<bootstrap>, nullable=False, server_default=DefaultClause(<sqlalchemy.sql.elements.TextClause object at 0x10caeb210>, for_update=False)), Column('created_at', DATETIME(), table=<bootstrap>), Column('started_at', DATETIME(), table=<bootstrap>), Column('completed_at', DATETIME(), table=<bootstrap>), Column('binlog_file', VARCHAR(length=255), table=<bootstrap>), Column('binlog_position', INTEGER(display_width=10, unsigned=True), table=<bootstrap>, server_default=DefaultClause(<sqlalchemy.sql.elements.TextClause object at 0x10caeb510>, for_update=False)), schema=None)"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "bootstrap"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'INSERT INTO bootstrap (id, database_name, table_name, is_complete, inserted_rows, total_rows, created_at, started_at, completed_at, binlog_file, binlog_position) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)'"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "str(bootstrap.insert())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'UPDATE bootstrap SET id=%s, database_name=%s, table_name=%s, is_complete=%s, inserted_rows=%s, total_rows=%s, created_at=%s, started_at=%s, completed_at=%s, binlog_file=%s, binlog_position=%s'"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "str(bootstrap.update())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'UPDATE bootstrap SET id=%s, database_name=%s, table_name=%s, is_complete=%s, inserted_rows=%s, total_rows=%s, created_at=%s, started_at=%s, completed_at=%s, binlog_file=%s, binlog_position=%s'"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "str(sa.update(bootstrap))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'UPDATE bootstrap SET total_rows=%s WHERE id=5'"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "str(sa.update(bootstrap).where('id=5').values(total_rows='100'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'INSERT INTO bootstrap (id, database_name, table_name, is_complete, inserted_rows, total_rows, created_at, started_at, completed_at, binlog_file, binlog_position) SELECT %s AS anon_1, %s AS anon_2, %s AS anon_3 \\nWHERE 1=0'"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "str(\n",
    "    bootstrap.insert().from_select(\n",
    "        bootstrap.columns,\n",
    "        sa.select([literal(1), literal(2), literal(3)]).where('1=0')\n",
    "        )\n",
    "    )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sqlalchemy import literal"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'bootstrap' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m\u001b[0m",
      "\u001b[0;31mNameError\u001b[0mTraceback (most recent call last)",
      "\u001b[0;32m<ipython-input-12-dafde88d5b72>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0;32mprint\u001b[0m \u001b[0msqla\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mschema\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mCreateTable\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbootstrap\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m: name 'bootstrap' is not defined"
     ]
    }
   ],
   "source": [
    "print sa.schema.CreateTable(bootstrap)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sqlalchemy.dialects.mysql import TINYINT"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "CREATE TABLE tables (\n",
      "\tid INTEGER NOT NULL, \n",
      "\tschema_id INTEGER, \n",
      "\tdatabase_id INTEGER, \n",
      "\tname VARCHAR(255), \n",
      "\tcharset VARCHAR(255), \n",
      "\tpk VARCHAR(1024), \n",
      "\tPRIMARY KEY (id)\n",
      ")\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "tables = sa.Table('tables', metadata, autoload=True, autoload_with=engine)\n",
    "print sa.schema.CreateTable(tables)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "tables_copy = sa.Table('tables_copy', metadata)\n",
    "for column in tables.columns:\n",
    "    tables_copy.append_column(column.copy())\n",
    "tables_copy.create()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "CREATE TABLE \"user\" (\n",
      "\tid INTEGER NOT NULL, \n",
      "\tname VARCHAR, \n",
      "\tPRIMARY KEY (id)\n",
      ") DISTSTYLE KEY DISTKEY (id) INTERLEAVED SORTKEY (id, name)\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "import sqlalchemy as sa\n",
    "from sqlalchemy.schema import CreateTable\n",
    "engine = sa.create_engine('redshift+psycopg2://[email protected]:5439/database')\n",
    "metadata = sa.MetaData()\n",
    "user = sa.Table(\n",
    "    'user',\n",
    "    metadata,\n",
    "    sa.Column('id', sa.Integer, primary_key=True),\n",
    "    sa.Column('name', sa.String),\n",
    "    redshift_diststyle='KEY',\n",
    "    redshift_distkey='id',\n",
    "    redshift_interleaved_sortkey=['id', 'name'],\n",
    ")\n",
    "print(CreateTable(user).compile(engine))\n",
    "# print(sa.schema.CreateTable(user))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "CREATE TABLE \"user\" (\n",
      "\tid SERIAL NOT NULL, \n",
      "\tname VARCHAR, \n",
      "\tdt TIMESTAMP WITHOUT TIME ZONE, \n",
      "\tPRIMARY KEY (id)\n",
      ")\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "engine = sa.create_engine('postgresql://username@host:5432/database')\n",
    "metadata = sa.MetaData()\n",
    "user = sa.Table(\n",
    "    'user',\n",
    "    metadata,\n",
    "    sa.Column('id', sa.Integer, primary_key=True, ),\n",
    "    sa.Column('name', sa.String),\n",
    "    sa.Column('dt', sa.DateTime),\n",
    ")\n",
    "print(CreateTable(user).compile(engine))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "CREATE TABLE user (\n",
      "\tid INTEGER NOT NULL AUTO_INCREMENT, \n",
      "\tname VARCHAR(50), \n",
      "\tdt DATETIME, \n",
      "\tPRIMARY KEY (id)\n",
      ")\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "engine = sa.create_engine('mysql://maxwell:[email protected]/maxwell')\n",
    "metadata = sa.MetaData()\n",
    "user = sa.Table(\n",
    "    'user',\n",
    "    metadata,\n",
    "    sa.Column('id', sa.Integer, primary_key=True, ),\n",
    "    sa.Column('name', sa.String(length=50)),\n",
    "    sa.Column('dt', sa.DateTime),\n",
    ")\n",
    "print(CreateTable(user).compile(engine))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "CREATE TABLE \"user\" (\n",
      "\tid INTEGER NOT NULL, \n",
      "\tname VARCHAR(50), \n",
      "\tdt DATETIME, \n",
      "\ttest INTEGER, \n",
      "\tPRIMARY KEY (id)\n",
      ")\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "engine = sa.create_engine('mssql+pyodbc://user:pwd@localhost/dbname')\n",
    "metadata = sa.MetaData()\n",
    "user = sa.Table(\n",
    "    'user',\n",
    "    metadata,\n",
    "    sa.Column('id', sa.Integer, primary_key=True, autoincrement=False),\n",
    "    sa.Column('name', sa.String(length=50)),\n",
    "    sa.Column('dt', sa.DateTime),\n",
    ")\n",
    "test = sa.Column('test', sa.Integer)\n",
    "user.append_column(test)\n",
    "# print(CreateTable(user).compile(engine))\n",
    "print(sa.schema.CreateTable(user))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}