Template files also available as gists; see article on www.jonathanwalls.com.

Generate SQL, Java properties and XML files using Jinja2 templates.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pyodbc\n",
    "import zipfile\n",
    "import uuid\n",
    "import jinja2"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Configuration"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Database and Filesystem"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "DB_SERVER = 'localhost'\n",
    "DB_NAME   = 'ccdb'\n",
    "DB_USER   = 'admin_name'\n",
    "DB_PASS   = 'admin_pass'\n",
    "DB_CONN_STRING = (\n",
    "                'DRIVER={SQL Server};SERVER=' + DB_SERVER +\n",
    "                 ';DATABASE=' + DB_NAME +\n",
    "                 ';UID=' + DB_USER +\n",
    "                 ';PWD=' + DB_PASS\n",
    "                )\n",
    "\n",
    "IN_DIR  = 'in/'\n",
    "OUT_DIR = 'out/'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Query Database\n",
    "- dimensions\n",
    "- measures\n",
    "- account levels\n",
    "- schemas"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "conn = pyodbc.connect(DB_CONN_STRING)\n",
    "cursor = conn.cursor()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "query = cursor.execute('''\n",
    "SELECT DISTINCT field_name FROM data_dim_type\n",
    "WHERE data_type_id IN (SELECT DISTINCT data_type_id FROM data_record)\n",
    "''')\n",
    "dimensions = [result[0] for result in query.fetchall()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "query = cursor.execute('''\n",
    "SELECT DISTINCT field_name FROM data_measure_type\n",
    "WHERE data_type_id IN (SELECT DISTINCT data_type_id FROM data_record)\n",
    "''')\n",
    "measures = [result[0] for result in query.fetchall()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "query = cursor.execute('''\n",
    "SELECT description FROM account_field\n",
    "ORDER BY id\n",
    "''')\n",
    "account_structure = [result[0] for result in query.fetchall()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "query = cursor.execute('''\n",
    "SELECT DISTINCT data_type_id FROM data_record\n",
    "''')\n",
    "schemas = [result[0] for result in query.fetchall()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "schema_list = ''.join(\"'\" + str(schema) + \"',\" for schema in schemas)\n",
    "schema_list = schema_list[:-1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "account_levels = []\n",
    "index = 1\n",
    "for level in account_structure:\n",
    "    account_levels.append((index, level))\n",
    "    index += 1"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Generate SQL to create v_usage_for_cc16_upload"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "env = jinja2.Environment(loader=jinja2.FileSystemLoader('in'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "USE [ccdb]\n",
      "GO\n",
      "\n",
      "CREATE VIEW [dbo].[v_usage_for_cc16_upload] AS \n",
      "SELECT \n",
      "     dr.id AS UsageRecordID\n",
      "    ,account_id AS AccountUID \n",
      "    ,dbo.cc_seconds_to_date(start_datetime) AS StartDate\n",
      "    ,dbo.cc_seconds_to_date(end_datetime) AS EndDate\n",
      "    \n",
      "    ,MAX(CASE WHEN ddt.field_name = 'backup_level' THEN dd.dimension END) AS 'backup_level'\n",
      "    ,MAX(CASE WHEN ddt.field_name = 'Business_Unit' THEN dd.dim\n"
     ]
    }
   ],
   "source": [
    "kwargs = {\n",
    "    'dimensions' : dimensions,\n",
    "    'measures'   : measures,\n",
    "    'schema_list': schema_list\n",
    "}\n",
    "print(env.get_template('template.create_usage_view.sql').render(**kwargs)[0:400])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Create Query"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "kwargs = {\n",
    "    'account_levels': account_levels,\n",
    "    'dimensions'    : dimensions,\n",
    "    'measures'      : measures\n",
    "}\n",
    "sql_query = env.get_template('template.wb_sql_query.sql').render(**kwargs)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Generate Workbook"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Properties file"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_process_uuid = str(uuid.uuid4())\n",
    "\n",
    "kwargs = {\n",
    "    'version'          : '1', \n",
    "    'data_process_uuid': data_process_uuid\n",
    "}\n",
    "wb_properties = env.get_template('template.process.properties').render(**kwargs)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Process file"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "kwargs = {\n",
    "    'version'          : '1',\n",
    "    'sql_query'        : sql_query,\n",
    "    'account_levels'   : account_levels,\n",
    "    'dimensions'       : dimensions,\n",
    "    'measures'         : measures,\n",
    "    'data_process_uuid': data_process_uuid,\n",
    "    'collection_uuid'  : str(uuid.uuid4()),\n",
    "    'flow_uuid'        : str(uuid.uuid4()),\n",
    "    'step_1_uuid'      : str(uuid.uuid4()),\n",
    "    'step_2_uuid'      : str(uuid.uuid4()),\n",
    "    'auth_key'         : ''\n",
    "}\n",
    "wb_xml = WB_XML = env.get_template('template.process.xml').render(**kwargs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('process.properties', 'w') as file:\n",
    "    file.write(wb_properties)\n",
    "    \n",
    "with open('process.xml', 'w') as file:\n",
    "    file.write(wb_xml)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "workbook = zipfile.ZipFile(OUT_DIR + 'cc16_upload.zip', 'w')\n",
    "workbook.write('process.properties')\n",
    "workbook.write('process.xml')\n",
    "workbook.close()"
   ]
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}