Firehoser Setup.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import boto3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "lambda_client = boto3.client('lambda')\n",
    "iam_client = boto3.client('iam')\n",
    "kinesis_client = boto3.client('kinesis')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "ASSUMED_ROLE_POLICY = \"\"\"{\n",
    "    \"Version\": \"2012-10-17\",\n",
    "    \"Statement\": [\n",
    "        {\n",
    "            \"Effect\": \"Allow\",\n",
    "            \"Principal\": {\n",
    "                \"Service\": \"lambda.amazonaws.com\"\n",
    "            },\n",
    "            \"Action\": \"sts:AssumeRole\"\n",
    "        }\n",
    "    ]\n",
    "}\"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "POLICY = \"\"\"{\n",
    "    \"Version\": \"2012-10-17\",\n",
    "    \"Statement\": [\n",
    "        {\n",
    "            \"Effect\": \"Allow\",\n",
    "            \"Action\": [\n",
    "                \"lambda:InvokeFunction\"\n",
    "            ],\n",
    "            \"Resource\": [\n",
    "                \"*\"\n",
    "            ]\n",
    "        },\n",
    "        {\n",
    "            \"Effect\": \"Allow\",\n",
    "            \"Action\": [\n",
    "                \"kinesis:GetRecords\",\n",
    "                \"kinesis:GetShardIterator\",\n",
    "                \"kinesis:DescribeStream\",\n",
    "                \"kinesis:ListStreams\",\n",
    "                \"kinesis:PutRecord\",\n",
    "                \"logs:CreateLogGroup\",\n",
    "                \"logs:CreateLogStream\",\n",
    "                \"logs:PutLogEvents\"\n",
    "            ],\n",
    "            \"Resource\": \"*\"\n",
    "        }\n",
    "    ]\n",
    "}\"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "response = iam_client.create_role(\n",
    "    Path='/',\n",
    "    RoleName='firehoser_backup_lambda_role',\n",
    "    AssumeRolePolicyDocument=POLICY\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "role_arn = response['Role']['Arn']\n",
    "role_name = response['Role']['RoleName']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "response = iam_client.put_role_policy(\n",
    "    RoleName=response['Role']['RoleName'],\n",
    "    PolicyName='firehoser_kinesis_policy',\n",
    "    PolicyDocument=POLICY\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "with open('firehoser.zip', 'rb') as zip_file:\n",
    "    r = zip_file.read()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import zlib\n",
    "import base64"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "response = lambda_client.create_function(\n",
    "    FunctionName='firehoser',\n",
    "    Runtime='python3.6',\n",
    "    Role=role_arn,\n",
    "    Handler='lambda.handler',\n",
    "    Code={\n",
    "        'ZipFile': r,\n",
    "    },\n",
    "    Description='Sample description',\n",
    "    Timeout=10,\n",
    "    MemorySize=512,\n",
    "    Environment={\n",
    "        'Variables': {\n",
    "            'FIREHOSE_STREAM_NAME': 'lambda'\n",
    "        }\n",
    "    }\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "stream_description = kinesis_client.describe_stream(\n",
    "    StreamName='oplog_updates'\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "response = lambda_client.create_event_source_mapping(\n",
    "    EventSourceArn=stream_description['StreamDescription']['StreamARN'],\n",
    "    FunctionName=response['FunctionName'],\n",
    "    Enabled=True,\n",
    "    BatchSize=450,\n",
    "    StartingPosition='LATEST'\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Cleanup"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "response = iam_client.delete_role_policy(\n",
    "    RoleName='firehoser_backup_lambda_role',\n",
    "    PolicyName='firehoser_kinesis_policy'\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "response = iam_client.delete_role(\n",
    "    RoleName='firehoser_backup_lambda_role'\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "response = lambda_client.delete_function(\n",
    "    FunctionName='firehoser'\n",
    ")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda root]",
   "language": "python",
   "name": "conda-root-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}