Firehose to Athena

firehose-to-athena.js

'use strict';

const aws = require('aws-sdk');
const s3 = new aws.S3({ apiVersion: '2006-03-01' });

exports.handler = (event, context, callback) => {

  const key = decodeURIComponent(event.Records[0].s3.object.key);
  const meta = key.split('/');

  console.log(meta);

  const part = {
    year: meta[1],
    month: meta[2],
    day: meta[3],
    hour: meta[4],
    name: meta[5],
  };

  const params = {
    Bucket: 'demo-kinesis-justademo',
    CopySource: `firehose/${key}`,
    Key: `athena/dt=${part.year}-${part.month}-${part.day}-${part.hour}/${part.name}`
  }

  s3.copyObject(params, function(err, data) {
    if(err) {
      console.log(err);
      callback(err.message);
      return;
    }

    console.log('copied');
    callback(null, 'copied')
  });
};