Created
December 13, 2016 16:48
-
-
Save lusentis/dd14c626195067d182d3bbe510be3bb3 to your computer and use it in GitHub Desktop.
lambda (draft) to replicate a dynamodb table to S3
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
export function replicator (replicatorInternalId, sourceLogicalName, destinationPhysicalName, destinationArn) { | |
if (replicatorInternalId === false) { | |
return {}; | |
} | |
const resourcesSuffix = sourceLogicalName[0].toUpperCase() + sourceLogicalName.slice(1) + '' + replicatorInternalId; | |
const lambdaInlineCode = ` | |
'use strict'; | |
console.log('Loading function'); | |
const destinationDynamoRegion = '${destinationArn.split(':')[3]}'; | |
const AWS = require('aws-sdk'); | |
const dynamo = new AWS.DynamoDB({ region: destinationDynamoRegion }); | |
const s3 = new AWS.S3({}); | |
const unwrapFirstValue = obj => { | |
const firstKey = Object.keys(obj)[0]; | |
return obj[firstKey]; | |
}; | |
exports.handler = (event, context, callback) => { | |
console.log('Received event:', JSON.stringify(event, null, 2)); | |
return Promise.all(event.Records.map(record => { | |
if (record.eventName !== 'MODIFY' && record.eventName !== 'INSERT') { | |
return; | |
} | |
const image = record.dynamodb.NewImage; | |
const key = record.dynamodb.Keys; | |
const updates = {}; | |
const s3Key = Object.keys(key).map(k => unwrapFirstValue(key[k])).join('/'); | |
return s3.putObject({ | |
Bucket: '\u0024{BucketReplicator}', | |
Key: '${resourcesSuffix}/' + s3Key + '.json', | |
Body: JSON.stringify(image, null, 2) | |
}).promise() | |
.then(() => { | |
Object.keys(image).forEach(attributeName => { | |
if (Object.keys(key).indexOf(attributeName) !== -1) { | |
// skip the primary key | |
return; | |
} | |
updates[attributeName] = { | |
Action: 'PUT', | |
Value: image[attributeName] | |
}; | |
}); | |
const updateParams = { | |
TableName: '${destinationPhysicalName}', | |
Key: key, | |
AttributeUpdates: updates, | |
}; | |
console.log('updateParams', updateParams); | |
return dynamo.updateItem(updateParams).promise(); | |
}); | |
})) | |
.then(result => { | |
console.log('final result', result); | |
callback(null, 'Successfully processed', event.Records.length, 'records.'); | |
}) | |
.catch(err => { | |
console.log('Error running replicator:', err.message, err.stack); | |
callback(err.message); | |
}); | |
}; | |
`; | |
const roleTemplate = { | |
[`LambdaRoleForReplicator${resourcesSuffix}`]: { | |
'Type': 'AWS::IAM::Role', | |
'Properties': { | |
'AssumeRolePolicyDocument': { | |
'Version': '2012-10-17', | |
'Statement': [{ | |
'Effect': 'Allow', | |
'Principal': { | |
'Service': ['lambda.amazonaws.com'] | |
}, | |
'Action': ['sts:AssumeRole'] | |
}] | |
}, | |
'Path': '/', | |
'Policies': [{ | |
'PolicyName': 'dawson-policy', | |
'PolicyDocument': { | |
'Version': '2012-10-17', | |
'Statement': [{ | |
'Effect': 'Allow', | |
'Action': [ | |
'dynamodb:DescribeStream', | |
'dynamodb:GetRecords', | |
'dynamodb:GetShardIterator', | |
'dynamodb:ListStreams' | |
], | |
'Resource': [{ 'Fn::GetAtt': [sourceLogicalName, 'StreamArn'] }] | |
}, { | |
'Effect': 'Allow', | |
'Action': [ | |
'dynamodb:UpdateItem' | |
], | |
'Resource': [destinationArn] | |
}, | |
{ | |
'Effect': 'Allow', | |
'Action': [ | |
'logs:CreateLogGroup', | |
'logs:CreateLogStream', | |
'logs:PutLogEvents' | |
], | |
'Resource': { 'Fn::Sub': 'arn:aws:logs:${AWS::Region}:${AWS::AccountId}:*' } // eslint-disable-line | |
}, | |
{ | |
'Effect': 'Allow', | |
'Action': [ | |
's3:PutObject' | |
], | |
'Resource': { 'Fn::Sub': 'arn:aws:s3:::${BucketReplicator}/*' } // eslint-disable-line | |
}] | |
} | |
}] | |
} | |
} | |
}; | |
const lambdaTemplate = { | |
[`LambdaReplicator${resourcesSuffix}`]: { | |
'Type': 'AWS::Lambda::Function', | |
'Properties': { | |
'Handler': 'index.handler', | |
'Role': { 'Fn::GetAtt': [`LambdaRoleForReplicator${resourcesSuffix}`, 'Arn'] }, | |
'Code': { ZipFile: { 'Fn::Sub': lambdaInlineCode } }, | |
'Runtime': 'nodejs4.3', | |
'MemorySize': 256, | |
'Timeout': 10 | |
} | |
} | |
}; | |
const eventMappingTemplate = { | |
[`LambdaEventSourceMappingReplicator${resourcesSuffix}`]: { | |
'Type': 'AWS::Lambda::EventSourceMapping', | |
'Properties': { | |
'BatchSize': 1, | |
'Enabled': true, | |
'EventSourceArn': { | |
'Fn::GetAtt': [sourceLogicalName, 'StreamArn'] | |
}, | |
'FunctionName': { 'Ref': `LambdaReplicator${resourcesSuffix}` }, | |
'StartingPosition': 'TRIM_HORIZON' | |
} | |
} | |
}; | |
const bucketTemplate = { | |
BucketReplicator: { // we keep one single bucket per whole account | |
'Type': 'AWS::S3::Bucket', | |
'Properties': { | |
'VersioningConfiguration': { 'Status': 'Enabled' }, | |
'LifecycleConfiguration': { | |
'Rules': [{ | |
'Prefix': '*', | |
'Status': 'Enabled', | |
'NoncurrentVersionExpirationInDays': 90, | |
'NoncurrentVersionTransition': { | |
'StorageClass': 'STANDARD_IA', | |
'TransitionInDays': 30 | |
} | |
}] | |
} | |
} | |
} | |
}; | |
return { | |
...roleTemplate, | |
...lambdaTemplate, | |
...eventMappingTemplate, | |
...bucketTemplate | |
}; | |
} |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment