Skip to content

Instantly share code, notes, and snippets.

@miyamoto-daisuke

miyamoto-daisuke/fluentd.template

Created October 9, 2013 09:45
Show Gist options

  • Select an option

    Learn more about clone URLs
  • Save miyamoto-daisuke/6898807 to your computer and use it in GitHub Desktop.

Select an option

Learn more about clone URLs
Save miyamoto-daisuke/6898807 to your computer and use it in GitHub Desktop.
Download ZIP
Raw
fluentd.template
{
"AWSTemplateFormatVersion": "2010-09-09",
"Description": "fluentd template",
"Parameters": {
"KeyName": {
"Description": "Name of an existing EC2 KeyPair to enable SSH access to the instances",
"Type": "String",
"MinLength": "1",
"MaxLength": "64",
"AllowedPattern": "[-_ a-zA-Z0-9]*",
"ConstraintDescription": "can contain only alphanumeric characters, spaces, dashes and underscores."
}
},
"Mappings": {
"AWSAmazonLinuxAMI": {
"us-east-1": { "name":"Virginia", "201303": "ami-3275ee5b", "201309": "ami-35792c5c" },
"us-west-2": { "name":"Oregon", "201303": "ami-ecbe2adc", "201309": "ami-d03ea1e0" },
"us-west-1": { "name":"California", "201303": "ami-66d1fc23", "201309": "ami-687b4f2d" },
"eu-west-1": { "name":"Ireland", "201303": "ami-44939930", "201309": "ami-149f7863" },
"ap-southeast-1": { "name":"Singapole", "201303": "ami-aa9ed2f8", "201309": "ami-14f2b946" },
"ap-southeast-2": { "name":"Sydney", "201303": "ami-363eaf0c", "201309": "ami-a148d59b" },
"ap-northeast-1": { "name":"Tokyo", "201303": "ami-173fbf16", "201309": "ami-3561fe34" },
"sa-east-1": { "name":"SaoPaulo", "201303": "ami-dd6bb0c0", "201309": "ami-9f6ec982" }
},
"AZ": {
"us-east-1": { "primary": "us-east-1d", "secondary": "us-east-1a" },
"us-west-2": { "primary": "us-west-2a", "secondary": "us-west-2b" },
"us-west-1": { "primary": "us-west-1a", "secondary": "us-west-1b" },
"eu-west-1": { "primary": "eu-west-1a", "secondary": "eu-west-1b" },
"ap-southeast-1": { "primary": "ap-southeast-1a", "secondary": "ap-southeast-1b" },
"ap-southeast-2": { "primary": "ap-southeast-2a", "secondary": "ap-southeast-2b" },
"ap-northeast-1": { "primary": "ap-northeast-1a", "secondary": "ap-northeast-1c" },
"sa-east-1": { "primary": "sa-east-1a", "secondary": "sa-east-1b" }
},
"AWSAPIEndpoint": {
"us-east-1": { "S3": "s3.amazonaws.com" },
"us-west-2": { "S3": "s3-us-west-2.amazonaws.com" },
"us-west-1": { "S3": "s3-us-west-1.amazonaws.com" },
"eu-west-1": { "S3": "s3-eu-west-1.amazonaws.com" },
"ap-southeast-1": { "S3": "s3-ap-southeast-1.amazonaws.com" },
"ap-southeast-2": { "S3": "s3-ap-southeast-2.amazonaws.com" },
"ap-northeast-1": { "S3": "s3-ap-northeast-1.amazonaws.com" },
"sa-east-1": { "S3": "s3-sa-east-1.amazonaws.com" }
},
"StackConfig" : {
"VPC" : { "CIDR" : "10.0.0.0/16" },
"FrontendSubnet1" : { "CIDR" : "10.0.0.0/24" },
"FrontendSubnet2" : { "CIDR" : "10.0.1.0/24" },
"ApplicationSubnet1": { "CIDR" : "10.0.2.0/24" },
"ApplicationSubnet2": { "CIDR" : "10.0.3.0/24" },
"DatabaseSubnet1" : { "CIDR" : "10.0.4.0/24" },
"DatabaseSubnet2" : { "CIDR" : "10.0.5.0/24" }
}
},
"Resources": {
"PowerUserRole" : {
"Type" : "AWS::IAM::Role",
"Properties" : {
"AssumeRolePolicyDocument" : {
"Statement": [ {
"Effect": "Allow",
"Principal": {
"Service": [ "ec2.amazonaws.com" ]
},
"Action": [ "sts:AssumeRole" ]
} ]
},
"Path" : "/",
"Policies" :[ {
"PolicyName" : "PowerUserPolicy",
"PolicyDocument" : {
"Statement": [ {
"Sid": "PowerUserStmt",
"Effect": "Allow",
"NotAction": "iam:*",
"Resource": "*"
} ]
}
}]
}
},
"PowerUserProfile" : {
"Type" : "AWS::IAM::InstanceProfile",
"Properties" : {
"Path": "/",
"Roles" : [ { "Ref" : "PowerUserRole" } ]
}
},
"VPC" : {
"Type" : "AWS::EC2::VPC",
"Properties" : {
"CidrBlock" : { "Fn::FindInMap" : [ "StackConfig", "VPC", "CIDR" ]},
"InstanceTenancy" : "default",
"Tags" : [
{"Key" : "Application", "Value" : { "Ref" : "AWS::StackId" } },
{"Key" : "Network", "Value" : "Public" }
]
}
},
"InternetGateway" : {
"Type" : "AWS::EC2::InternetGateway",
"Properties" : {
"Tags" : [
{"Key" : "Application", "Value" : { "Ref" : "AWS::StackId" } },
{"Key" : "Network", "Value" : "Public" }
]
}
},
"AttachGateway" : {
"Type" : "AWS::EC2::VPCGatewayAttachment",
"Properties" : {
"VpcId" : {"Ref" : "VPC"},
"InternetGatewayId" : {"Ref" : "InternetGateway"}
}
},
"PublicRouteTable" : {
"Type" : "AWS::EC2::RouteTable",
"DependsOn" : "AttachGateway",
"Properties" : {
"VpcId" : { "Ref" : "VPC" },
"Tags" : [
{"Key" : "Application", "Value" : { "Ref" : "AWS::StackId"} },
{"Key" : "Network", "Value" : "Public" }
]
}
},
"PublicRoute" : {
"Type" : "AWS::EC2::Route",
"Properties" : {
"RouteTableId" : { "Ref" : "PublicRouteTable" },
"DestinationCidrBlock" : "0.0.0.0/0",
"GatewayId" : { "Ref" : "InternetGateway" }
}
},
"FrontendSubnet1": {
"Type": "AWS::EC2::Subnet",
"Properties": {
"VpcId": { "Ref": "VPC" },
"AvailabilityZone": { "Fn::FindInMap": [ "AZ", { "Ref": "AWS::Region" }, "primary" ]},
"CidrBlock": { "Fn::FindInMap" : [ "StackConfig", "FrontendSubnet1", "CIDR" ]}
}
},
"FrontendSubnet2": {
"Type": "AWS::EC2::Subnet",
"Properties": {
"VpcId": { "Ref": "VPC" },
"AvailabilityZone": { "Fn::FindInMap": [ "AZ", { "Ref": "AWS::Region" }, "secondary" ]},
"CidrBlock": { "Fn::FindInMap" : [ "StackConfig", "FrontendSubnet2", "CIDR" ]}
}
},
"FrontendSubnet1RouteTableAssociation" : {
"Type" : "AWS::EC2::SubnetRouteTableAssociation",
"Properties" : {
"SubnetId" : { "Ref" : "FrontendSubnet1" },
"RouteTableId" : { "Ref" : "PublicRouteTable" }
}
},
"FrontendSubnet2RouteTableAssociation" : {
"Type" : "AWS::EC2::SubnetRouteTableAssociation",
"Properties" : {
"SubnetId" : { "Ref" : "FrontendSubnet2" },
"RouteTableId" : { "Ref" : "PublicRouteTable" }
}
},
"VPCDefaultSecurityGroup" : {
"Type" : "AWS::EC2::SecurityGroup",
"Properties" : {
"VpcId" : { "Ref" : "VPC" },
"GroupDescription" : "Allow access from bastion via port 22",
"SecurityGroupIngress" : [
{ "IpProtocol" : "tcp", "FromPort" : "0", "ToPort" : "65535", "CidrIp" : { "Fn::FindInMap" : [ "StackConfig", "VPC", "CIDR" ]} },
{ "IpProtocol" : "udp", "FromPort" : "0", "ToPort" : "65535", "CidrIp" : { "Fn::FindInMap" : [ "StackConfig", "VPC", "CIDR" ]} },
{ "IpProtocol" : "icmp", "FromPort" : "-1", "ToPort" : "-1", "CidrIp" : { "Fn::FindInMap" : [ "StackConfig", "VPC", "CIDR" ]} }
]
}
},
"SSHSecurityGroup" : {
"Type" : "AWS::EC2::SecurityGroup",
"Properties" : {
"VpcId" : { "Ref" : "VPC" },
"GroupDescription" : "Enable SSH access via port 22",
"SecurityGroupIngress" : [
{ "IpProtocol" : "tcp", "FromPort" : "22", "ToPort" : "22", "CidrIp" : "0.0.0.0/0" }
]
}
},
"LogBucket" : {
"Type" : "AWS::S3::Bucket",
"DeletionPolicy" : "Retain"
},
"FluentdInstanceEIP": {
"Type": "AWS::EC2::EIP",
"DependsOn" : "AttachGateway",
"Properties": {
"Domain": "vpc",
"InstanceId": { "Ref": "FluentdInstance" }
}
},
"FluentdInstance": {
"Type": "AWS::EC2::Instance",
"Properties": {
"InstanceType": "t1.micro",
"KeyName": { "Ref": "KeyName" },
"SubnetId": { "Ref" : "FrontendSubnet1" },
"ImageId": { "Fn::FindInMap": [ "AWSAmazonLinuxAMI", { "Ref": "AWS::Region" }, "201309" ]},
"IamInstanceProfile": { "Ref" : "PowerUserProfile" },
"SecurityGroupIds" : [
{ "Ref" : "SSHSecurityGroup" },
{ "Ref" : "VPCDefaultSecurityGroup" }
],
"Tags": [
{ "Key": "Name", "Value": "fluentd" }
],
"UserData" : { "Fn::Base64" : { "Fn::Join" : ["", [
"#! /bin/bash -v\n",
"yum update -y\n",
"# Helper function\n",
"function error_exit\n",
"{\n",
" /opt/aws/bin/cfn-signal -e 1 -r \"$1\" '", { "Ref" : "FluentdInstanceWaitHandle" }, "'\n",
" exit 1\n",
"}\n",
"# Install packages\n",
"/opt/aws/bin/cfn-init -c default -s ", { "Ref" : "AWS::StackId" }, " -r FluentdInstance ",
" --region ", { "Ref" : "AWS::Region" }, " || error_exit 'Failed to run cfn-init'\n",
"# All is well so signal success\n",
"/opt/aws/bin/cfn-signal -e $? -r \"fluentd server setup complete\" '", { "Ref" : "FluentdInstanceWaitHandle" }, "'\n"
]]}}
},
"Metadata" : {
"AWS::CloudFormation::Init" : {
"configSets" : {
"default" : [ "config1" , "config2" ]
},
"config1" : {
"files" : {
"/etc/yum.repos.d/td.repo" : {
"content" : { "Fn::Join" : ["", [
"[treasuredata]\n",
"name=TreasureData\n",
"baseurl=http://packages.treasure-data.com/redhat/$basearch\n",
"gpgcheck=0\n"
]]},
"mode" : "000644",
"owner" : "root",
"group" : "root"
}
}
},
"config2" : {
"packages" : {
"yum" : {
"td-agent" : []
}
},
"files" : {
"/etc/td-agent/td-agent.conf" : {
"content" : { "Fn::Join" : ["", [
"<source>\n",
" type forward\n",
" port 24224\n",
"</source>\n",
"\n",
"<source>\n",
" type config_expander\n",
" <config>\n",
" type tail\n",
" format syslog\n",
" path /var/log/messages\n",
" tag ${hostname}/syslog.messages\n",
" </config>\n",
"</source>\n",
"\n",
"<match *.**>\n",
" type forest\n",
" subtype s3\n",
"\n",
" <template>\n",
" s3_bucket ",{ "Ref" : "LogBucket" },"\n",
" s3_endpoint ", { "Fn::FindInMap": [ "AWSAPIEndpoint", { "Ref": "AWS::Region" }, "S3" ]}, "\n",
"\n",
" path ${tag}/\n",
" buffer_path /var/log/td-agent/buffer/${tag}\n",
"\n",
" time_slice_format %Y/%m/%d/ec2-%Y-%m-%d-%H\n",
" flush_interval 1m\n",
" </template>\n",
"</match>\n"
]]},
"mode" : "000644",
"owner" : "root",
"group" : "root"
}
},
"commands" : {
"0-chmod" : {
"command" : "chgrp td-agent /var/log/messages && chmod g+r /var/log/messages"
},
"1-fluent-update" : {
"command" : "/usr/lib64/fluent/ruby/bin/fluent-gem update"
},
"2-fluent-plugin-s3" : {
"command" : "/usr/lib64/fluent/ruby/bin/fluent-gem install fluent-plugin-s3"
},
"3-fluent-plugin-forest" : {
"command" : "/usr/lib64/fluent/ruby/bin/fluent-gem install fluent-plugin-forest"
},
"4-fluent-plugin-config-expander" : {
"command" : "/usr/lib64/fluent/ruby/bin/fluent-gem install fluent-plugin-config-expander"
}
},
"services" : {
"sysvinit" : {
"td-agent" : { "enabled" : "true", "ensureRunning" : "true" }
}
}
}
}
}
},
"FluentdInstanceWaitHandle" : {
"Type" : "AWS::CloudFormation::WaitConditionHandle"
},
"FluentdInstanceWaitCondition" : {
"Type" : "AWS::CloudFormation::WaitCondition",
"DependsOn" : "FluentdInstance",
"Properties" : {
"Handle" : {"Ref" : "FluentdInstanceWaitHandle"},
"Timeout" : "900"
}
}
}
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment