Cloudformation file that sets up Filebeat/Cloudtrail Elasticsearch forwarding (Cloudtrail writes to S3 bucket)

filebeat-cloudtrail-s3-elasticsearch.json

{
  "AWSTemplateFormatVersion": "2010-09-09",
  "Description": "Elastic SIEM - Filebeat ingestion of Cloudtrail logs from S3",
  "Parameters": {

    "KeyPair": {
      "Type": "AWS::EC2::KeyPair::KeyName",
      "Default": "",
      "Description": "Name of an existing EC2 KeyPair to enable SSH access"
    },
    "InstanceClass" : {
      "Type" : "String",
      "Description" : "EC2 instance class - should be large enough to accommodate task's CPU/memory reservation, if specified",
      "Default" : "xlarge",
      "AllowedValues": ["large","xlarge","2xlarge","4xlarge"]
    },

    "TaskName": {
      "Type": "String",
      "Description": "ECS cluster and task name",
      "Default": "siem-filebeat-cloudtrail",
      "MinLength": 3,
      "MaxLength": 25
    },
    "TaskCount": {
      "Type": "Number",
      "Description": "Number of ECS tasks to run",
      "Default": 2,
      "MinValue": 1,
      "MaxValue": 10
    },
    "TaskCPU": {
      "Type": "Number",
      "Description": "CPU units to allocate to ECS task. 1024 = one CPU. Leave at 0 to not restrict CPU.",
      "Default": 0,
      "AllowedValues": [0,512,1024,2048,4096,8192]
    },
    "TaskMemory": {
      "Type": "Number",
      "Description": "Memory units to allocate to ECS task, in MB. Leave at 0 to not restrict memory.",
      "Default": 0,
      "AllowedValues": [0,512,1024,2048,3072,4096,5120,6144,7168,8192,16384]
    },

    "ECSOnDemandPercentage" : {
      "Type" : "Number",
      "Description" : "Specify percentage of EC2 hosts that should run on-demand (0: run all EC2 as spot instances, 100: run all EC2 as on-demand instances).",
      "Default" : 0,
      "AllowedValues": [0,33,50,66,100]
    },

    "FilebeatTaskImage": {
      "Type": "String",
      "Description": "Filebeat ECS task Docker image path",
      "Default": "docker.elastic.co/beats/filebeat:7.12.1"
    },

    "ElasticsearchHostPort": {
      "Type": "String",
      "Description": "protocol://host:port value for Elasticsearch output",
      "Default": "http://es-host:9200"
    },
    "S3BucketName": {
      "Type": "String",
      "Description": "Name for S3 bucket to be used for CloudTrail storage. Leave blank to create one using this template (recommended).",
      "Default": "my-cloudtrail-bucket"
    },

    "LogRetentionDays": {
      "Type": "Number",
      "Description": "ECS log group retention days",
      "Default": 3,
      "AllowedValues": [1,3,5,7,14,30,60,90]
    }

  },

  "Mappings" : {
    "AWSRegionArch2AMI": {
      "us-east-1": {"64": "ami-007cd1678c6286a05", "RepoVersion": "2"},
      "us-west-2": {"64": "ami-01e1d12a048e67ed2", "RepoVersion": "2"}
    }
  },

  "Conditions" : {
    "TaskCPUDefined" : {"Fn::Not": [{"Fn::Equals": [{"Ref": "TaskCPU"}, 0]}]},
    "TaskMemoryDefined" : {"Fn::Not": [{"Fn::Equals": [{"Ref": "TaskMemory"}, 0]}]},
    "CreateS3Bucket" : {"Fn::Equals": [{"Ref": "S3BucketName"}, ""]}
  },

  "Resources": {

    "ECSSecurityGroup": {
      "Type": "AWS::EC2::SecurityGroup",
      "Properties": {
        "VpcId": {"Fn::ImportValue": "Main-vpcid"},
        "SecurityGroupIngress": [
          {"CidrIp": "10.0.0.0/8", "IpProtocol": "-1", "FromPort": "-1", "ToPort": "-1"}
        ],
        "GroupDescription": {"Fn::Sub": "${TaskName} ECS security group"}
      }
    },

    "ECSLogGroup": {
      "Type" : "AWS::Logs::LogGroup",
      "Properties" : {
        "LogGroupName" : {"Fn::Sub": "/ecs/${ECSCluster}/${TaskName}"},
        "RetentionInDays" : {"Ref": "LogRetentionDays"}
      }
    },

    "SIEMS3Bucket": {
      "Type": "AWS::S3::Bucket",
      "Properties" : {
        "NotificationConfiguration": {
          "QueueConfigurations" : [ {
            "Event" : "s3:ObjectCreated:*",
            "Queue" : {"Fn::Sub": "${SIEMS3Queue.Arn}"}
          } ]
        }
      },
      "Condition": "CreateS3Bucket"
    },
    "SIEMS3BucketPolicy": {
      "Type": "AWS::S3::BucketPolicy",
      "Properties": {
        "Bucket": {"Ref": "SIEMS3Bucket"},
        "PolicyDocument": {
          "Version": "2012-10-17",
          "Statement": [
            {
              "Sid": "AWSCloudTrailAclCheck",
              "Effect": "Allow",
              "Principal": {
                "Service": "cloudtrail.amazonaws.com"
              },
              "Action": "s3:GetBucketAcl",
              "Resource": {"Fn::Sub": "${SIEMS3Bucket.Arn}"}
            },
            {
              "Sid": "AWSCloudTrailWrite",
              "Effect": "Allow",
              "Principal": {
                "Service": "cloudtrail.amazonaws.com"
              },
              "Action": "s3:PutObject",
              "Resource": {"Fn::Sub": "${SIEMS3Bucket.Arn}/AWSLogs/*"},
              "Condition": {
                "StringEquals": {"s3:x-amz-acl": "bucket-owner-full-control"}
              }
            },
            {
              "Sid": "AllowReadFromECSRole",
              "Effect": "Allow",
              "Principal": {
                "AWS": {"Fn::Sub": "${ECSExecutionRole.Arn}"}
              },
              "Action": [
                "s3:Get*",
                "s3:List*"
              ],
              "Resource": [
                {"Fn::Sub": "${SIEMS3Bucket.Arn}"},
                {"Fn::Sub": "${SIEMS3Bucket.Arn}/*"}
              ]
            }
          ]
        }
      },
      "Condition": "CreateS3Bucket"
    },

    "SIEMS3Queue": {
      "Type": "AWS::SQS::Queue"
    },
    "SIEMS3QueuePolicy": {
      "Type" : "AWS::SQS::QueuePolicy",
      "Properties" : {
        "PolicyDocument" : {
          "Statement": [
            {
              "Sid": "S3SNSPolicy",
              "Effect": "Allow",
              "Principal": {
                "Service": "s3.amazonaws.com"
              },
              "Resource": "*",
              "Action": "sqs:SendMessage"
            }
          ]
        },
        "Queues" : [{"Ref": "SIEMS3Queue"}]
      }
    },

    "ECSRole": {
      "Type": "AWS::IAM::Role",
      "Properties": {
        "AssumeRolePolicyDocument": {
          "Statement": [
            {
              "Action": "sts:AssumeRole",
              "Effect": "Allow",
              "Principal": {
                "Service": "ec2.amazonaws.com"
              }
            }
          ],
          "Version": "2012-10-17"
        },
        "ManagedPolicyArns": [
          "arn:aws:iam::aws:policy/service-role/AmazonEC2ContainerServiceforEC2Role",
          "arn:aws:iam::aws:policy/AmazonSSMManagedInstanceCore"
        ],
        "Policies": [{
          "PolicyDocument": {
            "Version": "2012-10-17",
            "Statement": [
              {
                "Effect": "Allow",
                "Action": [
                  "ec2:Describe*",
                  "ec2:List*",
                  "elasticfilesystem:Describe*"
                ],
                "Resource": "*"
              },
              {
                "Effect": "Allow",
                "Action": [
                  "s3:List*",
                  "s3:Get*"
                ],
                "Resource": [
                  "arn:aws:s3:::edmunds-shared-services-bootstrap",
                  "arn:aws:s3:::edmunds-shared-services-bootstrap/*",
                ]
              },
              {
                "Effect": "Allow",
                "Action": [
                  "s3:ListAllMyBuckets",
                  "s3:HeadBucket"
                ],
                "Resource": "*"
              }
            ]
          },
          "PolicyName": {"Fn::Join": ["-", [{"Ref": "AWS::StackName"}, "EC2-Policy"]]}
        }]
      }
    },
    "ECSInstanceProfile": {
      "Type": "AWS::IAM::InstanceProfile",
      "Properties": {
        "Roles": [{"Ref": "ECSRole"}]
      }
    },

    "EcsInstanceLT" : {
      "Type" : "AWS::EC2::LaunchTemplate",
      "Properties" : {
        "LaunchTemplateData": {
          "ImageId": {"Fn::FindInMap": ["AWSRegionArch2AMI", {"Ref" : "AWS::Region"}, "64"]},
          "BlockDeviceMappings" : [
          ],
          "IamInstanceProfile": {"Name": { "Ref": "ECSInstanceProfile" }},
          "KeyName" : { "Ref" : "KeyPair" },
          "SecurityGroupIds" : [ {"Fn::ImportValue": "Main-basesecgroupid"}, { "Ref" : "ECSSecurityGroup" } ],
          "UserData" :
          { "Fn::Base64": { "Fn::Join" : ["", [
              "#cloud-config\n",
              "repo_releasever: ", {"Fn::FindInMap": ["AWSRegionArch2AMI", {"Ref" : "AWS::Region"}, "RepoVersion"]}, "\n",
              "runcmd:\n",
              " - set -x", "\n",
              " - export AWS_DEFAULT_REGION=", {"Ref": "AWS::Region"}, "\n",
              " - export INSTANCEID=$(curl http://169.254.169.254/latest/meta-data/instance-id)\n",
              " - export ECS_CLUSTER_NAME=", {"Ref": "ECSCluster"}, "\n",
              " - yum install -y https://s3.amazonaws.com/ec2-downloads-windows/SSMAgent/latest/linux_amd64/amazon-ssm-agent.rpm", "\n",
              " - echo -e \"ECS_CLUSTER=", {"Ref": "ECSCluster"}, "\" >/etc/ecs/ecs.config", "\n",
              " - export ECS_ENABLE_SPOT_INSTANCE_DRAINING=true", "\n"
            ] ] }
          }
        }
      }
    },
    "ECSAutoScaleGroup" : {
      "Type" : "AWS::AutoScaling::AutoScalingGroup",
      "Properties" : {
        "AvailabilityZones" : {"Fn::Split": [",", {"Fn::ImportValue": "Main-privatesubnetAZs"}]},
        "VPCZoneIdentifier" : {"Fn::Split": [",", {"Fn::ImportValue": "Main-privatesubnets"}]},
        "MixedInstancesPolicy" : {
          "InstancesDistribution" : {
            "OnDemandAllocationStrategy" : "prioritized",
            "OnDemandBaseCapacity" : 0,
            "OnDemandPercentageAboveBaseCapacity" : {"Ref": "ECSOnDemandPercentage"},
            "SpotAllocationStrategy" : "capacity-optimized"
          },
          "LaunchTemplate" : {
            "LaunchTemplateSpecification" : {
              "LaunchTemplateId": {"Ref": "EcsInstanceLT"},
              "Version": {"Fn::GetAtt": ["EcsInstanceLT", "LatestVersionNumber"]}
            },
            "Overrides" : [
              {"InstanceType" : {"Fn::Sub": "c4.${InstanceClass}"}},
              {"InstanceType" : {"Fn::Sub": "c5.${InstanceClass}"}},
              {"InstanceType" : {"Fn::Sub": "m4.${InstanceClass}"}},
              {"InstanceType" : {"Fn::Sub": "m5.${InstanceClass}"}}
            ]
          }
        },
        "MinSize" : {"Ref": "TaskCount"},
        "MaxSize" : {"Ref": "TaskCount"},
        "DesiredCapacity" : {"Ref": "TaskCount"},
        "Tags" : [
          { "Key" : "Name", "Value" : {"Fn::Join" : ["", [{ "Ref" : "TaskName" }, " - ECS Cluster"]]}, "PropagateAtLaunch" : "true" },
          { "Key" : "Cluster", "Value" : { "Ref" : "TaskName" }, "PropagateAtLaunch" : "true" },
          { "Key" : "Environment", "Value" : {"Fn::ImportValue": "Main-account-name"}, "PropagateAtLaunch" : "true" },
          { "Key" : "Application ID", "Value" : {"Ref": "TaskName"}, "PropagateAtLaunch" : "true"},
          { "Key" : "Application Role", "Value" : "ECS Task", "PropagateAtLaunch" : "true"}
        ]
      }
    },

    "ECSExecutionRole": {
      "Type": "AWS::IAM::Role",
      "Properties": {
        "AssumeRolePolicyDocument": {
           "Version" : "2012-10-17",
           "Statement": [ {
              "Effect": "Allow",
              "Principal": {
                 "Service": [ "ecs-tasks.amazonaws.com" ],
                 "AWS": [ {"Fn::Sub": "${ECSRole.Arn}"} ]
              },
              "Action": [ "sts:AssumeRole" ]
           } ]
        },
        "ManagedPolicyArns": [ "arn:aws:iam::aws:policy/service-role/AmazonECSTaskExecutionRolePolicy" ],
        "Policies": [{
          "PolicyDocument": {
              "Version": "2012-10-17",
              "Statement": [{
                  "Effect": "Allow",
                  "Action": ["sqs:ReceiveMessage", "sqs:DeleteMessage*", "sqs:Get*", "sqs:List*"],
                  "Resource": {"Fn::Sub": "${SIEMS3Queue.Arn}"}
                },
                {
                  "Effect": "Allow",
                  "Action": ["s3:Get*", "s3:List*"],
                  "Resource": {"Fn::If": [
                    "CreateS3Bucket",
                    [{"Fn::Sub": "${SIEMS3Bucket.Arn}"}, {"Fn::Sub": "${SIEMS3Bucket.Arn}/*"}],
                    [{"Fn::Sub": "arn:aws:s3:::${S3BucketName}"}, {"Fn::Sub": "arn:aws:s3:::${S3BucketName}/*"}]
                  ]}
                }
              ]
          },
          "PolicyName": {"Fn::Join": ["-", [{"Ref": "AWS::StackName"}, "ECS-Policy"]]}
        }],
        "Path": "/"
      }
    },

    "ECSCluster": {
      "Type" : "AWS::ECS::Cluster"
    },

    "ECSTaskDefinition": {
      "Type" : "AWS::ECS::TaskDefinition",
      "Properties" : {
        "Cpu" : {"Fn::If": ["TaskCPUDefined", {"Ref": "TaskCPU"}, {"Ref": "AWS::NoValue"}]},
        "Family" : {"Ref": "TaskName"},
        "Memory" : {"Fn::If": ["TaskMemoryDefined", {"Ref": "TaskMemory"}, {"Ref": "AWS::NoValue"}]},
        "NetworkMode": "host",
        "Volumes": [
        ],
        "RequiresCompatibilities": ["EC2"],
        "ContainerDefinitions" : [{ 
          "Name": {"Ref": "TaskName"},
          "Cpu": {"Fn::If": ["TaskCPUDefined", {"Ref": "TaskCPU"}, {"Ref": "AWS::NoValue"}]},
          "Memory": {"Fn::If": ["TaskMemoryDefined", {"Ref": "TaskMemory"}, {"Ref": "AWS::NoValue"}]},
          "MemoryReservation": {"Fn::If": ["TaskMemoryDefined", {"Ref": "TaskMemory"}, 512]},
          "Image": {"Ref": "FilebeatTaskImage"},
          "EntryPoint": ["/usr/share/filebeat/filebeat"],
          "Command": [
            "run",
            "-environment", "container",
            "-v",
            "--E", "output.elasticsearch.enabled=true",
            "--E", {"Fn::Sub": "output.elasticsearch.hosts=['${ElasticsearchHostPort}']"},
            "--E", "output.elasticsearch.username=filebeat",
            "--E", "output.elasticsearch.password=filebeat",
            "--E", "output.elasticsearch.bulk_max_size=500",
            "--E", "output.elasticsearch.worker=8",
            "--E", "output.elasticsearch.index=filebeat-%{[agent.version]}-cloudtrail-%{+yyyy.MM.dd}",
            "--E", "setup.template.name=filebeat",
            "--E", "setup.template.pattern=filebeat-*",
            "--E", "setup.ilm.enabled=false",
            "--E", "setup.template.settings.index.max_docvalue_fields_search=250",
            "--modules=aws",
            "--M", "aws.cloudtrail.enabled=true",
            "--M", {"Fn::Sub": "aws.cloudtrail.var.role_arn=${ECSExecutionRole.Arn}"},
            "--M", {"Fn::Sub": "aws.cloudtrail.var.queue_url=${SIEMS3Queue}"},
            "--M", "aws.cloudwatch.enabled=false",
            "--M", "aws.ec2.enabled=false",
            "--M", "aws.elb.enabled=false",
            "--M", "aws.s3access.enabled=false",
            "--M", "aws.vpcflow.enabled=false"
          ],
          "Environment": [
          ],
          "PortMappings": [
          ],
          "MountPoints": [
          ],
          "Ulimits": [{
              "SoftLimit": 65536,
              "HardLimit": 65536,
              "Name": "nofile"
          }],
          "LogConfiguration": {
            "LogDriver": "awslogs",
            "Options": {
              "awslogs-region": {"Ref": "AWS::Region"},
              "awslogs-group": {"Ref": "ECSLogGroup"},
              "awslogs-stream-prefix": {"Ref": "TaskName"}
            }
          }
        }],
        "ExecutionRoleArn": {"Fn::GetAtt": ["ECSExecutionRole", "Arn"]},
        "Tags" : [
          { "Key" : "Name", "Value" : { "Ref" : "TaskName" } },
          { "Key" : "Environment", "Value" : {"Fn::ImportValue": "Main-account-name"} },
          { "Key" : "Application ID", "Value" : {"Ref": "TaskName"} },
          { "Key" : "Application Role", "Value" : "ECS Task" }
        ]
      }
    },
    "ECSService": {
      "Type" : "AWS::ECS::Service",
      "Properties" : {
        "Cluster" : {"Fn::GetAtt": ["ECSCluster", "Arn"]},
        "DeploymentConfiguration" : {
          "MaximumPercent" : 200,
          "MinimumHealthyPercent" : 0
        },
        "DesiredCount" : {"Ref": "TaskCount"},
        "LaunchType" : "EC2",
        "TaskDefinition" : {"Ref": "ECSTaskDefinition"},
        "PlacementConstraints": [{
          "Type" : "distinctInstance"
        }],
        "Tags" : [
          { "Key" : "Name", "Value" : { "Ref" : "TaskName" } },
          { "Key" : "Environment", "Value" : {"Fn::ImportValue": "Main-account-name"} },
          { "Key" : "Application ID", "Value" : {"Ref": "TaskName"} },
          { "Key" : "Application Role", "Value" : "ECS Task" }
        ]
      }
    }

  },

  "Outputs" : {
    "SIEMS3QueueURL" : {
      "Description" : "S3 bucket SQS notification queue URL",
      "Value" : { "Fn::Sub" : "${SIEMS3Queue}" }
    },
    "ECSClusterURL" : {
      "Description" : "Filebeat ECS cluster URL",
      "Value" : { "Fn::Sub" : "https://${AWS::Region}.console.aws.amazon.com/ecs/home?region=${AWS::Region}#/clusters/${ECSCluster}/services" }
    },
  }

}