Classic ELB TCP Failure

curl-elbs.sh

#!/usr/bin/env bash

stack_name=$1
if [ -z "$stack_name" ]; then
    echo "Usage: $(basename $0) STACK_NAME" >&2
    exit 1
fi

if ! type aws > /dev/null 2>&1; then
    echo 'Is the awscli installed?' >&2
    exit 1
fi

outputs_json=$(aws cloudformation describe-stacks --stack-name $stack_name --output=json --query 'Stacks[0].Outputs')
if [[ -z $outputs_json ]]; then
    echo "Stack should be deployed via: aws cloudformation deploy --stack-name $stack_name --template-file elbtest.yml" >&2
    exit 1
fi
elb_labels=($(echo "$outputs_json" | jq -r '.[0].OutputKey') $(echo "$outputs_json" | jq -r '.[1].OutputKey'))
elb_hosts=($(echo "$outputs_json" | jq -r '.[0].OutputValue') $(echo "$outputs_json" | jq -r '.[1].OutputValue'))
elb_ipaddrs=("$(echo $(dig +short ${elb_hosts[0]}))" "$(echo $(dig +short ${elb_hosts[1]}))")

#echo ${elb_ipaddrs[0]}
#echo ${elb_ipaddrs[1]}
#exit

running=1

function curl_it() {
    elb_idx=$1
    ip_idx=$2
    elb_ip=$(echo ${elb_ipaddrs[$elb_idx]} | awk "{ print \$$(($ip_idx + 1)) }")
    http_host=${elb_hosts[$elb_idx]}
    curl --resolve $http_host:80:$elb_ip -m 1 -s --fail http://$http_host/instance-id.txt > $elb_idx.$ip_idx.out
}

function test_site() {
    count=0
    while [[ $running == 1 ]] && sleep 0.5 ; do
        rm -f {0,1}.{0,1}.out
        for i in 0 1; do for j in 0 1; do
            curl_it $i $j &
        done ; done
        wait
        #expected_str="I'm feeling very well thank you"
        if [[ $(($count % 15)) == 0 ]]; then
            printf '           | %-40s | %-40s\n' ${elb_labels[0]} ${elb_labels[1]}
            printf ' Unix Time | %-20s%-20s | %-20s%-20s\n' Az1 Az2 Az2 Az2
            echo '-----------+------------------------------------------+----------------------------------------'
        fi
        echo -n $(date +%s)
        for i in 0 1; do
            echo -n ' | '
            for j in 0 1; do
                instance_id=$(awk '{ print $2 }' < $i.$j.out)
                if [[ -n $instance_id ]]; then
                    printf '%-20s' $instance_id
                else
                    printf '%-20s' failure
                fi
            done
        done
        echo
        count=$(($count + 1))
    done
    rm -f {0,1}.{0,1}.out
}

function my_exit() {
    running=0
}

# trap ctrl-c and call ctrl_c()
trap my_exit INT

test_site

elbtest.yml

---

AWSTemplateFormatVersion: 2010-09-09
Description: Failure test

Parameters:
  Amzn2Ami:
    Description: AMI Id of Amazon Linux 2
    Type: AWS::EC2::Image::Id
    Default: ami-428aa838
  Ec2Key:
    Description: Name of EC2 Key Pair
    Type: AWS::EC2::KeyPair::KeyName
    Default: vrivellino

Resources:

  ElbSecGrp:
    Type: AWS::EC2::SecurityGroup
    Properties:
      GroupDescription: ELB for failure test
      SecurityGroupIngress:
        - IpProtocol: tcp
          FromPort: 80
          ToPort: 80
          CidrIp: 0.0.0.0/0
  InstanceSecGrp:
    Type: AWS::EC2::SecurityGroup
    Properties:
      GroupDescription: EC2 instances for failure test
      SecurityGroupIngress:
        - IpProtocol: tcp
          FromPort: 8000
          ToPort: 8000
          SourceSecurityGroupId: !GetAtt ElbSecGrp.GroupId
        - IpProtocol: tcp
          FromPort: 8080
          ToPort: 8080
          SourceSecurityGroupId: !GetAtt ElbSecGrp.GroupId
        - IpProtocol: tcp
          FromPort: 22
          ToPort: 22
          CidrIp: 45.62.176.5/32

  ElbTcp:
    Type: AWS::ElasticLoadBalancing::LoadBalancer
    Properties:
      LoadBalancerName: !Sub "${AWS::StackName}-tcp"
      AvailabilityZones:
        - !Select
          - 0
          - Fn::GetAZs: !Ref 'AWS::Region'
        - !Select
          - 1
          - Fn::GetAZs: !Ref 'AWS::Region'
      SecurityGroups:
        - !GetAtt ElbSecGrp.GroupId
      CrossZone: true
      Listeners:
        - LoadBalancerPort: 80
          Protocol: TCP
          InstancePort: 8000
          InstanceProtocol: TCP
      HealthCheck:
        Target: HTTP:8080/
        HealthyThreshold: 3
        UnhealthyThreshold: 4
        Interval: 10
        Timeout: 9
      ConnectionSettings:
        IdleTimeout: 300
      ConnectionDrainingPolicy:
        Enabled: true
        Timeout: 300
      Policies:
        - PolicyName: EnableProxyProtocol
          PolicyType: ProxyProtocolPolicyType
          Attributes:
            - Name: ProxyProtocol
              Value: true
          InstancePorts:
            - 8000

  ElbHttp:
    Type: AWS::ElasticLoadBalancing::LoadBalancer
    Properties:
      LoadBalancerName: !Sub "${AWS::StackName}-http"
      AvailabilityZones:
        - !Select
          - 0
          - Fn::GetAZs: !Ref 'AWS::Region'
        - !Select
          - 1
          - Fn::GetAZs: !Ref 'AWS::Region'
      SecurityGroups:
        - !GetAtt ElbSecGrp.GroupId
      CrossZone: true
      Listeners:
        - LoadBalancerPort: 80
          Protocol: HTTP
          InstancePort: 8080
          InstanceProtocol: HTTP
      HealthCheck:
        Target: HTTP:8080/
        HealthyThreshold: 3
        UnhealthyThreshold: 4
        Interval: 10
        Timeout: 9
      ConnectionSettings:
        IdleTimeout: 300
      ConnectionDrainingPolicy:
        Enabled: true
        Timeout: 300

  LaunchConfig:
    Type: AWS::AutoScaling::LaunchConfiguration
    Properties:
      ImageId: !Ref Amzn2Ami
      KeyName: !Ref Ec2Key
      InstanceType: t2.small
      SecurityGroups:
        - !GetAtt InstanceSecGrp.GroupId
      UserData:
        Fn::Base64: |
          #!/bin/bash
          set -ex
          amazon-linux-extras install nginx1.12
          cat > /etc/nginx/default.d/xtra-listen.conf << _EOF_
          listen 8080;
          listen 8000 proxy_protocol;
          set_real_ip_from 172.31.16.0/20;
          _EOF_
          ec2-metadata -i > /usr/share/nginx/html/instance-id.txt
          systemctl start nginx.service
          echo 'iptables -F INPUT' > /root/flush-iptables
          echo 'iptables -A INPUT -p tcp --dport 8000 -j DROP; iptables -A INPUT -p tcp --dport 8080 -j DROP' > /root/drop-http
          chmod +x /root/flush-iptables /root/drop-http

  LaunchGroup:
    Type: AWS::AutoScaling::AutoScalingGroup
    Properties:
      LaunchConfigurationName: !Ref LaunchConfig
      MinSize: 2
      MaxSize: 12
      DesiredCapacity: 2
      AvailabilityZones:
        - !Select
          - 0
          - Fn::GetAZs: !Ref 'AWS::Region'
        - !Select
          - 1
          - Fn::GetAZs: !Ref 'AWS::Region'
      MetricsCollection:
        - Granularity: 1Minute
      HealthCheckGracePeriod: 300
      Cooldown: 300
      LoadBalancerNames:
        - !Ref ElbTcp
        - !Ref ElbHttp
      HealthCheckType: EC2
      Tags:
        - Key: Name
          Value: !Ref AWS::StackName
          PropagateAtLaunch: true

Outputs:
  HostnameTcp:
    Description: Hostname of TCP ELB w/ Proxy Proto enabled
    Value: !GetAtt ElbTcp.DNSName
  HostnameHttp:
    Description: Hostname of HTTP ELB
    Value: !GetAtt ElbHttp.DNSName

Screen Shot 2018-03-01 at 12.39.12 PM.png