Example Google Kubernetes Engine (GKE) app with Managed Certificate and Cloud Armor rate limiting

gke-cloud-armor.sh

#!/usr/bin/env bash

#####################################################################
# REFERENCES
# - https://cloud.google.com/armor/docs/integrating-cloud-armor#with_ingress 
# - https://cloud.google.com/armor/docs/configure-security-policies
# - https://stackoverflow.com/questions/63841501/how-to-block-multiple-countries-with-one-expression-in-google-cloud-armor
# - https://cloud.google.com/kubernetes-engine/docs/how-to/managed-certs
# - https://cloud.google.com/kubernetes-engine/docs/how-to/ingress-features#create_backendconfig
#   - Optional: cloud.google.com/neg: '{"ingress": true}' and ClusterIP (vs NodePort)
#####################################################################

# EXAMPLES:
# - https://gist.github.com/mikesparr/89167550a80146f85525595393837c9e  (GLB -> GKE w/ NEG + GCE)
# - https://gist.github.com/mikesparr/bdb7225b87930cafe60192f71b5230c2  (GLB -> GKE w/ Ingress)

export PROJECT_ID=$(gcloud config get-value project)
export PROJECT_USER=$(gcloud config get-value core/account) # set current user
export PROJECT_NUMBER=$(gcloud projects describe $PROJECT_ID --format="value(projectNumber)")
export IDNS=${PROJECT_ID}.svc.id.goog # workflow identity domain

export GCP_REGION="us-west4" # CHANGEME (OPT)
export GCP_ZONE="us-west4-a" # CHANGEME (OPT)

export DOMAIN="msparr.com" # CHANGEME (OPT)
export TEST_NS="armor" # CHANGEME (OPT) - also using for subdomain for TLS
export NETWORK_NAME="default"

# enable apis
gcloud services enable compute.googleapis.com \
    container.googleapis.com

# configure gcloud sdk
gcloud config set compute/region $GCP_REGION
gcloud config set compute/zone $GCP_ZONE


#####################################################
# Security Policy
#####################################################

# create security policy rule
export POLICY_NAME="web-traffic-policy-us"
gcloud compute security-policies create $POLICY_NAME \
    --description "Rate limit US traffic"

# throttle requests in US if more then 10 per minute with 403 response
gcloud beta compute security-policies rules create 1000 \
    --security-policy $POLICY_NAME \
    --expression "origin.region_code == 'US'" \
    --action rate-based-ban \
    --rate-limit-threshold-count 10 \
    --rate-limit-threshold-interval-sec 60 \
    --ban-duration-sec 300 \
    --ban-threshold-count 1000 \
    --ban-threshold-interval-sec 600 \
    --conform-action allow \
    --exceed-action deny-403 \
    --enforce-on-key ALL


#####################################################
# GKE Cluster
#####################################################
export ADDRESS_NAME="secure-web"
export CLUSTER_NAME="west4"
export CERT_NAME="secure-web-cert"
export SERVICE_NAME="kuard-svc"
export SERVICE_PORT="80"
export INGRESS_NAME="secure-web-ingress"

# create static IP
gcloud compute addresses create $ADDRESS_NAME --global

# print out IP and update DNS records
export EXTERNAL_IP=$(gcloud compute addresses describe $ADDRESS_NAME --global --format="value(address)")
echo "*** UPDATE DNS RECORD FOR: $TEST_NS.$DOMAIN WITH IP: $EXTERNAL_IP ***\n"

# create GKE cluster (demo not secure: use authorized networks and/or private cluster)
gcloud beta container --project $PROJECT_ID clusters create $CLUSTER_NAME \
    --zone $GCP_REGION \
    --release-channel "regular" \
    --num-nodes "1" \
    --enable-ip-alias \
    --tags=allow-health-check

# deploy sample app to cluster
kubectl create ns $TEST_NS
kubectl create deployment kuard \
    --image=gcr.io/kuar-demo/kuard-amd64:blue \
    -n $TEST_NS

# create backend config linked to security policy
cat <<EOF | kubectl apply -f -
apiVersion: cloud.google.com/v1
kind: BackendConfig
metadata:
  name: my-backendconfig
  namespace: $TEST_NS
spec:
  securityPolicy:
    name: $POLICY_NAME
EOF

# create managed cert
cat <<EOF | kubectl apply -f -
apiVersion: networking.gke.io/v1
kind: ManagedCertificate
metadata:
  name: $CERT_NAME
  namespace: $TEST_NS
spec:
  domains:
    - "$TEST_NS.$DOMAIN"
EOF

# expose app with service and policy
cat <<EOF | kubectl apply -f -
apiVersion: v1
kind: Service
metadata:
  name: $SERVICE_NAME
  namespace: $TEST_NS
  annotations:
    cloud.google.com/backend-config: '{"default": "my-backendconfig"}'
  labels:
    app: kuard
spec:
  type: NodePort
  selector:
    app: kuard
  ports:
  - port: $SERVICE_PORT
    targetPort: 8080
    protocol: TCP
EOF

# create HTTP(S) load balancer with Ingress + ManagedCertificate
cat <<EOF | kubectl apply -f -
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
  name: $INGRESS_NAME
  namespace: $TEST_NS
  annotations:
    kubernetes.io/ingress.global-static-ip-name: $ADDRESS_NAME
    networking.gke.io/managed-certificates: $CERT_NAME
    kubernetes.io/ingress.class: "gce"
spec:
  defaultBackend:
    service:
      name: $SERVICE_NAME
      port:
        number: $SERVICE_PORT
EOF

# ** WAIT 10-20 MINUTES FOR NETWORK / CERT, ETC. **


#####################################################
# SIMULATE LOAD
#####################################################
export LOAD_TEST="generate_load.sh"
export ERROR_REPORT="print_error_rate.sh"
export QPS=10

# create load test file
cat > $LOAD_TEST << EOF
#!/bin/bash
# Usage: generate_load.sh <URL> <QPS>_

URL=\$1
QPS=\$2

while true
  do for N in \$(seq 1 $QPS)
    do curl -I -XGET -m 5 -s -w "%{http_code}\n" -o /dev/null https://\$URL/ >> output &
    done
  sleep 1
done
EOF

# print error rates
cat > $ERROR_REPORT << EOF
#!/bin/bash
# Usage: watch ./print_error_rate.sh

TOTAL=\$(cat output | wc -l);
SUCCESS=\$(grep "200" output |  wc -l);
ERROR1=\$(grep "000" output |  wc -l)
ERROR2=\$(grep "503" output |  wc -l)
ERROR3=\$(grep "500" output |  wc -l)
ERROR4=\$(grep "403" output |  wc -l)
SUCCESS_RATE=\$((\$SUCCESS * 100 / TOTAL))
ERROR_RATE=\$((\$ERROR1 * 100 / TOTAL))
ERROR_RATE_2=\$((\$ERROR2 * 100 / TOTAL))
ERROR_RATE_3=\$((\$ERROR3 * 100 / TOTAL))
ERROR_RATE_4=\$((\$ERROR4 * 100 / TOTAL))
echo "Success rate: \$SUCCESS/\$TOTAL (\$SUCCESS_RATE%)"
echo "App network Error rate: \$ERROR1/\$TOTAL (\$ERROR_RATE%)"
echo "Resource Error rate: \$ERROR2/\$TOTAL (\$ERROR_RATE_2%)"
echo "App Error rate: \$ERROR3/\$TOTAL (\$ERROR_RATE_3%)"
echo "Rate Limit rate: \$ERROR4/\$TOTAL (\$ERROR_RATE_4%)"
EOF

# make executable
chmod u+x $LOAD_TEST $ERROR_REPORT

# run load test
./$LOAD_TEST "$TEST_NS.$DOMAIN" $QPS 2>&1

# view report (refresh every 2 sec)
watch ./$ERROR_REPORT

Overview

This experiment demonstrates how to spin up a GKE cluster, deploy an app, and configure the BackendConfig service to attach a Cloud Armor policy to the app to rate limit it.

Results

Generate load and monitor response codes

This video capture illustrates how when generating load to the demo URL, initially 200 responses and then rate limiting kicks in and 403 responses, which reset after a minute (per connection).

View demo

After a duration of testing, 100% of traffic blocked

Reviewing logs

The logs reveal banned traffic due to policy

Backing off traffic and warnings reduce

After backing off traffic, the warnings (yellow) in logs reduce