Debug netfilter on a Cilium-managed kubernetes node

debug_iptables.sh

#!/bin/bash

STANDARD_CHAINS="CILIUM_INPUT CILIUM_FORWARD CILIUM_OUTPUT"
CUSTOM_CHAINS="CILIUM_PRE CILIUM_POST CILIUM_OUTPUT"
TABLES="raw mangle nat"
NAMESPACE="kube-system"

IP=""
NODE=""
ONLY_CLEAR=false

# get_pods_by_label fetches pods in the specified namespace with the specified
# labels
# $1 - namespace
# $2 - labels
# $3+ - extra agruments to kubectl
get_pods_by_label()
{
    ns=$1
    labels=$2
    shift
    shift
    kubectl -n $ns get pods -l "$labels" $@
}

# Get cilium pod on node
# $1 - node
gcpn()
{
    node=$1
    # NAME           READY   STATUS    RESTARTS   AGE   IP              NODE   NOMINATED NODE   READINESS GATES
    # cilium-794j6   1/1     Running   9          44h   192.168.86.57   node   <none>           <none>
    get_pods_by_label "$NAMESPACE" k8s-app=cilium -o wide | grep "$node" | sed 's/\([^ ]*\).*$/\1/'
}

# Get Cilium and exec on node
# $1 - node
# $2+ - command to run
gcx()
{
    node=$1
    shift
    kubectl exec -n $NAMESPACE -ti $(gcpn $node) -- "$@"
}

# fetch_existing_rules fetches all rules from the specified host which perform
# the LOG action, and assembles a list of iptables command in the specified
# file that will unwind/remove these rules from the system.
#
# $1: host
# $2: file to write iptables commands to
fetch_existing_rules() {
    host=$1
    tmpfile=$2

    gcx $host iptables-save | grep LOG > $tmpfile

    # Remove carriage returns
    sed -i 's/\r//g' $tmpfile
    # Turn this into a series of iptables commands
    sed -i 's/-A \([A-Z_]*\)\([a-z][a-z]*\)/iptables -D \1\2 -t \2/g' $tmpfile
    sed -i 's/-A /iptables -D/g' $tmpfile
    # Fix up w.x.y.z/32 which iptables doesn't accept
    sed -i 's/\([0-9.]*\)\/[0-9]*/\1/g' $tmpfile
}

# assemble_new_rules adds a list of iptables commands to the specified file to
# trace packet paths through the netfilter stack for the specified IP.
#
# $1: IP address to pay attention to
# $2: file to write rules to
assemble_new_rules() {
    addr=$1
    tmpfile=$2
    for chain in $CUSTOM_CHAINS; do
        for table in $TABLES; do
            c="${chain}_${table}"
            skip=false
            case $c in
            "CILIUM_POST_raw")
                skip=true
                ;;
            "CILIUM_OUTPUT_mangle")
                skip=true
                ;;
            *)
                ;;
            esac
            if $skip; then
                continue
            fi
            prefix=$(printf '%-20s' "$c: ")
            echo "iptables -t $table -I $c -p tcp -s $addr -j LOG --log-prefix=\"$prefix\" 2>/dev/null || echo \"Failed to configure $c\"" >> $tmpfile
            echo "iptables -t $table -I $c -p tcp -d $addr -j LOG --log-prefix=\"$prefix\" 2>/dev/null || echo \"Failed to configure $c\"" >> $tmpfile
        done
    done

    for chain in $STANDARD_CHAINS; do
        prefix=$(printf '%-20s' "$chain: ")
        echo "iptables -I $chain -p tcp -s $addr -j LOG --log-prefix=\"$prefix\" 2>/dev/null || echo \"Failed to configure $chain\"" >> $tmpfile
        echo "iptables -I $chain -p tcp -d $addr -j LOG --log-prefix=\"$prefix\" 2>/dev/null || echo \"Failed to configure $chain\"" >> $tmpfile
    done
}

# usage prints the usage for the command and exits with the specified code.
# $1: exit code
usage() {
    echo "usage: $0 [-c] [-n NAMESPACE] <NODE> <IP>" >&2
    echo >&2
    echo "Configure tracing iptables rules (-j LOG) in the Cilium pod on NODE" >&2
    echo "to trace TCP traffic to/from IP." >&2
    echo >&2
    echo "Flags" >&2
    echo " -c | --clear       Only clear the existing LOG iptables rules from NODE" >&2
    echo " -h | --help        Print this usage message and exit" >&2
    echo " -n | --namespace   Kubernetes namespace to locate Cilium pods" >&2
    exit $1
}

# handle_args checks the usage of the arguments and availability of dependencies.
handle_args() {
    while [[ "$1" =~ ^-.* ]]; do
        case $1 in
            -c | --clear )      shift
                                ONLY_CLEAR=true
                                ;;
            -h | --help )       usage 0
                                ;;
            -n | --namespace )  shift
                                NAMESPACE=$1
                                shift
                                ;;
            *)
                            usage 1
                            ;;
        esac
    done

    if [ $# -lt 2 ]; then
        usage 1
    fi

    if ! which kubectl >/dev/null; then
        echo "This command relies upon kubectl to exec into Cilium pods. Install kubectl." >&2
        echo >&2
        usage 1
    fi

    NODE="$1"
    IP=$2
    if [ "$(gcpn $NODE)" == "" ]; then
        echo "Cannot locate cilium pod on node \"$NODE\". Is it running there?" >&2
        echo >&2
        usage 1
    fi
}

# main is the primary launch point of this script.
main() {
    handle_args "$@"

    tmpfile=$(mktemp)
    trap "rm $tmpfile" EXIT

    echo "Clearing existing iptables -j LOG rules..." >&2
    fetch_existing_rules $NODE $tmpfile
    if ! $ONLY_CLEAR; then
        echo "Applying trace to IP $IP..." >&2
        assemble_new_rules $IP $tmpfile
    fi

    chmod +x $tmpfile
    kubectl -n $NAMESPACE cp $tmpfile "$(gcpn $NODE)":/tmp/iptables.sh
    gcx $NODE sh -c /tmp/iptables.sh
}

main "$@"