captured.sh

#!/usr/bin/env bash

# captured.sh — run a target executable with all its HTTP(S) traffic forced
# through a mitmproxy for both inspection and control.
#
# Full documentation lives in usage() below (`captured.sh --help`).

set -euo pipefail

# ----------------------------------------------------------------------------
# Defaults
# ----------------------------------------------------------------------------
MODE="web"
ALLOW_LIST="allow-list.txt"
DENY_STATUS="403"
DENY_MESSAGE="blocked by \`captured\` script allow-list"
PORT="47080"
WEB_PORT="47081"
LOG_PATH=""      # empty => stderr
KEEP="0"

CA_CERT="${HOME}/.mitmproxy/mitmproxy-ca-cert.pem"

PROG="$(basename "$0")"

usage() {
    cat <<'EOF'
captured.sh — run a target executable with all its HTTP(S) traffic forced
through a mitmproxy for both inspection and control

Usage:
  captured.sh [flags] -- <target> [target args...]

Flags (everything before the required `--`):
  --web              use mitmweb (default; auto-opens a browser)
  --tui              use mitmproxy TUI (needs the terminal; see note below)
  --dump             use mitmdump (streams flows to stderr)
  --allow-list PATH  allow-list file (default: ./allow-list.txt).
                     Check ALLOW LIST section below for details
  --deny-status N    HTTP status returned for requests matching no entry
                     (default: 403); per-entry codes override it
  --deny-message MSG body returned for any locally-answered request
                     (default: "blocked by captured.sh allow-list")
  --port N           proxy listen port (default: 47080)
  --web-port N       mitmweb UI port (default: 47081)
  --log PATH         send proxy output to PATH (default: --web => temp file,
                     --dump/--tui => stderr)
  --keep             leave the proxy running after the target exits
  -h, --help         show this help

ALLOW LIST:
  - Missing allow-list file  => allow-all (a warning is logged).
    Present but empty file   => deny-all.
  - The allow-list is hot-reloaded: edits take effect on the next request.
  - Blank lines are ignored; `#` starts a comment (whole-line or suffix),
    e.g. `* https://www.google.com/* forward  # allow all to google`.
  - Each entry has three fields — <method> <full URL> <action> — separated by
    runs of whitespace. <method> and <full URL> are independently a glob
    (default) or a `/<regex>/`, full-matched (case-sensitive) against the
    request's method and full URL; BOTH must match for the entry to apply:
        GET         https://api.example.com/v1/*      forward
        /GET|POST/  https://example.com/*             forward
        *           /https:\/\/example\.com\/(a|b)/   forward
    (The method field cannot contain whitespace; the URL field may, as long
    as it does not end in something that looks like an <action>.)
  - <action> is required — either `forward` (send the request upstream) or a
    3-digit status answered locally, without ever contacting the host:
        POST  https://hooks.slack.com/*  200   # swallow, tell the app it worked
        *     https://ads.example.com/*  204
    Entries without an <action> are skipped with a warning.
  - Requests matching no entry get --deny-status (default 403).

NOTES:
  - `--tui` renders in the terminal, but this script backgrounds the proxy so
    the target owns the foreground. Combining --tui with an interactive target
    will lead to garbled output
EOF
}

die() {
    echo "${PROG}: $*" >&2
    exit 1
}

# ----------------------------------------------------------------------------
# Parse captured.sh flags up to the required `--`
# ----------------------------------------------------------------------------
TARGET=()
saw_sep="0"
while [[ $# -gt 0 ]]; do
    case "$1" in
        --) saw_sep="1"; shift; TARGET=("$@"); break ;;
        --web)  MODE="web";  shift ;;
        --tui)  MODE="tui";  shift ;;
        --dump) MODE="dump"; shift ;;
        --allow-list) [[ $# -ge 2 ]] || die "--allow-list needs a path"; ALLOW_LIST="$2"; shift 2 ;;
        --deny-status)  [[ $# -ge 2 ]] || die "--deny-status needs a number"; DENY_STATUS="$2";  shift 2 ;;
        --deny-message) [[ $# -ge 2 ]] || die "--deny-message needs a string"; DENY_MESSAGE="$2"; shift 2 ;;
        --port)       [[ $# -ge 2 ]] || die "--port needs a number";     PORT="$2";       shift 2 ;;
        --web-port)   [[ $# -ge 2 ]] || die "--web-port needs a number"; WEB_PORT="$2";   shift 2 ;;
        --log)        [[ $# -ge 2 ]] || die "--log needs a path";        LOG_PATH="$2";   shift 2 ;;
        --keep) KEEP="1"; shift ;;
        -h|--help) usage; exit 0 ;;
        -*) die "unknown flag: $1 (see --help)" ;;
        *)  die "unexpected argument before '--': $1 (flags must precede '--')" ;;
    esac
done

[[ "$saw_sep" == "1" ]] || die "missing '--' separator.
usage: ${PROG} [flags] -- <target> [args...]"
[[ ${#TARGET[@]} -gt 0 ]] || die "no target command after '--'"

command -v mitmweb >/dev/null 2>&1 || die "mitmproxy not found on PATH (brew install mitmproxy)"

# Resolve allow-list to an absolute path so the addon finds it regardless of
# any cwd changes; a missing file is fine (=> allow-all), don't fail here.
if [[ -f "$ALLOW_LIST" ]]; then
    ALLOW_LIST_ABS="$(cd "$(dirname "$ALLOW_LIST")" && pwd)/$(basename "$ALLOW_LIST")"
else
    echo "${PROG}: allow-list '${ALLOW_LIST}' not found — allowing all domains" >&2
    # Pass the (absolute-ish) path anyway; the addon treats missing as allow-all
    # and will pick the file up if it later appears.
    case "$ALLOW_LIST" in
        /*) ALLOW_LIST_ABS="$ALLOW_LIST" ;;
        *)  ALLOW_LIST_ABS="$(pwd)/${ALLOW_LIST}" ;;
    esac
fi

# ----------------------------------------------------------------------------
# Write the inline mitmproxy addon to a temp file
# ----------------------------------------------------------------------------
ADDON_FILE="$(mktemp -t captured-allowlist.XXXXXX.py)"

cat > "$ADDON_FILE" <<'EOF'
"""Inline mitmproxy addon for captured.sh — deny all hosts except an allow-list.

The allow-list path comes from $CAPTURED_ALLOW_LIST. The file is hot-reloaded:
its mtime is checked on every request and reloaded when it changes.

Semantics:
  - file missing  -> allow all (bypass filtering)
  - file present  -> requests matching no entry are answered with DENY_STATUS
                     (an empty file therefore blocks everything)

Blank lines are ignored; `#` starts a comment (whole-line or line suffix) and
the rest of the line is dropped, so `#` cannot appear literally in a pattern.

Each entry is three fields — <method> <url> <action> — separated by whitespace
runs (the method field holds no whitespace; the URL field may). <method> and
<url> are independently a glob (default) or a `/<regex>/`, full-matched
case-sensitively against the request's method and full URL (with scheme); BOTH
must match for the entry to apply:
  - `/<regex>/`   -> Python regex, re.fullmatch
  - anything else -> glob, fnmatchcase

<action> is mandatory and explicit: `forward` sends the request upstream, a
3-digit status is answered locally without contacting the host.
"""
import fnmatch
import os
import re

from mitmproxy import http

ALLOW_LIST_PATH = os.environ.get("CAPTURED_ALLOW_LIST", "")
DENY_STATUS = int(os.environ.get("CAPTURED_DENY_STATUS", "403"))
DENY_MESSAGE = os.environ.get("CAPTURED_DENY_MESSAGE", "blocked by captured.sh allow-list")

# Trailing `forward` or 3-digit status; anchored at end so a URL pattern that
# contains whitespace keeps working.
ACTION_RE = re.compile(r"\s+(forward|\d{3})$", re.IGNORECASE)

_state = {"mtime": None, "rules": [], "allow_all": True, "loaded": False}


def _compile_field(text: str):
    """A field is a `/<regex>/` (Python regex) or otherwise a glob pattern.

    Returns a ("regex", compiled) or ("glob", pattern) matcher. Raises
    re.error on an invalid regex so the caller can skip the whole entry.
    """
    if len(text) >= 2 and text.startswith("/") and text.endswith("/"):
        return ("regex", re.compile(text[1:-1]))
    return ("glob", text)


def _load() -> None:
    """(Re)load the allow-list if the file appeared, vanished, or changed."""
    try:
        mtime = os.path.getmtime(ALLOW_LIST_PATH)
    except OSError:
        # File missing -> allow-all. Log the transition once.
        if not _state["allow_all"] or not _state["loaded"]:
            print(f"[captured] allow-list '{ALLOW_LIST_PATH}' missing -> allowing all")
        _state.update(mtime=None, rules=[], allow_all=True, loaded=True)
        return

    if _state["loaded"] and _state["mtime"] == mtime:
        return  # unchanged

    # A `#` starts a comment (whole-line or suffix); the rest of the line is
    # dropped. Blank/comment-only lines become empty and are skipped.
    with open(ALLOW_LIST_PATH) as f:
        entries = [
            stripped
            for line in f
            if (stripped := line.split("#", 1)[0].strip())
        ]

    # Compile each entry into (method_matcher, url_matcher, status), where each
    # matcher is a ("glob"|"regex", ...) tuple and status is None for `forward`.
    # Malformed entries and invalid regexes are skipped with a warning rather
    # than taking down the addon.
    rules = []
    for entry in entries:
        parts = entry.split(None, 1)  # split on first whitespace run -> 2 fields
        action = ACTION_RE.search(parts[1]) if len(parts) == 2 else None
        if action is None or not (url := parts[1][:action.start()]):
            print(f"[captured] skipping malformed entry "
                  f"(need '<method> <url> <forward|status>'): '{entry}'")
            continue
        verb = action.group(1).lower()
        status = None if verb == "forward" else int(verb)
        try:
            method_m, url_m = (_compile_field(p) for p in (parts[0], url))
        except re.error as exc:
            print(f"[captured] skipping entry with invalid regex '{entry}': {exc}")
            continue
        rules.append((method_m, url_m, status))

    _state.update(mtime=mtime, rules=rules, allow_all=False, loaded=True)
    print(f"[captured] loaded {len(rules)} allow-list entr"
          f"{'y' if len(rules) == 1 else 'ies'} from {ALLOW_LIST_PATH}")


def _field_match(matcher, value: str) -> bool:
    kind, m = matcher
    if kind == "regex":
        return m.fullmatch(value) is not None
    return fnmatch.fnmatchcase(value, m)


def _status_for(method: str, url: str):
    """Status to answer with locally, or None to forward the request."""
    for method_m, url_m, status in _state["rules"]:
        if _field_match(method_m, method) and _field_match(url_m, url):
            return status
    return DENY_STATUS  # matched no entry


def request(flow: http.HTTPFlow) -> None:
    _load()
    if _state["allow_all"]:
        return
    status = _status_for(flow.request.method, flow.request.pretty_url)
    if status is None:
        return
    # ponytail: one canned text/plain body for every locally-answered request;
    # add a per-entry body/headers field if a stub ever needs real JSON.
    flow.response = http.Response.make(
        status,
        (DENY_MESSAGE + "\n").encode(),
        {"Content-Type": "text/plain"},
    )
EOF

# ----------------------------------------------------------------------------
# Build the mitmproxy command for the chosen mode
# ----------------------------------------------------------------------------
CMD=()
case "$MODE" in
    web)  CMD=(mitmweb   --listen-port "$PORT" --web-port "$WEB_PORT" -s "$ADDON_FILE") ;;
    tui)  CMD=(mitmproxy --listen-port "$PORT" -s "$ADDON_FILE") ;;
    dump) CMD=(mitmdump  --listen-port "$PORT" -s "$ADDON_FILE") ;;
esac

# ----------------------------------------------------------------------------
# Start the proxy (backgrounded), routing output per --log
# ----------------------------------------------------------------------------
PROXY_PID=""

# Exec the given command in a NEW SESSION (its own process group, detached from
# the controlling terminal). This is how the proxy is shielded from Ctrl-C: a
# terminal SIGINT is delivered to the foreground process group only, so a
# detached proxy never receives it. A shell `trap '' INT` is NOT enough here —
# mitmproxy installs its own asyncio SIGINT/SIGTERM handlers at startup, which
# override any inherited SIG_IGN. cleanup() stops the proxy explicitly by PID.
# Prefer setsid (Linux); fall back to perl/python (always present on macOS).
detach() {
    if command -v setsid >/dev/null 2>&1; then
        exec setsid "$@"
    elif command -v perl >/dev/null 2>&1; then
        exec perl -e 'use POSIX qw(setsid); setsid() or die $!; exec @ARGV or die $!' -- "$@"
    else
        exec python3 -c 'import os,sys; os.setsid(); os.execvp(sys.argv[1], sys.argv[1:])' "$@"
    fi
}

start_proxy() {
    export CAPTURED_ALLOW_LIST="$ALLOW_LIST_ABS"
    export CAPTURED_DENY_STATUS="$DENY_STATUS"
    export CAPTURED_DENY_MESSAGE="$DENY_MESSAGE"

    # Resolve where proxy output goes:
    #   --log wins for every mode; otherwise --web => temp file, else => stderr.
    local effective_log="$LOG_PATH"
    if [[ -z "$effective_log" && "$MODE" == "web" ]]; then
        effective_log="$(mktemp -t captured-proxy.XXXXXX.log)"
        echo "${PROG}: proxy log -> ${effective_log}" >&2
    fi

    # Detach the proxy so it outlives the target across a terminal Ctrl-C and is
    # torn down only by cleanup(), after the target has fully exited.
    if [[ -n "$effective_log" ]]; then
        detach "${CMD[@]}" >"$effective_log" 2>&1 &
    else
        detach "${CMD[@]}" >&2 &
    fi
    PROXY_PID="$!"
}

# ----------------------------------------------------------------------------
# Teardown: kill proxy + remove temp addon, unless --keep
# ----------------------------------------------------------------------------
CLEANED="0"
cleanup() {
    local code="$?"
    # Idempotent: the INT/TERM trap runs cleanup then exits, which re-fires the
    # EXIT trap — only tear down once.
    [[ "$CLEANED" == "1" ]] && exit "$code"
    CLEANED="1"
    if [[ "$KEEP" == "1" ]]; then
        echo "${PROG}: --keep set; proxy still running (pid ${PROXY_PID}). Stop it with: kill ${PROXY_PID}" >&2
        echo "${PROG}: (addon left at ${ADDON_FILE})" >&2
    else
        if [[ -n "$PROXY_PID" ]] && kill -0 "$PROXY_PID" 2>/dev/null; then
            kill -TERM "$PROXY_PID" 2>/dev/null || true
            # Give it a moment to shut down cleanly; SIGKILL if it lingers.
            for _ in 1 2 3 4 5 6 7 8 9 10; do
                kill -0 "$PROXY_PID" 2>/dev/null || break
                sleep 0.1
            done
            kill -0 "$PROXY_PID" 2>/dev/null && kill -KILL "$PROXY_PID" 2>/dev/null || true
            wait "$PROXY_PID" 2>/dev/null || true
        fi
        rm -f "$ADDON_FILE"
    fi
    exit "$code"
}
trap cleanup EXIT INT TERM

# ----------------------------------------------------------------------------
# Readiness gate: wait for the listen port AND the CA cert to exist
# ----------------------------------------------------------------------------
port_listening() {
    (exec 3<>"/dev/tcp/127.0.0.1/${PORT}") 2>/dev/null && exec 3>&- 3<&- && return 0
    return 1
}

wait_ready() {
    local waited=0 max=100  # 100 * 0.1s = 10s
    while (( waited < max )); do
        if [[ -n "$PROXY_PID" ]] && ! kill -0 "$PROXY_PID" 2>/dev/null; then
            die "proxy exited during startup (see output above); port ${PORT} may be in use"
        fi
        if port_listening && [[ -f "$CA_CERT" ]]; then
            return 0
        fi
        sleep 0.1
        (( waited += 1 ))
    done
    die "proxy did not become ready within 10s (port ${PORT}, CA ${CA_CERT})"
}

# ----------------------------------------------------------------------------
# Target environment: proxy routing + CA trust across many toolchains
# ----------------------------------------------------------------------------
run_target() {
    local proxy_url="http://127.0.0.1:${PORT}"
    env \
        http_proxy="$proxy_url"  HTTP_PROXY="$proxy_url" \
        https_proxy="$proxy_url" HTTPS_PROXY="$proxy_url" \
        all_proxy="$proxy_url"   ALL_PROXY="$proxy_url" \
        no_proxy="localhost,127.0.0.1,::1" NO_PROXY="localhost,127.0.0.1,::1" \
        SSL_CERT_FILE="$CA_CERT" \
        CURL_CA_BUNDLE="$CA_CERT" \
        REQUESTS_CA_BUNDLE="$CA_CERT" \
        NODE_EXTRA_CA_CERTS="$CA_CERT" \
        GIT_SSL_CAINFO="$CA_CERT" \
        AWS_CA_BUNDLE="$CA_CERT" \
        CARGO_HTTP_CAINFO="$CA_CERT" \
        DENO_CERT="$CA_CERT" \
        PIP_CERT="$CA_CERT" \
        "${TARGET[@]}"
}

# ----------------------------------------------------------------------------
# Orchestrate
# ----------------------------------------------------------------------------
echo "${PROG}: starting mitmproxy (${MODE}) on port ${PORT}${LOG_PATH:+, log -> ${LOG_PATH}}" >&2
start_proxy
wait_ready
echo "${PROG}: proxy ready (pid ${PROXY_PID}); launching: ${TARGET[*]}" >&2

# Run the target in the FOREGROUND so it keeps normal signal semantics — an
# interactive target (rails console, a REPL) must be able to catch Ctrl-C
# itself. (Backgrounding it would make bash mark SIGINT as ignored-on-entry,
# which the target then cannot trap.)
#
# Shutdown ordering is guaranteed by two things:
#   1. The proxy ignores INT/HUP (see start_proxy), so a terminal Ctrl-C —
#      delivered to the whole foreground process group at once — does NOT kill
#      it. The target can perform final network IO against a live proxy.
#   2. bash DEFERS the INT/TERM cleanup trap until the foreground target has
#      exited, so the proxy is only torn down AFTER the target is gone.
set +e
run_target
target_code="$?"
set -e
exit "$target_code"