Skip to content

Instantly share code, notes, and snippets.

@taegyunkim
Last active March 26, 2026 03:03
Show Gist options
  • Select an option

  • Save taegyunkim/fba43c9a1f6fcde28ce7798875b558b0 to your computer and use it in GitHub Desktop.

Select an option

Save taegyunkim/fba43c9a1f6fcde28ce7798875b558b0 to your computer and use it in GitHub Desktop.
Reproduce: ddtrace stack profiler empty profiles with ulimit -s unlimited (SCP-1091)
#!/usr/bin/env bash
# Reproduces: CPU/wall-time profiles are empty when RLIMIT_STACK=unlimited on glibc
# Requires: Docker
#
# Usage:
# ./repro_rlimit_stack.sh # install ddtrace==4.2.1 from PyPI
# ./repro_rlimit_stack.sh 4.2.1 # install specific PyPI version
# ./repro_rlimit_stack.sh <40-char commit SHA> # install dev build from S3
#
# Root cause: create_thread_with_stack() passes RLIM_INFINITY to
# pthread_attr_setstacksize(), which glibc accepts (no upper-bound check).
# pthread_create() then fails to mmap() the ~18EB stack => ENOMEM.
# The sampling thread is never created; Python silently ignores the failure.
#
# Dev builds are published to S3 since Jan 2026 (PR #15572):
# https://github.com/DataDog/dd-trace-py/pull/15572
set -eu
# ── Argument parsing ──────────────────────────────────────────────────────────
TARGET="${1:-4.2.1}"
S3_BASE_URL="https://dd-trace-py-builds.s3.amazonaws.com"
# Detect whether TARGET looks like a commit SHA (40 hex chars) or a version
if [[ "$TARGET" =~ ^[0-9a-f]{40}$ ]]; then
INSTALL_MODE="commit"
COMMIT_SHA="$TARGET"
LABEL="commit ${COMMIT_SHA:0:12}"
else
INSTALL_MODE="pypi"
VERSION="$TARGET"
LABEL="v${VERSION} (PyPI)"
fi
# ── Build Docker image ────────────────────────────────────────────────────────
echo "=== Building test image: ddtrace ${LABEL} ==="
if [[ "$INSTALL_MODE" == "commit" ]]; then
docker build -t ddtrace-rlimit-repro - << DOCKERFILE
FROM python:3.11-slim
RUN apt-get update -qq && apt-get install -y -qq golang-go zstd
# Download dev wheel from the public S3 bucket for the given commit SHA.
# Tries several index filenames until one succeeds.
RUN success=0; \\
for idx in index-manylinux2014_x86_64.html index-manylinux2014.html index.html; do \\
echo "Trying ${S3_BASE_URL}/${COMMIT_SHA}/\${idx} ..."; \\
if pip download --no-index --no-deps -d /tmp/ddtrace-wheel \\
--find-links "${S3_BASE_URL}/${COMMIT_SHA}/\${idx}" --pre ddtrace 2>/dev/null; then \\
pip install /tmp/ddtrace-wheel/ddtrace*.whl -q && success=1 && break; \\
fi; \\
done; \\
[ \${success} -eq 1 ] || { echo "ERROR: No wheel found for commit ${COMMIT_SHA} on S3. The CI build may not have completed yet or this commit has no published wheel."; exit 1; }
DOCKERFILE
else
docker build -q -t ddtrace-rlimit-repro - << DOCKERFILE
FROM python:3.11-slim
RUN apt-get update -qq && apt-get install -y -qq golang-go zstd
RUN pip install ddtrace==${VERSION} -q
DOCKERFILE
fi
echo "Image built."
# ── Run tests inside Docker ───────────────────────────────────────────────────
echo ""
docker run --rm ddtrace-rlimit-repro bash -c '
echo "ddtrace: $(ddtrace-run --version)"
echo "libc: $(ldd --version 2>&1 | head -1)"
cat > /tmp/workload.py << PYEOF
import time
for i in range(5):
time.sleep(1)
x = sum(range(100_000))
print("workload done")
PYEOF
run() {
DD_PROFILING_ENABLED=1 DD_PROFILING_OUTPUT_PPROF="$1" \
ddtrace-run python3 /tmp/workload.py 2>/dev/null
}
decode() { zstd -d "$1" -o "$2" -f -q; }
# ── Test 1: normal stack limit ────────────────────────────────────────────────
echo ""
echo "════════════════════════════════════════"
echo " Test 1: normal stack (ulimit -s: $(ulimit -s) KB)"
echo "════════════════════════════════════════"
run /tmp/t1
echo "internal_metadata: $(cat /tmp/t1.*.internal_metadata.json)"
decode "$(ls /tmp/t1.*.pprof)" /tmp/t1_decoded.pprof
echo "pprof: compressed=$(wc -c < $(ls /tmp/t1.*.pprof))B decoded=$(wc -c < /tmp/t1_decoded.pprof)B"
# ── Test 2: unlimited stack ───────────────────────────────────────────────────
echo ""
echo "════════════════════════════════════════"
echo " Test 2: RLIMIT_STACK=unlimited"
echo "════════════════════════════════════════"
ulimit -s unlimited
echo "ulimit -s: $(ulimit -s)"
run /tmp/t2
echo "internal_metadata: $(cat /tmp/t2.*.internal_metadata.json)"
decode "$(ls /tmp/t2.*.pprof)" /tmp/t2_decoded.pprof
echo "pprof: compressed=$(wc -c < $(ls /tmp/t2.*.pprof))B decoded=$(wc -c < /tmp/t2_decoded.pprof)B"
# ── go tool pprof comparison ──────────────────────────────────────────────────
echo ""
echo "════════════════════════════════════════"
echo " go tool pprof -top -sample_index=wall-time"
echo "════════════════════════════════════════"
echo ""
echo "-- Test 1 (normal stack) --"
go tool pprof -top -sample_index=wall-time /tmp/t1_decoded.pprof 2>&1
echo ""
echo "-- Test 2 (unlimited stack) --"
go tool pprof -top -sample_index=wall-time /tmp/t2_decoded.pprof 2>&1
'
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment