Last active
March 26, 2026 03:03
-
-
Save taegyunkim/fba43c9a1f6fcde28ce7798875b558b0 to your computer and use it in GitHub Desktop.
Reproduce: ddtrace stack profiler empty profiles with ulimit -s unlimited (SCP-1091)
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| #!/usr/bin/env bash | |
| # Reproduces: CPU/wall-time profiles are empty when RLIMIT_STACK=unlimited on glibc | |
| # Requires: Docker | |
| # | |
| # Usage: | |
| # ./repro_rlimit_stack.sh # install ddtrace==4.2.1 from PyPI | |
| # ./repro_rlimit_stack.sh 4.2.1 # install specific PyPI version | |
| # ./repro_rlimit_stack.sh <40-char commit SHA> # install dev build from S3 | |
| # | |
| # Root cause: create_thread_with_stack() passes RLIM_INFINITY to | |
| # pthread_attr_setstacksize(), which glibc accepts (no upper-bound check). | |
| # pthread_create() then fails to mmap() the ~18EB stack => ENOMEM. | |
| # The sampling thread is never created; Python silently ignores the failure. | |
| # | |
| # Dev builds are published to S3 since Jan 2026 (PR #15572): | |
| # https://github.com/DataDog/dd-trace-py/pull/15572 | |
| set -eu | |
| # ── Argument parsing ────────────────────────────────────────────────────────── | |
| TARGET="${1:-4.2.1}" | |
| S3_BASE_URL="https://dd-trace-py-builds.s3.amazonaws.com" | |
| # Detect whether TARGET looks like a commit SHA (40 hex chars) or a version | |
| if [[ "$TARGET" =~ ^[0-9a-f]{40}$ ]]; then | |
| INSTALL_MODE="commit" | |
| COMMIT_SHA="$TARGET" | |
| LABEL="commit ${COMMIT_SHA:0:12}" | |
| else | |
| INSTALL_MODE="pypi" | |
| VERSION="$TARGET" | |
| LABEL="v${VERSION} (PyPI)" | |
| fi | |
| # ── Build Docker image ──────────────────────────────────────────────────────── | |
| echo "=== Building test image: ddtrace ${LABEL} ===" | |
| if [[ "$INSTALL_MODE" == "commit" ]]; then | |
| docker build -t ddtrace-rlimit-repro - << DOCKERFILE | |
| FROM python:3.11-slim | |
| RUN apt-get update -qq && apt-get install -y -qq golang-go zstd | |
| # Download dev wheel from the public S3 bucket for the given commit SHA. | |
| # Tries several index filenames until one succeeds. | |
| RUN success=0; \\ | |
| for idx in index-manylinux2014_x86_64.html index-manylinux2014.html index.html; do \\ | |
| echo "Trying ${S3_BASE_URL}/${COMMIT_SHA}/\${idx} ..."; \\ | |
| if pip download --no-index --no-deps -d /tmp/ddtrace-wheel \\ | |
| --find-links "${S3_BASE_URL}/${COMMIT_SHA}/\${idx}" --pre ddtrace 2>/dev/null; then \\ | |
| pip install /tmp/ddtrace-wheel/ddtrace*.whl -q && success=1 && break; \\ | |
| fi; \\ | |
| done; \\ | |
| [ \${success} -eq 1 ] || { echo "ERROR: No wheel found for commit ${COMMIT_SHA} on S3. The CI build may not have completed yet or this commit has no published wheel."; exit 1; } | |
| DOCKERFILE | |
| else | |
| docker build -q -t ddtrace-rlimit-repro - << DOCKERFILE | |
| FROM python:3.11-slim | |
| RUN apt-get update -qq && apt-get install -y -qq golang-go zstd | |
| RUN pip install ddtrace==${VERSION} -q | |
| DOCKERFILE | |
| fi | |
| echo "Image built." | |
| # ── Run tests inside Docker ─────────────────────────────────────────────────── | |
| echo "" | |
| docker run --rm ddtrace-rlimit-repro bash -c ' | |
| echo "ddtrace: $(ddtrace-run --version)" | |
| echo "libc: $(ldd --version 2>&1 | head -1)" | |
| cat > /tmp/workload.py << PYEOF | |
| import time | |
| for i in range(5): | |
| time.sleep(1) | |
| x = sum(range(100_000)) | |
| print("workload done") | |
| PYEOF | |
| run() { | |
| DD_PROFILING_ENABLED=1 DD_PROFILING_OUTPUT_PPROF="$1" \ | |
| ddtrace-run python3 /tmp/workload.py 2>/dev/null | |
| } | |
| decode() { zstd -d "$1" -o "$2" -f -q; } | |
| # ── Test 1: normal stack limit ──────────────────────────────────────────────── | |
| echo "" | |
| echo "════════════════════════════════════════" | |
| echo " Test 1: normal stack (ulimit -s: $(ulimit -s) KB)" | |
| echo "════════════════════════════════════════" | |
| run /tmp/t1 | |
| echo "internal_metadata: $(cat /tmp/t1.*.internal_metadata.json)" | |
| decode "$(ls /tmp/t1.*.pprof)" /tmp/t1_decoded.pprof | |
| echo "pprof: compressed=$(wc -c < $(ls /tmp/t1.*.pprof))B decoded=$(wc -c < /tmp/t1_decoded.pprof)B" | |
| # ── Test 2: unlimited stack ─────────────────────────────────────────────────── | |
| echo "" | |
| echo "════════════════════════════════════════" | |
| echo " Test 2: RLIMIT_STACK=unlimited" | |
| echo "════════════════════════════════════════" | |
| ulimit -s unlimited | |
| echo "ulimit -s: $(ulimit -s)" | |
| run /tmp/t2 | |
| echo "internal_metadata: $(cat /tmp/t2.*.internal_metadata.json)" | |
| decode "$(ls /tmp/t2.*.pprof)" /tmp/t2_decoded.pprof | |
| echo "pprof: compressed=$(wc -c < $(ls /tmp/t2.*.pprof))B decoded=$(wc -c < /tmp/t2_decoded.pprof)B" | |
| # ── go tool pprof comparison ────────────────────────────────────────────────── | |
| echo "" | |
| echo "════════════════════════════════════════" | |
| echo " go tool pprof -top -sample_index=wall-time" | |
| echo "════════════════════════════════════════" | |
| echo "" | |
| echo "-- Test 1 (normal stack) --" | |
| go tool pprof -top -sample_index=wall-time /tmp/t1_decoded.pprof 2>&1 | |
| echo "" | |
| echo "-- Test 2 (unlimited stack) --" | |
| go tool pprof -top -sample_index=wall-time /tmp/t2_decoded.pprof 2>&1 | |
| ' |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment