Csound recursive struct lifetime benchmarks and charts

README.md

Csound recursive struct lifetime benchmarks

This bundle contains the benchmark runner, chart generator, raw measurements, and rendered charts used for the recursive struct-array lifetime pull request.

Contents

• benchmark.py: interleaved process runner and 25 ms CPU/RSS sampler.
• plot.py: generates the three Grafana-style charts from the CSV files.
• summary.csv: per-process wall time, CPU time, return code, and peak RSS.
• traces.csv: sampled CPU and resident memory over time.
• environment.txt: platform and revision metadata for the published run.
• The three PNG files are the rendered charts used in the pull request.

Gist uses a flat file namespace. When run locally, benchmark.py writes data under raw/, and plot.py accepts data either there or beside the script. New charts are written under charts/.

The runner is POSIX-specific because it uses fork, wait4, and ps to keep per-process resource accounting separate. The chart generator requires Matplotlib and NumPy.

Run

Both build directories must contain a csound executable and its locally built library and opcode modules. The structured workload may be a CSD, or an orchestra file when the required Csound arguments are supplied with repeated --structured-arg=VALUE options.

python3 benchmark.py \
  --develop /path/to/develop/build \
  --branch /path/to/branch/build \
  --stria /path/to/Chowning-Stria.csd \
  --structured-workload /path/to/recursive-workload.csd \
  --develop-commit 0c9f95fbc676ad2258d1dca94ba904e591500efa \
  --branch-commit 9910a8e287fc599bed3f22e97a2ccf4fcfbf0892

The runner performs one Stria warmup per revision, five interleaved measured Stria runs, and three interleaved structured-workload runs. It writes CSV data and individual process logs under raw/.

To recreate the charts:

python3 -m pip install matplotlib numpy
python3 plot.py

Interpretation

The Stria measurements are a regression check. Their small differences are within run-to-run noise and are not presented as a speedup.

The recursive workload is an outcome and memory comparison. The develop build aborted in all three measurements, while the branch completed all three. A runtime comparison would therefore be misleading. Peak RSS is an end-to-end process measurement rather than a byte-exact count of live Csound allocations.

benchmark.py

#!/usr/bin/env python3

from __future__ import annotations

import argparse
import csv
import os
import platform
import resource
import subprocess
import sys
import time
from pathlib import Path


ROOT = Path(__file__).resolve().parent
RAW = ROOT / "raw"


def rss_megabytes(raw_rss: int) -> float:
    divisor = 1024 * 1024 if sys.platform == "darwin" else 1024
    return raw_rss / divisor


def sample_process(pid: int) -> tuple[float, float] | None:
    result = subprocess.run(
        ["ps", "-o", "rss=,%cpu=", "-p", str(pid)],
        capture_output=True,
        check=False,
        text=True,
    )
    fields = result.stdout.split()
    if len(fields) != 2:
        return None
    return int(fields[0]) / 1024, float(fields[1])


def run_process(
    *,
    variant: str,
    benchmark: str,
    iteration: int,
    binary: Path,
    framework: Path,
    arguments: list[str],
    cwd: Path,
    summary_writer: csv.DictWriter,
    trace_writer: csv.DictWriter,
) -> dict[str, float | int | str]:
    log_path = RAW / f"{benchmark}-{variant}-{iteration}.log"
    environment = os.environ.copy()
    if sys.platform == "darwin":
        environment["DYLD_FRAMEWORK_PATH"] = str(framework)
    else:
        previous = environment.get("LD_LIBRARY_PATH", "")
        environment["LD_LIBRARY_PATH"] = (
            f"{framework}:{previous}" if previous else str(framework)
        )
    environment["OPCODE7DIR64"] = str(framework)
    started = time.perf_counter()
    samples: list[tuple[float, float, float]] = []

    pid = os.fork()
    if pid == 0:
        descriptor = os.open(
            log_path,
            os.O_WRONLY | os.O_CREAT | os.O_TRUNC,
            0o644,
        )
        os.dup2(descriptor, 1)
        os.dup2(descriptor, 2)
        os.close(descriptor)
        os.chdir(cwd)
        os.execve(binary, [str(binary), *arguments], environment)

    status = 0
    usage = None
    while True:
        waited_pid, status, usage = os.wait4(pid, os.WNOHANG)
        elapsed = time.perf_counter() - started
        sample = sample_process(pid)
        if sample is not None:
            samples.append((elapsed, sample[0], sample[1]))
        if waited_pid == pid:
            break
        time.sleep(0.025)

    elapsed = time.perf_counter() - started
    return_code = os.waitstatus_to_exitcode(status)
    user_seconds = usage.ru_utime
    system_seconds = usage.ru_stime
    cpu_seconds = user_seconds + system_seconds
    peak_rss = rss_megabytes(usage.ru_maxrss)
    average_cpu = cpu_seconds / elapsed * 100 if elapsed > 0 else 0

    row = {
        "benchmark": benchmark,
        "variant": variant,
        "iteration": iteration,
        "return_code": return_code,
        "wall_seconds": round(elapsed, 6),
        "user_seconds": round(user_seconds, 6),
        "system_seconds": round(system_seconds, 6),
        "cpu_seconds": round(cpu_seconds, 6),
        "average_cpu_percent": round(average_cpu, 3),
        "peak_rss_mb": round(peak_rss, 3),
    }
    summary_writer.writerow(row)

    for elapsed_sample, rss_mb, cpu_percent in samples:
        trace_writer.writerow(
            {
                "benchmark": benchmark,
                "variant": variant,
                "iteration": iteration,
                "elapsed_seconds": round(elapsed_sample, 6),
                "rss_mb": round(rss_mb, 3),
                "cpu_percent": round(cpu_percent, 3),
            }
        )
    return row


def main() -> None:
    parser = argparse.ArgumentParser()
    parser.add_argument("--develop", type=Path, required=True)
    parser.add_argument("--branch", type=Path, required=True)
    parser.add_argument("--stria", type=Path, required=True)
    parser.add_argument("--structured-workload", type=Path, required=True)
    parser.add_argument("--structured-cwd", type=Path)
    parser.add_argument(
        "--structured-arg",
        action="append",
        default=[],
        help="extra Csound argument; use --structured-arg=VALUE",
    )
    parser.add_argument("--stria-runs", type=int, default=5)
    parser.add_argument("--structured-runs", type=int, default=3)
    parser.add_argument("--develop-commit", default="unknown")
    parser.add_argument("--branch-commit", default="unknown")
    args = parser.parse_args()
    RAW.mkdir(exist_ok=True)

    variants = {
        "develop": (args.develop / "csound", args.develop),
        "branch": (args.branch / "csound", args.branch),
    }
    summary_path = RAW / "summary.csv"
    trace_path = RAW / "traces.csv"
    summary_fields = [
        "benchmark",
        "variant",
        "iteration",
        "return_code",
        "wall_seconds",
        "user_seconds",
        "system_seconds",
        "cpu_seconds",
        "average_cpu_percent",
        "peak_rss_mb",
    ]
    trace_fields = [
        "benchmark",
        "variant",
        "iteration",
        "elapsed_seconds",
        "rss_mb",
        "cpu_percent",
    ]

    with summary_path.open("w", newline="", encoding="ascii") as summary_file, \
         trace_path.open("w", newline="", encoding="ascii") as trace_file:
        summary_writer = csv.DictWriter(summary_file, fieldnames=summary_fields)
        trace_writer = csv.DictWriter(trace_file, fieldnames=trace_fields)
        summary_writer.writeheader()
        trace_writer.writeheader()

        # Warm both binaries and the filesystem cache before measured renders.
        for variant, (binary, framework) in variants.items():
            run_process(
                variant=variant,
                benchmark="stria-warmup",
                iteration=0,
                binary=binary,
                framework=framework,
                arguments=["-n", "-d", "-m0", str(args.stria)],
                cwd=ROOT,
                summary_writer=summary_writer,
                trace_writer=trace_writer,
            )

        for iteration in range(1, args.stria_runs + 1):
            order = ("develop", "branch") if iteration % 2 else ("branch", "develop")
            for variant in order:
                binary, framework = variants[variant]
                run_process(
                    variant=variant,
                    benchmark="stria",
                    iteration=iteration,
                    binary=binary,
                    framework=framework,
                    arguments=["-n", "-d", "-m0", str(args.stria)],
                    cwd=ROOT,
                    summary_writer=summary_writer,
                    trace_writer=trace_writer,
                )

        structured_cwd = (
            args.structured_cwd
            if args.structured_cwd is not None
            else args.structured_workload.parent
        )
        for iteration in range(1, args.structured_runs + 1):
            order = ("develop", "branch") if iteration % 2 else ("branch", "develop")
            for variant in order:
                binary, framework = variants[variant]
                run_process(
                    variant=variant,
                    benchmark="recursive-structured-workload",
                    iteration=iteration,
                    binary=binary,
                    framework=framework,
                    arguments=[
                        "-n",
                        "-d",
                        "-m0",
                        *args.structured_arg,
                        str(args.structured_workload),
                    ],
                    cwd=structured_cwd,
                    summary_writer=summary_writer,
                    trace_writer=trace_writer,
                )

    metadata = RAW / "environment.txt"
    metadata.write_text(
        "\n".join(
            [
                f"platform={platform.platform()}",
                f"machine={platform.machine()}",
                f"python={platform.python_version()}",
                f"develop_commit={args.develop_commit}",
                f"branch_commit={args.branch_commit}",
                f"stria_source={args.stria}",
                f"structured_workload={args.structured_workload}",
                "sample_interval_seconds=0.025",
            ]
        ) + "\n",
        encoding="ascii",
    )


if __name__ == "__main__":
    main()

environment.txt

platform=macOS-26.5.2-arm64-arm-64bit
machine=arm64
python=3.9.6
develop_commit=0c9f95fbc676ad2258d1dca94ba904e591500efa
branch_commit=9910a8e287fc599bed3f22e97a2ccf4fcfbf0892
stria_source=CsoundQt Examples/Chowning-Stria.csd
sample_interval_seconds=0.025

plot.py

#!/usr/bin/env python3

from __future__ import annotations

import csv
import statistics
from collections import defaultdict
from pathlib import Path

import matplotlib.pyplot as plt
import numpy as np


ROOT = Path(__file__).resolve().parent
RAW = ROOT / "raw" if (ROOT / "raw").is_dir() else ROOT
OUT = ROOT / "charts"
OUT.mkdir(exist_ok=True)

BACKGROUND = "#111217"
PANEL = "#181B1F"
GRID = "#2A2E35"
TEXT = "#D8D9DA"
MUTED = "#8E9AA7"
DEVELOP = "#FF9830"
BRANCH = "#73BF69"
RED = "#F2495C"
BLUE = "#5794F2"


def read_csv(name: str) -> list[dict[str, str]]:
    with (RAW / name).open(newline="", encoding="ascii") as source:
        return list(csv.DictReader(source))


def configure() -> None:
    plt.rcParams.update(
        {
            "figure.facecolor": BACKGROUND,
            "axes.facecolor": PANEL,
            "savefig.facecolor": BACKGROUND,
            "axes.edgecolor": GRID,
            "axes.labelcolor": TEXT,
            "axes.titlecolor": TEXT,
            "xtick.color": MUTED,
            "ytick.color": MUTED,
            "text.color": TEXT,
            "grid.color": GRID,
            "grid.alpha": 0.9,
            "font.family": "DejaVu Sans",
            "font.size": 12,
            "axes.titleweight": "normal",
        }
    )


def panel_style(axis: plt.Axes) -> None:
    axis.grid(axis="y", linewidth=0.8)
    axis.set_axisbelow(True)
    axis.spines["top"].set_visible(False)
    axis.spines["right"].set_visible(False)


def save(figure: plt.Figure, name: str) -> None:
    figure.savefig(OUT / name, dpi=180, bbox_inches="tight", pad_inches=0.25)
    plt.close(figure)


def stria_chart(summary: list[dict[str, str]]) -> None:
    rows = [row for row in summary if row["benchmark"] == "stria"]
    metrics = [
        ("wall_seconds", "Wall time", "seconds"),
        ("cpu_seconds", "CPU time", "CPU-seconds"),
    ]
    figure, axes = plt.subplots(1, 2, figsize=(14, 6.4))
    figure.subplots_adjust(top=0.72, bottom=0.18, left=0.07, right=0.98, wspace=0.20)
    figure.suptitle(
        "Stria offline synthesis: no measurable regression",
        x=0.07,
        y=0.96,
        ha="left",
        fontsize=22,
        fontweight="normal",
    )
    figure.text(
        0.07,
        0.845,
        "5 interleaved RelWithDebInfo runs per revision after warmup | synthesis only, no file I/O",
        color=MUTED,
        fontsize=12,
    )

    for axis, (key, title, unit) in zip(axes, metrics):
        panel_style(axis)
        axis.set_title(title, loc="left", pad=16, fontsize=15)
        values_by_variant = {
            variant: [
                float(row[key]) for row in rows if row["variant"] == variant
            ]
            for variant in ("develop", "branch")
        }
        all_values = values_by_variant["develop"] + values_by_variant["branch"]
        padding = max(max(all_values) - min(all_values), 0.05) * 0.8
        axis.set_ylim(min(all_values) - padding, max(all_values) + padding)
        axis.set_xlim(-0.5, 1.5)
        axis.set_xticks([0, 1], ["develop", "this branch"])
        axis.set_ylabel(unit)
        jitter = np.linspace(-0.12, 0.12, 5)

        for x, (variant, color) in enumerate(
            (("develop", DEVELOP), ("branch", BRANCH))
        ):
            values = values_by_variant[variant]
            axis.scatter(
                x + jitter,
                values,
                s=62,
                color=color,
                edgecolor=BACKGROUND,
                linewidth=1.2,
                zorder=3,
            )
            median = statistics.median(values)
            axis.hlines(median, x - 0.25, x + 0.25, color=color, linewidth=4)
            axis.text(
                x,
                median + padding * 0.30,
                f"median {median:.3f}",
                ha="center",
                color=TEXT,
                fontsize=11,
            )

        develop_median = statistics.median(values_by_variant["develop"])
        branch_median = statistics.median(values_by_variant["branch"])
        delta = (branch_median - develop_median) / develop_median * 100
        axis.text(
            0.98,
            0.06,
            f"branch delta {delta:+.2f}%",
            transform=axis.transAxes,
            ha="right",
            color=MUTED,
            fontsize=11,
        )

    figure.text(
        0.07,
        0.055,
        "LOWER IS BETTER  |  Axes are magnified to show run-to-run variance.",
        color=MUTED,
        fontsize=11,
    )
    save(figure, "stria-performance.png")


def trace_groups(traces: list[dict[str, str]], benchmark: str):
    grouped: dict[tuple[str, int], list[tuple[float, float]]] = defaultdict(list)
    for row in traces:
        if row["benchmark"] != benchmark:
            continue
        rss = float(row["rss_mb"])
        if rss > 0:
            grouped[(row["variant"], int(row["iteration"]))].append(
                (float(row["elapsed_seconds"]), rss)
            )
    return grouped


def memory_trace_chart(
    summary: list[dict[str, str]], traces: list[dict[str, str]]
) -> None:
    grouped = trace_groups(traces, "recursive-structured-workload")
    figure, axis = plt.subplots(figsize=(14, 7.2))
    figure.subplots_adjust(top=0.73, bottom=0.17, left=0.08, right=0.98)
    figure.suptitle(
        "Recursive structured workload: memory over time",
        x=0.08,
        y=0.96,
        ha="left",
        fontsize=22,
        fontweight="normal",
    )
    figure.text(
        0.08,
        0.845,
        "3 RelWithDebInfo runs per revision | sampled every 25 ms | each line is one process",
        color=MUTED,
        fontsize=12,
    )
    panel_style(axis)
    axis.set_xlabel("elapsed seconds")
    axis.set_ylabel("resident memory (MiB)")
    axis.set_xlim(0, 14)
    axis.set_ylim(0, 350)

    for variant, color in (("develop", DEVELOP), ("branch", BRANCH)):
        variant_series = []
        for iteration in range(1, 4):
            series = grouped[(variant, iteration)]
            times = np.array([item[0] for item in series])
            rss = np.array([item[1] for item in series])
            variant_series.append((times, rss))
            axis.plot(times, rss, color=color, alpha=0.28, linewidth=1.2)

        end_times = [series[0][-1] for series in variant_series]
        grid = np.arange(0, max(end_times) + 0.001, 0.025)
        interpolated = []
        for times, rss in variant_series:
            values = np.interp(grid, times, rss)
            values[grid > times[-1]] = np.nan
            interpolated.append(values)
        matrix = np.array(interpolated)
        median = np.nanmedian(matrix, axis=0)
        axis.plot(grid, median, color=color, linewidth=3.2, label=variant)

    develop_end = statistics.median(
        grouped[("develop", iteration)][-1][0] for iteration in range(1, 4)
    )
    develop_end_rss = statistics.median(
        grouped[("develop", iteration)][-1][1] for iteration in range(1, 4)
    )
    branch_end = statistics.median(
        grouped[("branch", iteration)][-1][0] for iteration in range(1, 4)
    )
    branch_end_rss = statistics.median(
        grouped[("branch", iteration)][-1][1] for iteration in range(1, 4)
    )

    axis.scatter([develop_end], [develop_end_rss], marker="x", s=120, color=RED, linewidth=3)
    axis.annotate(
        "develop: 3/3 aborted",
        (develop_end, develop_end_rss),
        xytext=(2.5, 325),
        color=TEXT,
        arrowprops={"arrowstyle": "-", "color": RED, "lw": 1.5},
    )
    axis.scatter([branch_end], [branch_end_rss], marker="o", s=70, color=BRANCH)
    axis.annotate(
        "branch: 3/3 completed",
        (branch_end, branch_end_rss),
        xytext=(10.0, 185),
        color=TEXT,
        arrowprops={"arrowstyle": "-", "color": BRANCH, "lw": 1.5},
    )
    legend = axis.legend(loc="upper right", frameon=True)
    legend.get_frame().set_facecolor(PANEL)
    legend.get_frame().set_edgecolor(GRID)
    for text in legend.get_texts():
        text.set_color(TEXT)

    figure.text(
        0.08,
        0.055,
        "LOWER IS BETTER  |  develop terminates early; the branch finishes the same workload.",
        color=MUTED,
        fontsize=11,
    )
    save(figure, "recursive-memory-over-time.png")


def outcome_chart(summary: list[dict[str, str]]) -> None:
    rows = [
        row
        for row in summary
        if row["benchmark"] == "recursive-structured-workload"
    ]
    variants = ("develop", "branch")
    colors = (DEVELOP, BRANCH)
    peaks = {
        variant: [float(row["peak_rss_mb"]) for row in rows if row["variant"] == variant]
        for variant in variants
    }
    successes = {
        variant: sum(
            int(row["return_code"]) == 0 for row in rows if row["variant"] == variant
        )
        / 3
        * 100
        for variant in variants
    }

    figure, axes = plt.subplots(1, 2, figsize=(14, 6.4))
    figure.subplots_adjust(top=0.72, bottom=0.18, left=0.07, right=0.98, wspace=0.20)
    figure.suptitle(
        "Recursive structured workload: resource and outcome",
        x=0.07,
        y=0.96,
        ha="left",
        fontsize=22,
        fontweight="normal",
    )
    figure.text(
        0.07,
        0.845,
        "Median and range of 3 RelWithDebInfo runs per revision",
        color=MUTED,
        fontsize=12,
    )

    memory_axis, outcome_axis = axes
    for axis in axes:
        panel_style(axis)
        axis.set_xticks([0, 1], ["develop", "this branch"])

    memory_axis.set_title("Peak resident memory", loc="left", pad=16, fontsize=15)
    memory_axis.set_ylabel("MiB")
    medians = [statistics.median(peaks[variant]) for variant in variants]
    error_low = [medians[index] - min(peaks[variant]) for index, variant in enumerate(variants)]
    error_high = [max(peaks[variant]) - medians[index] for index, variant in enumerate(variants)]
    memory_axis.bar([0, 1], medians, color=colors, width=0.56)
    memory_axis.errorbar(
        [0, 1],
        medians,
        yerr=np.array([error_low, error_high]),
        fmt="none",
        ecolor=TEXT,
        elinewidth=1.5,
        capsize=5,
    )
    memory_axis.set_ylim(0, 360)
    for index, value in enumerate(medians):
        memory_axis.text(index, value + 16, f"{value:.1f} MiB", ha="center")
    reduction = (medians[0] - medians[1]) / medians[0] * 100
    memory_axis.text(
        0.98,
        0.92,
        f"{reduction:.1f}% lower",
        transform=memory_axis.transAxes,
        ha="right",
        color=BRANCH,
        fontsize=13,
    )

    outcome_axis.set_title("Successful completion", loc="left", pad=16, fontsize=15)
    outcome_values = [successes[variant] for variant in variants]
    outcome_axis.bar([0, 1], outcome_values, color=colors, width=0.56)
    outcome_axis.set_ylim(0, 110)
    outcome_axis.set_ylabel("percent of runs")
    outcome_axis.text(0, 5, "0%  |  3 aborts", ha="center", color=TEXT)
    outcome_axis.text(1, outcome_values[1] + 4, "100%", ha="center")
    outcome_axis.text(1, 8, "3 completions", ha="center", color=BACKGROUND)

    figure.text(
        0.07,
        0.055,
        "MEMORY: LOWER IS BETTER  |  COMPLETION: HIGHER IS BETTER",
        color=MUTED,
        fontsize=11,
    )
    save(figure, "recursive-workload-summary.png")


def main() -> None:
    configure()
    summary = read_csv("summary.csv")
    traces = read_csv("traces.csv")
    stria_chart(summary)
    memory_trace_chart(summary, traces)
    outcome_chart(summary)


if __name__ == "__main__":
    main()

recursive-memory-over-time.png

recursive-workload-summary.png

stria-performance.png

summary.csv

traces.csv