Agent Swarm - CI Metrics code

ci-metrics.ts

/**
 * ci-metrics — generic CI metric ingestion + PR regression comment.
 *
 * Ingest any numeric metric from CI (docker image sizes, test durations, flake
 * counts, bundle sizes, ...), persist it in swarm KV (latest snapshot + dated
 * history + a capped per-metric series for trend charts), and — when the run is
 * a PR — upsert a single sticky PR comment with a table diffing the PR against
 * the base branch's latest snapshot.
 *
 * Repeated calls within the same commit MERGE (one call per CI job is fine —
 * the comment grows as jobs finish, it does not spam).
 */

const NS = "ci-metrics";
const DEFAULT_REPO = "desplega-ai/agent-swarm";
const HISTORY_TTL_SEC = 60 * 60 * 24 * 180; // 180d
const SERIES_CAP_DEFAULT = 200;
const NOISE_PCT = 0.5; // |Δ%| under this is reported as "no change"

type MetricInput = {
  name: string;
  value: number;
  unit?: string; // bytes | kb | mb | ms | s | count | percent | ratio | <free-form>
  group?: string; // section heading in the PR comment, e.g. "docker" / "tests"
  label?: string; // display name override
  higherIsBetter?: boolean; // default false (lower is better)
  meta?: Record<string, unknown>; // arbitrary context: platform, tag, digest, ...
};

type StoredMetric = MetricInput & { at: string };

type Snapshot = {
  repo: string;
  scope: string;
  branch: string | null;
  sha: string | null;
  prNumber: number | null;
  runId: string | null;
  runUrl: string | null;
  updatedAt: string;
  metrics: Record<string, StoredMetric>;
};

type Args = {
  action?: "ingest" | "query";

  // --- identity ---
  repo?: string; // "owner/name" (default desplega-ai/agent-swarm)
  branch?: string;
  sha?: string;
  prNumber?: number | string; // presence switches on PR mode
  runId?: string | number;
  runUrl?: string;
  baseBranch?: string; // default "main"

  // --- payload (any one of these shapes) ---
  metrics?: MetricInput[] | Record<string, number | MetricInput>;
  name?: string;
  value?: number | string;
  unit?: string;
  group?: string;
  label?: string;
  higherIsBetter?: boolean;
  meta?: Record<string, unknown>;

  // --- comment behaviour ---
  comment?: boolean; // default: true when prNumber is present
  commentKey?: string; // one sticky comment per key (default "default")
  title?: string; // comment heading
  dryRun?: boolean; // compute + return markdown, write nothing, comment nothing

  // --- retention ---
  seriesCap?: number; // points kept per metric series (default 200)

  // --- query mode ---
  scope?: string; // "branch/main" | "pr/1021"
  metric?: string; // series name; omit to get the latest snapshot
  limit?: number;
};

// ---------------------------------------------------------------- utilities

function sanitize(s: string): string {
  return String(s)
    .replace(/[^a-zA-Z0-9._:/%-]/g, "-")
    .replace(/^-+|-+$/g, "")
    .slice(0, 180);
}

function num(v: unknown): number | null {
  const n = typeof v === "string" ? Number(v) : (v as number);
  return typeof n === "number" && Number.isFinite(n) ? n : null;
}

async function kvGet(ctx: any, key: string): Promise<any> {
  const r: any = await ctx.swarm.kv_get({ key, namespace: NS });
  if (r?.success && r?.data && "value" in r.data) return r.data.value;
  return null;
}

async function kvSet(ctx: any, key: string, value: unknown, ttlSeconds?: number): Promise<void> {
  await ctx.swarm.kv_set({ key, value, namespace: NS, ...(ttlSeconds ? { ttlSeconds } : {}) });
}

async function kvList(ctx: any, prefix: string, limit = 100): Promise<any[]> {
  const r: any = await ctx.swarm.kv_list({ prefix, namespace: NS, limit });
  return (r?.data?.entries ?? []) as any[];
}

// ---------------------------------------------------------------- formatting

function fmtBytes(v: number): string {
  const abs = Math.abs(v);
  if (abs >= 1024 ** 3) return `${(v / 1024 ** 3).toFixed(2)} GiB`;
  if (abs >= 1024 ** 2) return `${(v / 1024 ** 2).toFixed(1)} MiB`;
  if (abs >= 1024) return `${(v / 1024).toFixed(1)} KiB`;
  return `${Math.round(v)} B`;
}

function fmtSeconds(v: number): string {
  const abs = Math.abs(v);
  if (abs >= 3600) return `${Math.floor(v / 3600)}h ${Math.round((Math.abs(v) % 3600) / 60)}m`;
  if (abs >= 60) return `${Math.floor(v / 60)}m ${Math.round(Math.abs(v) % 60)}s`;
  return `${v.toFixed(2)} s`;
}

function fmtValue(v: number, unit?: string): string {
  const u = (unit || "").toLowerCase().trim();
  if (u === "bytes" || u === "byte" || u === "b") return fmtBytes(v);
  if (u === "kb" || u === "kib") return fmtBytes(v * 1024);
  if (u === "mb" || u === "mib") return fmtBytes(v * 1024 * 1024);
  if (u === "gb" || u === "gib") return fmtBytes(v * 1024 * 1024 * 1024);
  if (u === "ms") return Math.abs(v) >= 1000 ? fmtSeconds(v / 1000) : `${Math.round(v)} ms`;
  if (u === "s" || u === "sec" || u === "secs" || u === "seconds") return fmtSeconds(v);
  if (u === "min" || u === "minutes") return fmtSeconds(v * 60);
  if (u === "percent" || u === "pct" || u === "%") return `${v.toFixed(2)}%`;
  if (u === "ratio") return v.toFixed(4);
  if (!u || u === "count" || u === "n") return Number.isInteger(v) ? String(v) : v.toFixed(2);
  return `${Number.isInteger(v) ? v : v.toFixed(2)} ${unit}`;
}

function fmtDelta(delta: number, unit?: string): string {
  const sign = delta > 0 ? "+" : delta < 0 ? "-" : "";
  return `${sign}${fmtValue(Math.abs(delta), unit)}`;
}

// ---------------------------------------------------------------- normalising

function normaliseMetrics(args: Args): MetricInput[] {
  const out: MetricInput[] = [];
  const push = (m: any) => {
    const value = num(m?.value);
    if (!m?.name || value === null) return;
    out.push({
      name: String(m.name).slice(0, 120),
      value,
      unit: m.unit ? String(m.unit) : undefined,
      group: m.group ? String(m.group) : undefined,
      label: m.label ? String(m.label) : undefined,
      higherIsBetter: m.higherIsBetter === true,
      meta: m.meta && typeof m.meta === "object" ? m.meta : undefined,
    });
  };

  const raw = args.metrics;
  if (Array.isArray(raw)) {
    for (const m of raw) push(m);
  } else if (raw && typeof raw === "object") {
    for (const [name, v] of Object.entries(raw as Record<string, any>)) {
      if (v !== null && typeof v === "object") push({ ...v, name: (v as any).name ?? name });
      else push({ name, value: v, unit: args.unit, group: args.group });
    }
  }

  if (args.name !== undefined) {
    push({
      name: args.name,
      value: args.value,
      unit: args.unit,
      group: args.group,
      label: args.label,
      higherIsBetter: args.higherIsBetter,
      meta: args.meta,
    });
  }

  return out.slice(0, 100);
}

// ---------------------------------------------------------------- comparison

type Row = {
  name: string;
  label: string;
  group: string;
  unit?: string;
  higherIsBetter: boolean;
  head: number | null;
  base: number | null;
};

function buildRows(head: Snapshot, base: Snapshot | null): Row[] {
  const names = new Set<string>([
    ...Object.keys(head.metrics || {}),
    ...Object.keys(base?.metrics || {}),
  ]);
  const rows: Row[] = [];
  for (const name of names) {
    const h = head.metrics?.[name];
    const b = base?.metrics?.[name];
    const src = h || b;
    if (!src) continue;
    rows.push({
      name,
      label: src.label || name,
      group: src.group || "",
      unit: src.unit || b?.unit,
      higherIsBetter: src.higherIsBetter === true,
      head: h ? h.value : null,
      base: b ? b.value : null,
    });
  }
  rows.sort((a, b) =>
    a.group === b.group ? a.name.localeCompare(b.name) : a.group.localeCompare(b.group),
  );
  return rows;
}

function verdict(row: Row): { icon: string; regression: boolean } {
  if (row.head === null) return { icon: "⏳", regression: false }; // not reported by this run (yet)
  if (row.base === null || row.base === 0) return { icon: "🆕", regression: false };
  const delta = row.head - row.base;
  const pct = (delta / Math.abs(row.base)) * 100;
  if (Math.abs(pct) < NOISE_PCT) return { icon: "⚪", regression: false };
  const worse = row.higherIsBetter ? delta < 0 : delta > 0;
  return { icon: worse ? "🔴" : "🟢", regression: worse };
}

function renderMarkdown(opts: {
  head: Snapshot;
  base: Snapshot | null;
  rows: Row[];
  title: string;
  commentKey: string;
  baseBranch: string;
}): { body: string; regressions: number } {
  const { head, base, rows, title, commentKey, baseBranch } = opts;
  const shortSha = (s: string | null) => (s ? s.slice(0, 7) : "?");
  const lines: string[] = [];

  const verdicts = new Map<string, { icon: string; regression: boolean }>();
  let regressions = 0;
  let improvements = 0;
  for (const r of rows) {
    const v = verdict(r);
    verdicts.set(r.name, v);
    if (v.regression) regressions++;
    else if (v.icon === "🟢") improvements++;
  }

  lines.push(`<!-- ci-metrics:${commentKey} -->`);
  lines.push(`### 📊 ${title}`);
  lines.push("");
  lines.push(
    `\`${head.branch ?? "PR"}\` @ \`${shortSha(head.sha)}\` vs \`${baseBranch}\` @ \`${shortSha(base?.sha ?? null)}\``,
  );
  lines.push("");
  if (base) {
    const summary =
      regressions > 0
        ? `🔴 **${regressions} regression${regressions === 1 ? "" : "s"}**${improvements ? ` · 🟢 ${improvements} improvement${improvements === 1 ? "" : "s"}` : ""}`
        : improvements > 0
          ? `🟢 **${improvements} improvement${improvements === 1 ? "" : "s"}**, no regressions`
          : "⚪ no significant change";
    lines.push(summary);
    lines.push("");
  }

  const groups = Array.from(new Set(rows.map((r) => r.group)));
  for (const g of groups) {
    const groupRows = rows.filter((r) => r.group === g);
    if (!groupRows.length) continue;
    if (g && groups.length > 1) lines.push(`**${g}**`);
    lines.push(`| Metric | \`${baseBranch}\` | this PR | Δ | |`);
    lines.push("| --- | ---: | ---: | ---: | :--: |");
    for (const r of groupRows) {
      const v = verdicts.get(r.name) ?? verdict(r);
      const baseCell = r.base === null ? "—" : fmtValue(r.base, r.unit);
      const headCell = r.head === null ? "—" : fmtValue(r.head, r.unit);
      let deltaCell = "—";
      if (r.head !== null && r.base !== null) {
        const delta = r.head - r.base;
        const pct = r.base !== 0 ? (delta / Math.abs(r.base)) * 100 : null;
        deltaCell =
          delta === 0
            ? "0"
            : `${fmtDelta(delta, r.unit)}${pct !== null ? ` (${pct > 0 ? "+" : ""}${pct.toFixed(2)}%)` : ""}`;
      }
      lines.push(`| \`${r.label}\` | ${baseCell} | ${headCell} | ${deltaCell} | ${v.icon} |`);
    }
    lines.push("");
  }

  if (!rows.length) lines.push("_No metrics reported yet._\n");

  const runRef = head.runUrl ? `[run](${head.runUrl})` : head.runId ? `run \`${head.runId}\`` : null;
  const footer = [
    `updated ${new Date(head.updatedAt).toISOString().replace(/\.\d+Z$/, "Z")}`,
    runRef,
    "swarm script `ci-metrics`",
  ]
    .filter(Boolean)
    .join(" · ");
  lines.push(`<sub>${footer}</sub>`);

  return { body: lines.join("\n"), regressions };
}

// ---------------------------------------------------------------- github

async function upsertPrComment(
  ctx: any,
  repo: string,
  prNumber: number,
  marker: string,
  body: string,
): Promise<{ action: string; url: string | null; id: string | null }> {
  const [owner, name] = repo.split("/");
  const q = `query($owner:String!,$name:String!,$number:Int!){
    repository(owner:$owner,name:$name){
      pullRequest(number:$number){
        id
        comments(last:100){ nodes { id url body author { login } } }
      }
    }
  }`;
  const res: any = await ctx.api.ghGraphql.graphql(q, { owner, name, number: prNumber });
  const pr = res?.repository?.pullRequest;
  if (!pr?.id) throw new Error(`PR ${repo}#${prNumber} not found`);

  const existing = (pr.comments?.nodes ?? []).find(
    (c: any) => typeof c?.body === "string" && c.body.includes(marker),
  );

  if (existing) {
    const m = `mutation($id:ID!,$body:String!){
      updateIssueComment(input:{id:$id, body:$body}){ issueComment { id url } }
    }`;
    const r: any = await ctx.api.ghGraphql.graphql(m, { id: existing.id, body });
    return {
      action: "updated",
      url: r?.updateIssueComment?.issueComment?.url ?? existing.url ?? null,
      id: existing.id ?? null,
    };
  }

  const m = `mutation($id:ID!,$body:String!){
    addComment(input:{subjectId:$id, body:$body}){ commentEdge { node { id url } } }
  }`;
  const r: any = await ctx.api.ghGraphql.graphql(m, { id: pr.id, body });
  return {
    action: "created",
    url: r?.addComment?.commentEdge?.node?.url ?? null,
    id: r?.addComment?.commentEdge?.node?.id ?? null,
  };
}

async function fetchPrRefs(
  ctx: any,
  repo: string,
  prNumber: number,
): Promise<{ branch: string | null; sha: string | null; baseBranch: string | null }> {
  const [owner, name] = repo.split("/");
  const q = `query($owner:String!,$name:String!,$number:Int!){
    repository(owner:$owner,name:$name){
      pullRequest(number:$number){ headRefName baseRefName headRefOid }
    }
  }`;
  try {
    const r: any = await ctx.api.ghGraphql.graphql(q, { owner, name, number: prNumber });
    const pr = r?.repository?.pullRequest;
    return {
      branch: pr?.headRefName ?? null,
      sha: pr?.headRefOid ?? null,
      baseBranch: pr?.baseRefName ?? null,
    };
  } catch {
    return { branch: null, sha: null, baseBranch: null };
  }
}

// ---------------------------------------------------------------- main

export default async function main(args: Args, ctx: any) {
  const a: Args = args || ({} as Args);
  const repo = sanitize(a.repo || DEFAULT_REPO);
  if (!/^[^/]+\/[^/]+$/.test(repo)) throw new Error(`repo must be "owner/name", got "${repo}"`);

  // ---------------- query mode ----------------
  if (a.action === "query") {
    const scope = sanitize(a.scope || "branch/main");
    if (a.metric) {
      const series = (await kvGet(ctx, `${repo}/${scope}/series/${sanitize(a.metric)}`)) ?? [];
      const limit = Math.min(Math.max(Number(a.limit) || 100, 1), 500);
      return { ok: true, repo, scope, metric: a.metric, points: series.slice(-limit) };
    }
    const latest = await kvGet(ctx, `${repo}/${scope}/latest`);
    const seriesKeys = (await kvList(ctx, `${repo}/${scope}/series/`, 200)).map(
      (e: any) => String(e.key).split("/series/")[1],
    );
    return { ok: true, repo, scope, latest, series: seriesKeys };
  }

  // ---------------- ingest mode ----------------
  const metrics = normaliseMetrics(a);
  if (!metrics.length) {
    throw new Error(
      'no metrics supplied — pass {name,value,unit} or {metrics:[{name,value,unit},...]} or {metrics:{"img.server":123}}',
    );
  }

  const prNumber =
    a.prNumber !== undefined && a.prNumber !== null && `${a.prNumber}` !== ""
      ? Number(a.prNumber)
      : null;
  if (prNumber !== null && !Number.isFinite(prNumber))
    throw new Error(`invalid prNumber: ${a.prNumber}`);

  let branch = a.branch ? String(a.branch) : null;
  let sha = a.sha ? String(a.sha) : null;
  let baseBranch = a.baseBranch ? String(a.baseBranch) : null;

  if (prNumber !== null && (!branch || !sha || !baseBranch)) {
    const refs = await fetchPrRefs(ctx, repo, prNumber);
    branch = branch || refs.branch;
    sha = sha || refs.sha;
    baseBranch = baseBranch || refs.baseBranch;
  }
  baseBranch = baseBranch || "main";

  const scope = prNumber !== null ? `pr/${prNumber}` : `branch/${sanitize(branch || "unknown")}`;
  const latestKey = `${repo}/${scope}/latest`;
  const now = new Date().toISOString();

  // merge into the existing snapshot when it is the same commit (one call per CI job)
  const prev: Snapshot | null = await kvGet(ctx, latestKey);
  const sameCommit = !!prev && !!sha && prev.sha === sha;
  const merged: Record<string, StoredMetric> = sameCommit ? { ...(prev?.metrics ?? {}) } : {};
  for (const m of metrics) merged[m.name] = { ...m, at: now };

  const snapshot: Snapshot = {
    repo,
    scope,
    branch,
    sha,
    prNumber,
    runId: a.runId !== undefined && a.runId !== null ? String(a.runId) : (prev?.runId ?? null),
    runUrl: a.runUrl ?? (sameCommit ? (prev?.runUrl ?? null) : null),
    updatedAt: now,
    metrics: merged,
  };

  const baselineScope = `branch/${sanitize(baseBranch)}`;
  const base: Snapshot | null =
    scope === baselineScope ? null : await kvGet(ctx, `${repo}/${baselineScope}/latest`);

  const rows = buildRows(snapshot, base);
  const title = a.title || "CI metrics";
  const commentKey = sanitize(a.commentKey || "default");
  const { body, regressions } = renderMarkdown({
    head: snapshot,
    base,
    rows,
    title,
    commentKey,
    baseBranch,
  });

  const written: string[] = [];
  if (!a.dryRun) {
    await kvSet(ctx, latestKey, snapshot);
    written.push(latestKey);

    const historyKey = `${repo}/${scope}/history/${sanitize(now)}`;
    await kvSet(ctx, historyKey, snapshot, HISTORY_TTL_SEC);
    written.push(historyKey);

    const cap = Math.min(Math.max(Number(a.seriesCap) || SERIES_CAP_DEFAULT, 10), 1000);
    for (const m of metrics) {
      const key = `${repo}/${scope}/series/${sanitize(m.name)}`;
      const series: any[] = (await kvGet(ctx, key)) ?? [];
      series.push({
        at: now,
        value: m.value,
        unit: m.unit ?? null,
        sha,
        branch,
        runId: snapshot.runId,
      });
      await kvSet(ctx, key, series.slice(-cap));
      written.push(key);
    }
  }

  let comment: { action: string; url: string | null; id: string | null } | null = null;
  const wantComment = a.comment !== undefined ? a.comment === true : prNumber !== null;
  if (wantComment && prNumber !== null && !a.dryRun) {
    comment = await upsertPrComment(ctx, repo, prNumber, `<!-- ci-metrics:${commentKey} -->`, body);
  }

  return {
    ok: true,
    repo,
    scope,
    baselineScope,
    sha,
    branch,
    prNumber,
    dryRun: a.dryRun === true,
    metricsIngested: metrics.map((m) => m.name),
    metricsTracked: Object.keys(merged).length,
    hasBaseline: !!base,
    regressions,
    comment,
    keysWritten: written,
    markdown: body,
  };
}