LuaCarrierPerfMatrix

LuaCarrierPerfMatrix.exs

defmodule LuaCarrierPerfMatrix do
  @moduledoc false

  @default_warmup 10
  @default_samples 60
  @default_timeout_ms 30_000
  @default_fuel 100_000_000

  @carriers [
    %{
      id: "luerl",
      name: "LuErl",
      tool: Athanor.Service.LuErl.Tool.EvalLua
    },
    %{
      id: "carrier_lua",
      name: "Native Lua carrier",
      tool: Athanor.Service.Lua.Tool.EvalLua
    },
    %{
      id: "carrier_lua_tvlabs",
      name: "Lua (TVLabs) carrier",
      tool: Athanor.Service.LuaTVLabs.Tool.EvalLua
    }
  ]

  def run(argv) do
    opts = parse_args(argv)

    Logger.configure(level: :warning)

    workspace = prepare_workspace!(Map.fetch!(opts, :workspace))
    {:ok, networking} = Athanor.Networking.start_link(workspace_name: "lua-carrier-perf")
    {:ok, native_carrier, native_socket_dir} = start_native_carrier()

    try do
      carriers = prepare_carriers(networking, workspace, native_carrier)
      workloads = workloads(workspace)

      results =
        for carrier <- carriers,
            workload <- workloads do
          measure_workload(carrier, workload, opts)
        end

      report = %{
        generated_at: DateTime.utc_now() |> DateTime.to_iso8601(),
        host: host_info(),
        options: Map.take(opts, [:warmup, :samples, :timeout_ms]),
        workloads: Enum.map(workloads, &Map.drop(&1, [:script])),
        results: results
      }

      write_outputs!(report, opts)
      print_report(report)
    after
      stop_native_carrier(native_carrier)
      File.rm_rf(native_socket_dir)
    end
  end

  defp parse_args(argv) do
    {opts, _rest, invalid} =
      OptionParser.parse(argv,
        strict: [
          samples: :integer,
          warmup: :integer,
          timeout_ms: :integer,
          output_json: :string,
          output_md: :string,
          workspace: :string
        ]
      )

    case invalid do
      [] -> :ok
      [{arg, nil} | _] -> raise ArgumentError, "unknown argument: #{arg}"
      [{arg, value} | _] -> raise ArgumentError, "invalid argument: #{arg}=#{value}"
    end

    %{
      samples: Keyword.get(opts, :samples, @default_samples),
      warmup: Keyword.get(opts, :warmup, @default_warmup),
      timeout_ms: Keyword.get(opts, :timeout_ms, @default_timeout_ms),
      output_json: Keyword.get(opts, :output_json, "tmp/lua_carrier_perf_matrix.json"),
      output_md: Keyword.get(opts, :output_md, "tmp/lua_carrier_perf_matrix.md"),
      workspace: Keyword.get(opts, :workspace, "tmp/lua_carrier_perf_workspace")
    }
  end

  defp prepare_workspace!(workspace) do
    workspace = Path.expand(workspace, File.cwd!())
    File.rm_rf!(workspace)
    File.mkdir_p!(workspace)

    read_path = Path.join(workspace, "read_4k.txt")
    File.write!(read_path, String.duplicate("abcdefgh", 512))

    dir_path = Path.join(workspace, "dir_100")
    File.mkdir_p!(dir_path)

    for index <- 1..100 do
      index = Integer.to_string(index) |> String.pad_leading(3, "0")
      File.write!(Path.join(dir_path, "entry_#{index}.txt"), index)
    end

    workspace
  end

  defp start_native_carrier do
    case :exec.start([]) do
      {:ok, _pid} -> :ok
      {:error, {:already_started, _pid}} -> :ok
    end

    socket_dir =
      Path.join(System.tmp_dir!(), "lua_carrier_perf_#{System.unique_integer([:positive])}")

    File.mkdir_p!(socket_dir)

    control_socket_path = Path.join(socket_dir, "control.sock")
    data_socket_path = Path.join(socket_dir, "data.sock")

    {:ok, pid} =
      Carrier.Lua.CarrierServer.start_link(
        control_socket_path: control_socket_path,
        data_socket_path: data_socket_path,
        boot_timeout: 60_000
      )

    {:ok, pid, socket_dir}
  end

  defp stop_native_carrier(pid) when is_pid(pid) do
    :gen_statem.stop(pid, :normal, 5_000)
  catch
    :exit, _reason -> :ok
  end

  defp prepare_carriers(networking, workspace, native_carrier) do
    Enum.map(@carriers, fn carrier ->
      {:ok, state} = init_carrier(carrier.id, carrier.tool, networking, workspace, native_carrier)
      Map.put(carrier, :state, state)
    end)
  end

  defp init_carrier("luerl", tool, networking, workspace, _native_carrier) do
    tool.init(
      allowed_paths: [workspace],
      allowed_fuel: :infinity,
      networking: networking
    )
  end

  defp init_carrier("carrier_lua", tool, networking, workspace, native_carrier) do
    tool.init(
      lua: native_carrier,
      allowed_paths: [workspace],
      allowed_fuel: @default_fuel,
      native_timeout_ms: @default_timeout_ms,
      networking: networking
    )
  end

  defp init_carrier("carrier_lua_tvlabs", tool, networking, workspace, _native_carrier) do
    tool.init(
      allowed_paths: [workspace],
      allowed_fuel: @default_fuel,
      native_timeout_ms: @default_timeout_ms,
      networking: networking
    )
  end

  defp workloads(workspace) do
    read_path = Path.join(workspace, "read_4k.txt")
    dir_path = Path.join(workspace, "dir_100")

    [
      %{
        id: "return_1",
        name: "Return constant",
        category: "eval_overhead",
        script: "return 1"
      },
      %{
        id: "sum_loop_10k",
        name: "Numeric loop 10k",
        category: "vm_arithmetic",
        script: """
        local s = 0
        for i = 1, 10000 do
          s = s + i
        end
        return tostring(s)
        """
      },
      %{
        id: "sum_loop_100k",
        name: "Numeric loop 100k",
        category: "vm_arithmetic",
        script: """
        local s = 0
        for i = 1, 100000 do
          s = s + i
        end
        return tostring(s)
        """
      },
      %{
        id: "fib_20",
        name: "Recursive fib(20)",
        category: "calls",
        script: """
        local function fib(n)
          if n < 2 then return n end
          return fib(n - 1) + fib(n - 2)
        end
        return tostring(fib(20))
        """
      },
      %{
        id: "table_build_1k",
        name: "Table build/traverse 1k",
        category: "tables",
        script: """
        local t = {}
        for i = 1, 1000 do
          t[i] = i * 3
        end
        local s = 0
        for i = 1, #t do
          s = s + t[i]
        end
        return tostring(s)
        """
      },
      %{
        id: "table_sort_1k",
        name: "Table sort 1k",
        category: "tables",
        script: """
        local t = {}
        for i = 1, 1000 do
          t[i] = (i * 1103515245 + 12345) % 65536
        end
        table.sort(t)
        return tostring(t[1] + t[#t])
        """
      },
      %{
        id: "string_concat_1k",
        name: "String concat via table.concat 1k",
        category: "strings",
        script: """
        local t = {}
        for i = 1, 1000 do
          t[i] = "abcdef"
        end
        return tostring(#table.concat(t, ","))
        """
      },
      %{
        id: "pattern_gsub_1k",
        name: "Pattern gsub 1k",
        category: "strings",
        script: """
        local s = "alpha123 beta456 gamma789"
        local out = ""
        for i = 1, 1000 do
          out = string.gsub(s, "%d+", "N")
        end
        return tostring(#out)
        """
      },
      %{
        id: "json_roundtrip",
        name: "JSON decode/encode",
        category: "host_helpers",
        script: """
        local value = json.decode('{"name":"ada","items":[1,2,3],"meta":{"ok":true}}')
        value.items[4] = 4
        return json.encode(value)
        """
      },
      %{
        id: "host_read_4k",
        name: "Host read 4 KiB",
        category: "host_io",
        script: """
        local content, err = host.read_file(#{lua_string(read_path)})
        if content == nil then error(err) end
        return tostring(#content)
        """
      },
      %{
        id: "host_list_dir_100",
        name: "Host list dir 100 entries",
        category: "host_io",
        script: """
        local entries, err = host.list_dir(#{lua_string(dir_path)})
        if entries == nil then error(err) end
        return tostring(#entries)
        """
      }
    ]
  end

  defp lua_string(value), do: Jason.encode!(value)

  defp measure_workload(carrier, workload, opts) do
    samples = Map.fetch!(opts, :samples)
    warmup = Map.fetch!(opts, :warmup)
    timeout_ms = Map.fetch!(opts, :timeout_ms)

    Enum.each(1..warmup//1, fn _sample ->
      run_eval(carrier, workload, timeout_ms)
    end)

    :erlang.garbage_collect()
    before_memory = :erlang.memory(:total)

    measured =
      Enum.map(1..samples//1, fn _sample ->
        measure_once(carrier, workload, timeout_ms)
      end)

    after_memory = :erlang.memory(:total)

    timings =
      measured
      |> Enum.filter(&match?({:ok, _micros, _result}, &1))
      |> Enum.map(fn {:ok, micros, _result} -> micros end)

    failures =
      measured
      |> Enum.filter(&match?({:error, _micros, _reason}, &1))
      |> Enum.map(fn {:error, micros, reason} -> %{micros: micros, reason: inspect(reason)} end)

    %{
      carrier: carrier.id,
      carrier_name: carrier.name,
      workload: workload.id,
      workload_name: workload.name,
      category: workload.category,
      samples: samples,
      successes: length(timings),
      failures: failures,
      timing_us: summarise(timings),
      memory_total_before: before_memory,
      memory_total_after: after_memory,
      memory_total_delta: after_memory - before_memory
    }
  end

  defp measure_once(carrier, workload, timeout_ms) do
    started = System.monotonic_time(:microsecond)

    result = run_eval(carrier, workload, timeout_ms)

    elapsed = System.monotonic_time(:microsecond) - started

    case result do
      {:ok, value} -> {:ok, elapsed, value}
      {:error, reason} -> {:error, elapsed, reason}
    end
  end

  defp run_eval(carrier, workload, timeout_ms) do
    task =
      Task.async(fn ->
        case carrier.tool.call(carrier.state, %{script: workload.script}) do
          {:finished, result, _attrs} -> {:ok, result}
          {:failed, message, attrs} -> {:error, {message, attrs}}
        end
      end)

    Task.await(task, timeout_ms + 5_000)
  catch
    :exit, {:timeout, _task} -> {:error, :benchmark_timeout}
  end

  defp summarise([]) do
    %{
      min: nil,
      p50: nil,
      p95: nil,
      p99: nil,
      max: nil,
      mean: nil
    }
  end

  defp summarise(values) do
    sorted = Enum.sort(values)

    %{
      min: List.first(sorted),
      p50: percentile(sorted, 0.50),
      p95: percentile(sorted, 0.95),
      p99: percentile(sorted, 0.99),
      max: List.last(sorted),
      mean: div(Enum.sum(sorted), length(sorted))
    }
  end

  defp percentile(sorted, fraction) do
    index = ceil(length(sorted) * fraction) - 1
    Enum.at(sorted, max(index, 0))
  end

  defp host_info do
    %{
      os: :os.type() |> Tuple.to_list() |> Enum.join("/"),
      otp_release: List.to_string(:erlang.system_info(:otp_release)),
      elixir: System.version(),
      schedulers: :erlang.system_info(:schedulers),
      schedulers_online: :erlang.system_info(:schedulers_online),
      system_architecture: to_string(:erlang.system_info(:system_architecture))
    }
  end

  defp write_outputs!(report, opts) do
    json_path = Path.expand(Map.fetch!(opts, :output_json), File.cwd!())
    md_path = Path.expand(Map.fetch!(opts, :output_md), File.cwd!())

    File.mkdir_p!(Path.dirname(json_path))
    File.mkdir_p!(Path.dirname(md_path))

    File.write!(json_path, Jason.encode!(report, pretty: true))
    File.write!(md_path, markdown(report))
  end

  defp print_report(report) do
    IO.puts(markdown(report))
  end

  defp markdown(report) do
    rows =
      Enum.map(report.results, fn result ->
        timing = result.timing_us

        [
          result.workload,
          result.carrier,
          Integer.to_string(result.successes),
          Integer.to_string(length(result.failures)),
          format_us(timing.p50),
          format_us(timing.p95),
          format_us(timing.p99),
          format_us(timing.mean)
        ]
      end)

    [
      "# Lua Carrier Performance Matrix",
      "",
      "Generated: #{report.generated_at}",
      "",
      "Host: OTP #{report.host.otp_release}, Elixir #{report.host.elixir}, schedulers #{report.host.schedulers_online}/#{report.host.schedulers}, #{report.host.system_architecture}",
      "",
      "Warmup samples: #{report.options.warmup}",
      "Measured samples: #{report.options.samples}",
      "",
      "| Workload | Carrier | OK | Fail | p50 | p95 | p99 | Mean |",
      "| --- | --- | ---: | ---: | ---: | ---: | ---: | ---: |",
      Enum.map_join(rows, "\n", fn row ->
        "| " <> Enum.join(row, " | ") <> " |"
      end),
      "",
      "All timings are wall-clock microseconds around the Athanor `eval_lua` tool call.",
      ""
    ]
    |> Enum.join("\n")
  end

  defp format_us(nil), do: "n/a"
  defp format_us(value), do: Integer.to_string(value)
end

LuaCarrierPerfMatrix.run(System.argv())