Compare control and candidate build runtimes

circle_scientist.rb

#!/usr/bin/env ruby

require "thor"
require "net/http"
require "json"

class CircleScientist < Thor
  option :circle_api_token, required: true
  option :control_branch, default: "master"
  option :candidate_branch, required: true
  option :allow_failures, type: :boolean, default: false
  option :step_name

  desc "compare", "Compare control and candidate build runtimes"
  def compare
    control_build_summaries = fetch_build_summaries(branch: options[:control_branch])
    candidate_build_summaries = fetch_build_summaries(branch: options[:candidate_branch])

    control_build_runtimes = fetch_build_runtimes(build_summaries: control_build_summaries)
    candidate_build_runtimes = fetch_build_runtimes(build_summaries: candidate_build_summaries)

    control_average_runtime = control_build_runtimes.sum / control_build_runtimes.size
    candidate_average_runtime = candidate_build_runtimes.sum / candidate_build_runtimes.size

    puts "control average runtime: #{control_average_runtime / 1000.0} seconds"
    puts "candidate average runtime: #{candidate_average_runtime / 1000.0} seconds"

    if candidate_average_runtime < control_average_runtime
      puts "candidate is #{100 * (control_average_runtime - candidate_average_runtime) / control_average_runtime.to_f}% faster"
    else
      puts "candidate is #{100 * (candidate_average_runtime - control_average_runtime) / control_average_runtime.to_f}% slower"
    end
  end
  default_task :compare

  no_commands do
    def fetch_build_summaries(branch:)
      uri = URI.parse("https://circleci.com/api/v1.1/project/github/intercom/intercom/tree/#{branch}")
      params = { "circle-token": options[:circle_api_token] }
      uri.query = URI.encode_www_form(params)
      response = Net::HTTP.get_response(uri)
      raise "CircleCI request failed, please check your API key" unless response.code == "200"
      JSON.parse(response.body)
    end

    def fetch_build_runtimes(build_summaries:)
      build_steps = build_summaries.map do |build_summary|
        fetch_build_details(build_num: build_summary["build_num"])
      end.select do |build|
        allowed_statuses.include?(build["status"])
      end.map do |build|
        steps_for_build(build: build)
      end

      build_steps = reject_truncated_builds(build_steps)

      build_steps.map do |steps|
        total_runtime_for_steps(steps: steps)
      end
    end

    def fetch_build_details(build_num:)
      uri = URI.parse("https://circleci.com/api/v1.1/project/github/intercom/intercom/#{build_num}")
      params = { "circle-token": options[:circle_api_token] }
      uri.query = URI.encode_www_form(params)
      response = Net::HTTP.get_response(uri)
      raise "CircleCI request failed, please check your API key" unless response.code == "200"
      JSON.parse(response.body)
    end

    def allowed_statuses
      if options[:allow_failures]
        ["failed", "success"]
      else
        ["success"]
      end
    end

    def steps_for_build(build:)
      if options[:step_name]
        build["steps"].select do |step|
          step["name"] == options[:step_name]
        end
      else
        build["steps"]
      end
    end

    def total_runtime_for_steps(steps:)
      steps.flat_map do |step|
        step["actions"]
      end.sum do |action|
        action["run_time_millis"]
      end
    end

    def reject_truncated_builds(build_steps)
      build_steps.reject { |steps| steps.size < build_steps.max_by(&:size).size }
    end
  end
end

CircleScientist.start(ARGV)

excellent