Parser + compiler benchmarking

benchmark.rb

srcdir = Dir.pwd
files = Dir.glob(File.join(srcdir, 'lib', '**', '*.rb'))
files = files.select { |f| File.file?(f) }
sources = files.map { |f| [f, File.read(f)] }

puts "#{sources.length} files, #{sources.sum { |_, s| s.bytesize }} bytes total"
puts ""

# Verify both parsers produce valid results
errors_prism = 0
errors_parsey = 0
sources.each { |f, s|
  begin; RubyVM::InstructionSequence.compile_prism(s, f); rescue => e; errors_prism += 1; end
  begin; RubyVM::InstructionSequence.compile_parsey(s, f); rescue => e; errors_parsey += 1; end
}
puts "Prism errors: #{errors_prism}, Parse.y errors: #{errors_parsey}"

# Verify both actually use the right parser
iseq_p = RubyVM::InstructionSequence.compile_prism("1+1")
iseq_y = RubyVM::InstructionSequence.compile_parsey("1+1")
puts "compile_prism parser:  #{iseq_p.to_a[4][:parser]}"
puts "compile_parsey parser: #{iseq_y.to_a[4][:parser]}"
puts ""

# Heavy warmup
3.times {
  sources.each { |f, s|
    RubyVM::InstructionSequence.compile_prism(s, f) rescue nil
    RubyVM::InstructionSequence.compile_parsey(s, f) rescue nil
  }
}
GC.start; GC.start; GC.start

n = 15

# Interleave measurements to cancel thermal drift
prism_times = []
parsey_times = []

n.times { |i|
  # Alternate which goes first
  if i.even?
    GC.disable
    t0 = Process.clock_gettime(Process::CLOCK_MONOTONIC)
    sources.each { |f, s| RubyVM::InstructionSequence.compile_prism(s, f) rescue nil }
    t1 = Process.clock_gettime(Process::CLOCK_MONOTONIC)
    GC.enable; GC.start
    prism_times << (t1 - t0)

    GC.disable
    t0 = Process.clock_gettime(Process::CLOCK_MONOTONIC)
    sources.each { |f, s| RubyVM::InstructionSequence.compile_parsey(s, f) rescue nil }
    t1 = Process.clock_gettime(Process::CLOCK_MONOTONIC)
    GC.enable; GC.start
    parsey_times << (t1 - t0)
  else
    GC.disable
    t0 = Process.clock_gettime(Process::CLOCK_MONOTONIC)
    sources.each { |f, s| RubyVM::InstructionSequence.compile_parsey(s, f) rescue nil }
    t1 = Process.clock_gettime(Process::CLOCK_MONOTONIC)
    GC.enable; GC.start
    parsey_times << (t1 - t0)

    GC.disable
    t0 = Process.clock_gettime(Process::CLOCK_MONOTONIC)
    sources.each { |f, s| RubyVM::InstructionSequence.compile_prism(s, f) rescue nil }
    t1 = Process.clock_gettime(Process::CLOCK_MONOTONIC)
    GC.enable; GC.start
    prism_times << (t1 - t0)
  end
}

prism_times.sort!
parsey_times.sort!

# Drop top 3 and bottom 3 for robust average
pt = prism_times[3..-4]
pp = parsey_times[3..-4]

prism_avg = pt.sum / pt.length
parsey_avg = pp.sum / pp.length

puts "Prism:   #{'%.3f' % prism_avg}s avg (#{'%.0f' % (sources.length / prism_avg)} files/s)"
puts "  min=#{'%.3f' % prism_times.first} median=#{'%.3f' % prism_times[prism_times.length/2]} max=#{'%.3f' % prism_times.last}"
puts "  all: [#{prism_times.map{|t|'%.3f'%t}.join(', ')}]"
puts ""
puts "Parse.y: #{'%.3f' % parsey_avg}s avg (#{'%.0f' % (sources.length / parsey_avg)} files/s)"
puts "  min=#{'%.3f' % parsey_times.first} median=#{'%.3f' % parsey_times[parsey_times.length/2]} max=#{'%.3f' % parsey_times.last}"
puts "  all: [#{parsey_times.map{|t|'%.3f'%t}.join(', ')}]"
puts ""

diff = ((prism_avg - parsey_avg) / parsey_avg * 100)
if diff > 0
  puts "Prism is #{'%.1f' % diff}% SLOWER than parse.y"
else
  puts "Prism is #{'%.1f' % diff.abs}% FASTER than parse.y"
end

# Memory comparison
puts ""
puts "=== Memory ==="
GC.start; GC.start; GC.start
GC.disable

before_prism = GC.stat(:malloc_increase_bytes)
sources.each { |f, s| RubyVM::InstructionSequence.compile_prism(s, f) rescue nil }
after_prism = GC.stat(:malloc_increase_bytes)
GC.enable; GC.start; GC.start; GC.start
GC.disable

before_parsey = GC.stat(:malloc_increase_bytes)
sources.each { |f, s| RubyVM::InstructionSequence.compile_parsey(s, f) rescue nil }
after_parsey = GC.stat(:malloc_increase_bytes)
GC.enable; GC.start

puts "Prism   malloc increase: #{(after_prism - before_prism) / 1024} KB"
puts "Parse.y malloc increase: #{(after_parsey - before_parsey) / 1024} KB"

# Peak RSS
pid = Process.pid
GC.start; GC.start; GC.start
rss_before = `ps -o rss= -p #{pid}`.strip.to_i

sources.each { |f, s| RubyVM::InstructionSequence.compile_prism(s, f) rescue nil }
rss_prism = `ps -o rss= -p #{pid}`.strip.to_i

GC.start; GC.start; GC.start
rss_middle = `ps -o rss= -p #{pid}`.strip.to_i

sources.each { |f, s| RubyVM::InstructionSequence.compile_parsey(s, f) rescue nil }
rss_parsey = `ps -o rss= -p #{pid}`.strip.to_i

puts "Prism   RSS delta:  #{rss_prism - rss_before} KB"
puts "Parse.y RSS delta:  #{rss_parsey - rss_middle} KB"