site crawler ruby script with spidr https://github.com/postmodern/spidr

crawl.rb

#!/usr/bin/env ruby

require 'rubygems'
require 'csv'
require 'spidr'

CSV.open('pets_crawl.csv', 'wb') do |csv|
    csv << ['page title', 'page url']
    Spidr.site('http://www.petsmart.com', :ignore_links => [/gsi/]) do |spider|
      spider.every_html_page do |page|
         puts "#{page.title}, #{page.url}"
         csv << [page.title, page.url]
      end
    end
end