broken-links-check.rb

#!/usr/bin/env ruby

require "uri"
require "net/http"

# dumb websites
# maps hosts to allowed >=400 http status codes
exceptions = {
  "twitter.com": 400,
  "www.fiverr.com": 403
}

check_every = 3600 / 2 # seconds

def check_broken_links interval, domain, exceptions
  puts "Checking #{domain}"
  puts "Using exceptions #{exceptions}"

  filename = "broken-links-analysis.#{domain}.log"

  stale = true
  if File.file? filename
    stale = Time.now - (File.mtime filename) > interval
  end

  if stale
    puts "Analysis file is stale, re-analyzing..."
    system "wget --spider -o #{filename} -e robots=off -w 0.25 -r -p #{domain}"
  end

  contents = ""
  File.open filename, "r:UTF-8" do |f| contents = f.read end

  broken_links_start_pattern = /^Found \d+ broken links\.$/

  broken_links = []
  reached_results = false
  contents.each_line do |line|
    if broken_links_start_pattern.match? (line.strip)
      reached_results = true
      next
    end

    if reached_results and line.start_with? "FINISHED"
      reached_results = false
      next
    end

    if reached_results and line.strip != ""
      broken_links.push URI.parse(line.strip)
    end
  end

  really_broken = []
  broken_links.each do |url|
    response = Net::HTTP.get_response url
    if response.code.to_i >= 400 
      unless exceptions.has_key? url.host.to_sym and response.code.to_i == exceptions[url.host.to_sym]
        puts "#{url} broken: got status #{response.code}"
        really_broken.push url
      end
    end
  end

  return really_broken
end

def push_kuma_status error_messages
  uptime_kuma_push_url = URI.parse "https://status.ewen.works/api/push/VhYojQ7UZU"
  push_params = {
    "status": if error_messages.empty? then "up" else "down" end,
    "msg": if error_messages.empty? then "OK" else error_messages.join ", " end,
    "ping": ""
  }

  uptime_kuma_push_url.query = URI.encode_www_form push_params.to_a

  Net::HTTP.get_response uptime_kuma_push_url
end

while true
  broken_links = []
  for domain in %w(en.ewen.works fr.ewen.works)
    (check_broken_links check_every, domain, exceptions).each do |link|
      broken_links.push "On #{domain}: #{link}"
    end
  end

  push_kuma_status broken_links
  sleep check_every
end